diff --git a/.coveragerc b/.coveragerc
index 249fcf15ddb4b..3fa052108477e 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -23,3 +23,4 @@ omit =
     scripts/*
     dev/*
     airflow/migrations/*
+    airflow/www_rbac/node_modules/**
diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000000000..368fdb4331a12
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 110
+ignore = E731,W504
+exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*/_vendor/*
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 6000d0e5ff450..539ca28fca423 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -30,4 +30,4 @@ Make sure you have checked _all_ steps below.
 
 
 ### Code Quality
-- [ ] Passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
+- [ ] Passes `flake8`
diff --git a/.gitignore b/.gitignore
index 0e53aaded07b6..0ccc81842735e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,6 +83,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/_api/
 
 # PyBuilder
 target/
@@ -138,3 +139,22 @@ rat-results.txt
 *.generated
 *.tar.gz
 scripts/ci/kubernetes/kube/.generated/airflow.yaml
+
+# Node & Webpack Stuff
+*.entry.js
+node_modules
+npm-debug.log*
+static/dist
+derby.log
+metastore_db
+
+# Airflow log files when airflow is run locally
+airflow-*.err
+airflow-*.out
+airflow-*.log
+airflow-*.pid
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
diff --git a/.rat-excludes b/.rat-excludes
index e5373aacd5fbc..fb9361f7f39fc 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -1,16 +1,21 @@
+# Note: these patterns are applied to single files or directories, not full paths
+# coverage/* will ignore any coverage dir, but airflow/www/static/coverage/* will match nothing
+
 .gitignore
 .gitattributes
+.airflowignore
 .coverage
 .coveragerc
 .codecov.yml
+.eslintrc
+.eslintignore
+.flake8
 .rat-excludes
 requirements.txt
 .*log
 .travis.yml
 .*pyc
 .*lock
-docs
-.*md
 dist
 build
 airflow.egg-info
@@ -18,16 +23,23 @@ apache_airflow.egg-info
 .idea
 metastore_db
 .*sql
+.*svg
 .*csv
 CHANGELOG.txt
 .*zip
 .*lock
+# Generated doc files
+.*html
+_build/*
+_static/*
+.buildinfo
+searchindex.js
+
 # Apache Rat does not detect BSD-2 clause properly
 # it is compatible according to http://www.apache.org/legal/resolved.html#category-a
 kerberos_auth.py
 airflow_api_auth_backend_kerberos_auth_py.html
 licenses/*
-airflow/www/static/docs
 parallel.js
 underscore.js
 jquery.dataTables.min.js
@@ -37,3 +49,16 @@ bootstrap-toggle.min.js
 bootstrap-toggle.min.css
 d3.v3.min.js
 ace.js
+node_modules/*
+.*json
+coverage/*
+git_version
+flake8_diff.sh
+coverage.xml
+
+rat-results.txt
+apache-airflow-.*\+source.tar.gz.*
+apache-airflow-.*\+bin.tar.gz.*
+
+# vendored modules
+_vendor/*
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 643548c03e07a..c6a4da8d690c8 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,4 +22,4 @@ python:
         - doc
         - docker
         - gcp_api
-        - emr 
+        - emr
diff --git a/.travis.yml b/.travis.yml
index 01c08d9537f64..594bbfd8288af 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,96 +16,73 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-sudo: true
-dist: trusty
+dist: xenial
 language: python
-jdk:
-  - oraclejdk8
-services:
-  - cassandra
-  - mysql
-  - postgresql
-  - rabbitmq
-addons:
-  apt:
-    packages:
-      - slapd
-      - ldap-utils
-      - openssh-server
-      - mysql-server-5.6
-      - mysql-client-core-5.6
-      - mysql-client-5.6
-      - krb5-user
-      - krb5-kdc
-      - krb5-admin-server
-      - oracle-java8-installer
-      - python-selinux
-  postgresql: "9.2"
 python:
-  - "2.7"
-  - "3.5"
+  - "3.6"
 env:
   global:
     - TRAVIS_CACHE=$HOME/.travis_cache/
-    - KRB5_CONFIG=/etc/krb5.conf
-    - KRB5_KTNAME=/etc/airflow.keytab
-    # Travis on google cloud engine has a global /etc/boto.cfg that
-    # does not work with python 3
-    - BOTO_CONFIG=/tmp/bogusvalue
   matrix:
-    - TOX_ENV=py27-backend_mysql
-    - TOX_ENV=py27-backend_sqlite
-    - TOX_ENV=py27-backend_postgres
-    - TOX_ENV=py35-backend_mysql
-    - TOX_ENV=py35-backend_sqlite
-    - TOX_ENV=py35-backend_postgres
-    - TOX_ENV=flake8
-    - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
-    - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0
-matrix:
-  exclude:
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_mysql
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_sqlite
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_postgres
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_mysql
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_sqlite
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_postgres
-    - python: "2.7"
-      env: TOX_ENV=flake8
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
-    - python: "2.7"
-      env: TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0
+    - TOX_ENV=py27-backend_mysql-env_docker
+    - TOX_ENV=py27-backend_sqlite-env_docker
+    - TOX_ENV=py27-backend_postgres-env_docker
+    - TOX_ENV=py35-backend_mysql-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py35-backend_sqlite-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py35-backend_postgres-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py27-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.9.0
+    - TOX_ENV=py35-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.13.0 PYTHON_VERSION=3
+
+stages:
+  - pre-test
+  - test
+
+jobs:
+  include:
+    - name: Flake8
+      stage: pre-test
+      install: pip install flake8
+      script: flake8
+    - name: mypy
+      stage: pre-test
+      install: pip install mypy
+      script: mypy airflow tests
+    - name: Check license header
+      stage: pre-test
+      install: skip
+      script: scripts/ci/6-check-license.sh
+    - name: Check docs
+      stage: pre-test
+      install: pip install -e .[doc]
+      script: docs/build.sh
+
 cache:
   directories:
     - $HOME/.wheelhouse/
+    - $HOME/.cache/pip
     - $HOME/.travis_cache/
 before_install:
-  - yes | ssh-keygen -t rsa -C your_email@youremail.com -P '' -f ~/.ssh/id_rsa
-  - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-  - ln -s ~/.ssh/authorized_keys ~/.ssh/authorized_keys2
-  - chmod 600 ~/.ssh/*
-  - jdk_switcher use oraclejdk8
-install:
-  - pip install --upgrade pip
-  - pip install tox
-  - pip install codecov
-before_script:
-  - cat "$TRAVIS_BUILD_DIR/scripts/ci/my.cnf" | sudo tee -a /etc/mysql/my.cnf
-  - mysql -e 'drop database if exists airflow; create database airflow' -u root
-  - sudo service mysql restart
-  - psql -c 'create database airflow;' -U postgres
-  - export PATH=${PATH}:/tmp/hive/bin
   # Required for K8s v1.10.x. See
   # https://github.com/kubernetes/kubernetes/issues/61058#issuecomment-372764783
-  - sudo mount --make-shared / && sudo service docker restart
+  - if [ ! -z "$KUBERNETES_VERSION" ]; then sudo mount --make-shared / && sudo service docker restart; fi
+install:
+  - pip install --upgrade pip
+  - docker-compose -f scripts/ci/docker-compose.yml pull --quiet --parallel
 script:
-  - ./scripts/ci/travis_script.sh
-after_success:
-  - codecov
+  - if [ -z "$KUBERNETES_VERSION" ]; then
+      docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+    fi
+  - if [ ! -z "$KUBERNETES_VERSION" ]; then
+      ./scripts/ci/kubernetes/minikube/stop_minikube.sh &&
+      ./scripts/ci/kubernetes/setup_kubernetes.sh &&
+      ./scripts/ci/kubernetes/kube/deploy.sh -d persistent_mode &&
+      MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+    fi
+  - if [ ! -z "$KUBERNETES_VERSION" ]; then
+      ./scripts/ci/kubernetes/minikube/stop_minikube.sh &&
+      ./scripts/ci/kubernetes/setup_kubernetes.sh &&
+      ./scripts/ci/kubernetes/kube/deploy.sh -d git_mode &&
+      MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+    fi
+before_cache:
+  - sudo chown -R travis:travis $HOME/.cache/pip $HOME/.wheelhouse/
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index fa4e6547a7e1d..02221bfe0de65 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,920 +1,2438 @@
-AIRFLOW 1.9.0, 2018-01-02
+Airflow 1.10.3, 2019-04-09
+--------------------------
+
+New Feature
+"""""""""""
+
+- [AIRFLOW-4232] Add ``none_skipped`` trigger rule (#5032)
+- [AIRFLOW-3971] Add Google Cloud Natural Language operators (#4980)
+- [AIRFLOW-4069] Add Opsgenie Alert Hook and Operator (#4903)
+- [AIRFLOW-3552] Fix encoding issue in ImapAttachmentToS3Operator (#5040)
+- [AIRFLOW-3552] Add ImapToS3TransferOperator (#4476)
+- [AIRFLOW-1526] Add dingding hook and operator (#4895)
+- [AIRFLOW-3490] Add BigQueryHook's Ability to Patch Table/View (#4299)
+- [AIRFLOW-3918] Add ssh private-key support to git-sync for KubernetesExecutor (#4777)
+- [AIRFLOW-3659] Create Google Cloud Transfer Service Operators (#4792)
+- [AIRFLOW-3939] Add Google Cloud Translate operator (#4755)
+- [AIRFLOW-3541] Add Avro logical type conversion to bigquery hook (#4553)
+- [AIRFLOW-4106] instrument staving tasks in pool (#4927)
+- [AIRFLOW-2568] Azure Container Instances operator (#4121)
+- [AIRFLOW-4107] instrument executor (#4928)
+- [AIRFLOW-4033] record stats of task duration (#4858)
+- [AIRFLOW-3892] Create Redis pub sub sensor (#4712)
+- [AIRFLOW-4124] add get_table and get_table_location in aws_glue_hook and tests (#4942)
+- [AIRFLOW-1262] Adds missing docs for email configuration (#4557)
+- [AIRFLOW-3701] Add Google Cloud Vision Product Search operators (#4665)
+- [AIRFLOW-3766] Add support for kubernetes annotations (#4589)
+- [AIRFLOW-3741] Add extra config to Oracle hook (#4584)
+- [AIRFLOW-1262] Allow configuration of email alert subject and body (#2338)
+- [AIRFLOW-2985] Operators for S3 object copying/deleting (#3823)
+- [AIRFLOW-2993] s3_to_sftp and sftp_to_s3 operators (#3828)
+- [AIRFLOW-3799] Add compose method to GoogleCloudStorageHook (#4641)
+- [AIRFLOW-3218] add support for poking a whole DAG (#4058)
+- [AIRFLOW-3315] Add ImapAttachmentSensor (#4161)
+- [AIRFLOW-2780] Add IMAP Hook to retrieve email attachments (#4119)
+- [AIRFLOW-3556] Add cross join set dependency function (#4356)
+
+Improvement
+"""""""""""
+
+- [AIRFLOW-4120] Modify SchedulerJob.manage_slas to respect zero timedelta SLAs (#4939)
+- [AIRFLOW-3823] Exclude branch's downstream tasks from the tasks to skip (#4666)
+- [AIRFLOW-3274] Add run_as_user and fs_group options for Kubernetes (#4648)
+- [AIRFLOW-4247] Template Region on the DataprocOperators (#5046)
+- [AIRFLOW-4008] add envFrom for Kubernetes Executor (#4952)
+- [AIRFLOW-3947] Flash msg for no DAG-level access error (#4767)
+- [AIRFLOW-3287] Moving database clean-up code into the CoreTest.tearDown() (#4122)
+- [AIRFLOW-4058] Name models test file to get automatically picked up (#4901)
+- [AIRFLOW-3830] Remove DagBag from /dag_details (#4831)
+- [AIRFLOW-3596] Clean up undefined template variables. (#4401)
+- [AIRFLOW-3573] Remove DagStat table (#4378)
+- [AIRFLOW-3623] Fix bugs in Download task logs (#5005)
+- [AIRFLOW-4173] Improve SchedulerJob.process_file() (#4993)
+- [AIRFLOW-3540] Warn if old airflow.cfg file is found (#5006)
+- [AIRFLOW-4000] Return response when no file (#4822)
+- [AIRFLOW-3383] Rotate fernet keys. (#4225)
+- [AIRFLOW-3003] Pull the krb5 image instead of building (#3844)
+- [AIRFLOW-3862] Check types with mypy. (#4685)
+- [AIRFLOW-251] Add option SQL_ALCHEMY_SCHEMA parameter to specify schema for metadata (#4199)
+- [AIRFLOW-1814] Temple PythonOperator {op_args,op_kwargs} fields (#4691)
+- [AIRFLOW-3730] Standarization use of logs mechanisms (#4556)
+- [AIRFLOW-3770] Validation of documentation on CI] (#4593)
+- [AIRFLOW-3866] Run docker-compose pull silently in CI (#4688)
+- [AIRFLOW-3685] Move licence header check (#4497)
+- [AIRFLOW-3670] Add stages to Travis build (#4477)
+- [AIRFLOW-3937] KubernetesPodOperator support for envFrom configMapRef and secretRef (#4772)
+- [AIRFLOW-3408] Remove outdated info from Systemd Instructions (#4269)
+- [AIRFLOW-3202] add missing documentation for AWS hooks/operator (#4048)
+- [AIRFLOW-3908] Add more Google Cloud Vision operators (#4791)
+- [AIRFLOW-2915] Add example DAG for GoogleCloudStorageToBigQueryOperator (#3763)
+- [AIRFLOW-3062] Add Qubole in integration docs (#3946)
+- [AIRFLOW-3288] Add SNS integration (#4123)
+- [AIRFLOW-3148] Remove unnecessary arg "parameters" in RedshiftToS3Transfer (#3995)
+- [AIRFLOW-3049] Add extra operations for Mongo hook (#3890)
+- [AIRFLOW-3559] Add missing options to DatadogHook. (#4362)
+- [AIRFLOW-1191] Simplify override of spark submit command. (#4360)
+- [AIRFLOW-3155] Add ability to filter by a last modified time in GCS Operator (#4008)
+- [AIRFLOW-2864] Fix docstrings for SubDagOperator (#3712)
+- [AIRFLOW-4062] Improve docs on install extra package commands (#4966)
+- [AIRFLOW-3743] Unify different methods of working out AIRFLOW_HOME (#4705)
+- [AIRFLOW-4002] Option to open debugger on errors in `airflow test`. (#4828)
+- [AIRFLOW-3997] Extend Variable.get so it can return None when var not found (#4819)
+- [AIRFLOW-4009] Fix docstring issue in GCSToBQOperator (#4836)
+- [AIRFLOW-3980] Unify logger (#4804)
+- [AIRFLOW-4076] Correct port type of beeline_default in init_db (#4908)
+- [AIRFLOW-4046] Add validations for poke_interval & timeout for Sensor (#4878)
+- [AIRFLOW-3744] Abandon the use of obsolete aliases of methods (#4568)
+- [AIRFLOW-3865] Add API endpoint to get python code of dag by id (#4687)
+- [AIRFLOW-3516] Support to create k8 worker pods in batches (#4434)
+- [AIRFLOW-2843] Add flag in ExternalTaskSensor to check if external DAG/task exists (#4547)
+- [AIRFLOW-2224] Add support CSV files in MySqlToGoogleCloudStorageOperator (#4738)
+- [AIRFLOW-3895] GoogleCloudStorageHook/Op create_bucket takes optional resource params (#4717)
+- [AIRFLOW-3950] Improve AirflowSecurityManager.update_admin_perm_view (#4774)
+- [AIRFLOW-4006] Make better use of Set in AirflowSecurityManager (#4833)
+- [AIRFLOW-3917] Specify alternate kube config file/context when running out of cluster (#4859)
+- [AIRFLOW-3911] Change Harvesting DAG parsing results to DEBUG log level (#4729)
+- [AIRFLOW-3584] Use ORM DAGs for index view. (#4390)
+- [AIRFLOW-2821] Refine Doc "Plugins" (#3664)
+- [AIRFLOW-3561] Improve queries (#4368)
+- [AIRFLOW-3600] Remove dagbag from trigger (#4407)
+- [AIRFLOW-3713] Updated documentation for GCP optional project_id (#4541)
+- [AIRFLOW-2767] - Upgrade gunicorn to 19.5.0 to avoid moderate-severity CVE (#4795)
+- [AIRFLOW-3795] provide_context param is now used (#4735)
+- [AIRFLOW-4012]  - Upgrade tabulate to 0.8.3 (#4838)
+- [AIRFLOW-3623] Support download logs by attempts from UI (#4425)
+- [AIRFLOW-2715] Use region setting when launching Dataflow templates (#4139)
+- [AIRFLOW-3932] Update unit tests and documentation for safe mode flag. (#4760)
+- [AIRFLOW-3932] Optionally skip dag discovery heuristic. (#4746)
+- [AIRFLOW-3258] K8S executor environment variables section. (#4627)
+- [AIRFLOW-3931] set network, subnetwork when launching dataflow template (#4744)
+- [AIRFLOW-4095] Add template_fields for S3CopyObjectOperator & S3DeleteObjectsOperator (#4920)
+- [AIRFLOW-2798] Remove needless code from models.py
+- [AIRFLOW-3731] Constrain mysqlclient to <1.4 (#4558)
+- [AIRFLOW-3139] include parameters into log.info in SQL operators, if any (#3986)
+- [AIRFLOW-3174] Refine Docstring for SQL Operators & Hooks (#4043)
+- [AIRFLOW-3933] Fix various typos (#4747)
+- [AIRFLOW-3905] Allow using "parameters" in SqlSensor (#4723)
+- [AIRFLOW-2761] Parallelize enqueue in celery executor (#4234)
+- [AIRFLOW-3540] Respect environment config when looking up config file. (#4340)
+- [AIRFLOW-2156] Parallelize Celery Executor task state fetching (#3830)
+- [AIRFLOW-3702] Add backfill option to run backwards (#4676)
+- [AIRFLOW-3821] Add replicas logic to GCP SQL example DAG (#4662)
+- [AIRFLOW-3547] Fixed Jinja templating in SparkSubmitOperator (#4347)
+- [AIRFLOW-3647] Add archives config option to SparkSubmitOperator (#4467)
+- [AIRFLOW-3802] Updated documentation for HiveServer2Hook (#4647)
+- [AIRFLOW-3817] - Corrected task ids returned by BranchPythonOperator to match the dummy operator ids (#4659)
+- [AIRFLOW-3782] Clarify docs around celery worker_autoscale in default_airflow.cfg (#4609)
+- [AIRFLOW-1945] Add Autoscale config for Celery workers (#3989)
+- [AIRFLOW-3590]: Change log message of executor exit status (#4616)
+- [AIRFLOW-3591] Fix start date, end date, duration for rescheduled tasks (#4502)
+- [AIRFLOW-3709] Validate `allowed_states` for ExternalTaskSensor (#4536)
+- [AIRFLOW-3522] Add support for sending Slack attachments (#4332)
+- [AIRFLOW-3569] Add "Trigger DAG" button in DAG page (/www only) (#4373)
+- [AIRFLOW-3569] Add "Trigger DAG" button in DAG page (/www_rbac only) (#4373)
+- [AIRFLOW-3044] Dataflow operators accept templated job_name param (#3887)
+- [AIRFLOW-3023] Fix docstring datatypes
+- [AIRFLOW-2928] Use uuid4 instead of uuid1 (#3779)
+- [AIRFLOW-2988] Run specifically python2 for dataflow (#3826)
+- [AIRFLOW-3697] Vendorize nvd3 and slugify (#4513)
+- [AIRFLOW-3692] Remove ENV variables to avoid GPL (#4506)
+- [AIRFLOW-3907] Upgrade flask and set cookie security flags. (#4725)
+- [AIRFLOW-3698] Add documentation for AWS Connection (#4514)
+- [AIRFLOW-3616][AIRFLOW-1215] Add aliases for schema with underscore (#4523)
+- [AIRFLOW-3375] Support returning multiple tasks with BranchPythonOperator (#4215)
+- [AIRFLOW-3742] Fix handling of "fallback" for AirflowConfigParsxer.getint/boolean (#4674)
+- [AIRFLOW-3742] Respect the `fallback` arg in airflow.configuration.get (#4567)
+- [AIRFLOW-3789] Fix flake8 3.7 errors. (#4617)
+- [AIRFLOW-3602] Improve ImapHook handling of retrieving no attachments (#4475)
+- [AIRFLOW-3631] Update flake8 and fix lint. (#4436)
+
+Bug fixes
+"""""""""
+
+- [AIRFLOW-2994] Fix command status check in Qubole Check operator (#3790)
+- [AIRFLOW-2563] Fix PigCliHook Python 3 string/bytes use
+- [AIRFLOW-4248] Fix 'FileExistsError' makedirs race in file_processor_handler (#5047)
+- [AIRFLOW-4240] State-changing actions should be POST requests (#5039)
+- [AIRFLOW-4246] Flask-Oauthlib needs downstream dependencies pinning due to breaking changes (#5045)
+- [AIRFLOW-3887] Downgrade dagre-d3 to 0.4.18 (#4713)
+- [AIRFLOW-3419] Fix S3Hook.select_key on Python3 (#4970)
+- [AIRFLOW-4127] Correct AzureContainerInstanceHook._get_instance_view's return (#4945)
+- [AIRFLOW-4172] Fix changes for driver class path option in Spark Submit (#4992)
+- [AIRFLOW-3615] Preserve case of UNIX socket paths in Connections (#4591)
+- [AIRFLOW-3417] ECSOperator: pass platformVersion only for FARGATE launch type (#4256)
+- [AIRFLOW-3884] Fixing doc checker, no warnings allowed anymore and fixed the current… (#4702)
+- [AIRFLOW-2652] implement / enhance baseOperator deepcopy
+- [AIRFLOW-4001] Update docs about how to run tests (#4826)
+- [AIRFLOW-3699] Speed up Flake8 (#4515)
+- [AIRFLOW-4160] Fix redirecting of 'Trigger Dag' Button in DAG Page (#4982)
+- [AIRFLOW-3650] Skip running on mysql for the flaky test (#4457)
+- [AIRFLOW-3423] Fix mongo hook to work with anonymous access (#4258)
+- [AIRFLOW-3982] Fix race condition in CI test (#4968)
+- [AIRFLOW-3982] Update DagRun state based on its own tasks (#4808)
+- [AIRFLOW-3737] Kubernetes executor cannot handle long dag/task names (#4636)
+- [AIRFLOW-3945] Stop inserting row when permission views unchanged (#4764)
+- [AIRFLOW-4123] Add Exception handling for _change_state method in K8 Executor (#4941)
+- [AIRFLOW-3771] Minor refactor securityManager (#4594)
+- [AIRFLOW-987] pass kerberos cli args keytab and principal to kerberos.run() (#4238)
+- [AIRFLOW-3736] Allow int value in SqoopOperator.extra_import_options(#4906)
+- [AIRFLOW-4063] Fix exception string in BigQueryHook [2/2] (#4902)
+- [AIRFLOW-4063] Fix exception string in BigQueryHook (#4899)
+- [AIRFLOW-4037] Log response in SimpleHttpOperator even if the response check fails
+- [AIRFLOW-4044] The documentation of `query_params` in `BigQueryOperator` is wrong.  (#4876)
+- [AIRFLOW-4015] Make missing API endpoints available in classic mode
+- [AIRFLOW-3153] Send DAG processing stats to statsd (#4748)
+- [AIRFLOW-2966] Catch ApiException in the Kubernetes Executor (#4209)
+- [AIRFLOW-4129] Escape HTML in generated tooltips (#4950)
+- [AIRFLOW-4070] AirflowException -> log.warning for duplicate task dependencies (#4904)
+- [AIRFLOW-4054] Fix assertEqualIgnoreMultipleSpaces util & add tests (#4886)
+- [AIRFLOW-3239] Fix test recovery further (#4074)
+- [AIRFLOW-4053] Fix KubePodOperator Xcom on Kube 1.13.0 (#4883)
+- [AIRFLOW-2961] Refactor tests.BackfillJobTest.test_backfill_examples test (#3811)
+- [AIRFLOW-3606] Fix Flake8 test & fix the Flake8 errors introduced since Flake8 test was broken (#4415)
+- [AIRFLOW-3543] Fix deletion of DAG with rescheduled tasks (#4646)
+- [AIRFLOW-2548] Output plugin import errors to web UI (#3930)
+- [AIRFLOW-4019] Fix AWS Athena Sensor object has no attribute 'mode' (#4844)
+- [AIRFLOW-3758] Fix circular import in WasbTaskHandler (#4601)
+- [AIRFLOW-3706] Fix tooltip max-width by correcting ordering of CSS files (#4947)
+- [AIRFLOW-4100] Correctly JSON escape data for tree/graph views (#4921)
+- [AIRFLOW-3636] Fix a test introduced in #4425 (#4446)
+- [AIRFLOW-3977] Add examples of trigger rules in doc (#4805)
+- [AIRFLOW-2511] Fix improper failed session commit handling causing deadlocks (#4769)
+- [AIRFLOW-3962] Added graceful handling for creation of dag_run of a dag which doesn't have any task (#4781)
+- [AIRFLOW-3881] Correct to_csv row number (#4699)
+- [AIRFLOW-3875] Simplify SlackWebhookHook code and change docstring (#4696)
+- [AIRFLOW-3733] Don't raise NameError in HQL hook to_csv when no rows returned (#4560)
+- [AIRFLOW-3734] Fix hql not run when partition is None (#4561)
+- [AIRFLOW-3767] Correct bulk insert function (#4773)
+- [AIRFLOW-4087] remove sudo in basetaskrunner on_finish (#4916)
+- [AIRFLOW-3768] Escape search parameter in pagination controls (#4911)
+- [AIRFLOW-4045] Fix hard-coded URLs in FAB-based UI (#4914)
+- [AIRFLOW-3123] Use a stack for DAG context management (#3956)
+- [AIRFLOW-3060] DAG context manager fails to exit properly in certain circumstances
+- [AIRFLOW-3924] Fix try number in alert emails (#4741)
+- [AIRFLOW-4083] Add tests for link generation utils (#4912)
+- [AIRFLOW-2190] Send correct HTTP status for base_url not found (#4910)
+- [AIRFLOW-4015] Add get_dag_runs GET endpoint to "classic" API (#4884)
+- [AIRFLOW-3239] Enable existing CI tests (#4131)
+- [AIRFLOW-1390] Update Alembic to 0.9 (#3935)
+- [AIRFLOW-3885] Fix race condition in scheduler test (#4737)
+- [AIRFLOW-3885] ~10x speed-up of SchedulerJobTest suite (#4730)
+- [AIRFLOW-3780] Fix some incorrect when base_url is used (#4643)
+- [AIRFLOW-3807] Fix Graph View Highlighting of Tasks (#4653)
+- [AIRFLOW-3009] Import Hashable from collection.abc to fix Python 3.7 deprecation warning (#3849)
+- [AIRFLOW-2231] Fix relativedelta DAG schedule_interval (#3174)
+- [AIRFLOW-2641] Fix MySqlToHiveTransfer to handle MySQL DECIMAL correctly
+- [AIRFLOW-3751] Option to allow malformed schemas for LDAP authentication (#4574)
+- [AIRFLOW-2888] Add deprecation path for task_runner config change (#4851)
+- [AIRFLOW-2930] Fix celery excecutor scheduler crash (#3784)
+- [AIRFLOW-2888] Remove shell=True and bash from task launch (#3740)
+- [AIRFLOW-3885] ~2.5x speed-up for backfill tests (#4731)
+- [AIRFLOW-3885] ~20x speed-up of slowest unit test (#4726)
+- [AIRFLOW-2508] Handle non string types in Operators templatized fields (#4292)
+- [AIRFLOW-3792] Fix validation in BQ for useLegacySQL & queryParameters (#4626)
+- [AIRFLOW-3749] Fix Edit Dag Run page when using RBAC (#4613)
+- [AIRFLOW-3801] Fix DagBag collect dags invocation to prevent examples to be loaded (#4677)
+- [AIRFLOW-3774] Register blueprints with RBAC web app (#4598)
+- [AIRFLOW-3719] Handle StopIteration in CloudWatch logs retrieval (#4516)
+- [AIRFLOW-3108] Define get_autocommit method for MsSqlHook (#4525)
+- [AIRFLOW-3074] Add relevant ECS options to ECS operator. (#3908)
+- [AIRFLOW-3353] Upgrade Redis client (#4834)
+- [AIRFLOW-3250] Fix for Redis Hook for not authorised connection calls (#4090)
+- [AIRFLOW-2009] Fix dataflow hook connection-id (#4563)
+- [AIRFLOW-2190] Fix TypeError when returning 404 (#4596)
+- [AIRFLOW-2876] Update Tenacity to 4.12 (#3723)
+- [AIRFLOW-3923] Update flask-admin dependency to 1.5.3 to resolve security vulnerabilities from safety (#4739)
+- [AIRFLOW-3683] Fix formatting of error message for invalid TriggerRule (#4490)
+- [AIRFLOW-2787] Allow is_backfill to handle NULL DagRun.run_id (#3629)
+- [AIRFLOW-3780] Fix some incorrect when base_url is used (#4643)
+- [AIRFLOW-3639] Fix request creation in Jenkins Operator (#4450)
+- [AIRFLOW-3779] Don't install enum34 backport when not needed (#4620)
+- [AIRFLOW-3079] Improve migration scripts to support MSSQL Server (#3964)
+- [AIRFLOW-2735] Use equality, not identity, check for detecting AWS Batch failures[]
+- [AIRFLOW-2706] AWS Batch Operator should use top-level job state to determine status
+- [AIRFLOW-XXX] Fix typo in http_operator.py
+- [AIRFLOW-XXX] Solve lodash security warning (#4820)
+- [AIRFLOW-XXX] Pin version of tornado pulled in by Celery. (#4815)
+- [AIRFLOW-XXX] Upgrade FAB to 1.12.3 (#4694)
+- [AIRFLOW-XXX] Pin pinodb dependency (#4704)
+- [AIRFLOW-XXX] Pin version of Pip in tests to work around pypa/pip#6163 (#4576)
+- [AIRFLOW-XXX] Fix spark submit hook KeyError (#4578)
+- [AIRFLOW-XXX] Pin psycopg2 due to breaking change (#5036)
+- [AIRFLOW-XXX] Pin Sendgrid dep. (#5031)
+- [AIRFLOW-XXX] Fix flaky test - test_execution_unlimited_parallelism (#4988)
+
+Misc/Interal
+""""""""""""
+
+- [AIRFLOW-4016] Clear runs for BackfillJobTest (#4839)
+- [AIRFLOW-4177] Check types in tests
+- [AIRFLOW-4144] add description of is_delete_operator_pod (#4943)
+- [AIRFLOW-3476,3477] Move Kube classes out of models.py (#4443)
+- [AIRFLOW-3464] Move SkipMixin out of models.py (#4386)
+- [AIRFLOW-3463] Move Log out of models.py (#4639)
+- [AIRFLOW-3458] Move connection tests (#4680)
+- [AIRFLOW-3461] Move TaskFail out of models.py (#4630)
+- [AIRFLOW-3462] Move TaskReschedule out of models.py (#4618)
+- [AIRFLOW-3474] Move SlaMiss out of models.py (#4608)
+- [AIRFLOW-3475] Move ImportError out of models.py (#4383)
+- [AIRFLOW-3459] Move DagPickle to separate file (#4374)
+- [AIRFLOW-3925] Don't pull docker-images on pretest (#4740)
+- [AIRFLOW-4154] Correct string formatting in jobs.py (#4972)
+- [AIRFLOW-3458] Deprecation path for moving models.Connection
+- [AIRFLOW-3458] Move models.Connection into separate file (#4335)
+- [AIRFLOW-XXX] Remove old/non-test files that nose ignores (#4930)
+
+Doc-only changes
+""""""""""""""""
+
+- [AIRFLOW-3996] Add view source link to included fragments
+- [AIRFLOW-3811] automatic generation of API Reference in docs (#4788)
+- [AIRFLOW-3810] Remove duplicate autoclass directive (#4656)
+- [AIRFLOW-XXX] Mention that statsd must be installed to gather metrics (#5038)
+- [AIRFLOW-XXX] Add contents to cli (#4825)
+- [AIRFLOW-XXX] fix check docs failure on CI (#4998)
+- [AIRFLOW-XXX] Fix syntax docs errors (#4789)
+- [AIRFLOW-XXX] Docs rendering improvement (#4684)
+- [AIRFLOW-XXX] Automatically link Jira/GH on doc's changelog page (#4587)
+- [AIRFLOW-XXX] Mention Oracle in the Extra Packages documentation (#4987)
+- [AIRFLOW-XXX] Drop deprecated sudo option; use default docker compose on Travis. (#4732)
+- [AIRFLOW-XXX] Update kubernetes.rst docs (#3875)
+- [AIRFLOW-XXX] Improvements to formatted content in documentation (#4835)
+- [AIRFLOW-XXX] Add Daniel to committer list (#4961)
+- [AIRFLOW-XXX] Add Xiaodong Deng to committers list
+- [AIRFLOW-XXX] Add history become ASF top level project (#4757)
+- [AIRFLOW-XXX] Move out the examples from integration.rst (#4672)
+- [AIRFLOW-XXX] Extract reverse proxy info to a separate file (#4657)
+- [AIRFLOW-XXX] Reduction of the number of warnings in the documentation (#4585)
+- [AIRFLOW-XXX] Fix GCS Operator docstrings (#4054)
+- [AIRFLOW-XXX] Fix Docstrings in Hooks, Sensors & Operators (#4137)
+- [AIRFLOW-XXX] Split guide for operators to multiple files (#4814)
+- [AIRFLOW-XXX] Split connection guide to multiple files (#4824)
+- [AIRFLOW-XXX] Remove almost all warnings from building docs (#4588)
+- [AIRFLOW-XXX] Add backreference in docs between operator and integration (#4671)
+- [AIRFLOW-XXX] Improve linking to classes (#4655)
+- [AIRFLOW-XXX] Mock optional modules when building docs (#4586)
+- [AIRFLOW-XXX] Update plugin macros documentation (#4971)
+- [AIRFLOW-XXX] Add missing docstring for 'autodetect' in GCS to BQ Operator (#4979)
+- [AIRFLOW-XXX] Add missing GCP operators to Docs (#4260)
+- [AIRFLOW-XXX] Fixing the issue in Documentation (#3756)
+- [AIRFLOW-XXX] Add Hint at user defined macros (#4885)
+- [AIRFLOW-XXX] Correct schedule_interval in Scheduler docs (#4157)
+- [AIRFLOW-XXX] Improve airflow-jira script to make RelManager's life easier (#4857)
+- [AIRFLOW-XXX] Add missing class references to docs (#4644)
+- [AIRFLOW-XXX] Fix typo (#4564)
+- [AIRFLOW-XXX] Add a doc about fab security (#4595)
+- [AIRFLOW-XXX] Speed up DagBagTest cases (#3974)
+- [AIRFLOW-XXX] Reduction of the number of warnings in the documentation (#4585)
+
+
+Airflow 1.10.2, 2019-01-19
+--------------------------
+
+New features
+""""""""""""
+
+- [AIRFLOW-2658] Add GCP specific k8s pod operator (#3532)
+- [AIRFLOW-2440] Google Cloud SQL import/export operator (#4251)
+- [AIRFLOW-3212] Add AwsGlueCatalogPartitionSensor (#4112)
+- [AIRFLOW-2750] Add subcommands to delete and list users
+- [AIRFLOW-3480] Add GCP Spanner Database Operators (#4353)
+- [AIRFLOW-3560] Add DayOfWeek Sensor (#4363)
+- [AIRFLOW-3371] BigQueryHook's Ability to Create View (#4213)
+- [AIRFLOW-3332] Add method to allow inserting rows into BQ table (#4179)
+- [AIRFLOW-3055] add get_dataset and get_datasets_list to bigquery_hook (#3894)
+- [AIRFLOW-2887] Added BigQueryCreateEmptyDatasetOperator and create_emty_dataset to bigquery_hook (#3876)
+- [AIRFLOW-2758] Add a sensor for MongoDB
+- [AIRFLOW-2640] Add Cassandra table sensor
+- [AIRFLOW-3398] Google Cloud Spanner instance database query operator (#4314)
+- [AIRFLOW-3310] Google Cloud Spanner deploy / delete operators (#4286)
+- [AIRFLOW-3406] Implement an Azure CosmosDB operator (#4265)
+- [AIRFLOW-3434] Allows creating intermediate dirs in SFTPOperator (#4270)
+- [AIRFLOW-3345] Add Google Cloud Storage (GCS) operators for ACL (#4192)
+- [AIRFLOW-3266] Add AWS Athena Hook and Operator (#4111)
+- [AIRFLOW-3346] Add hook and operator for GCP transfer service (#4189)
+- [AIRFLOW-2983] Add prev_ds_nodash and next_ds_nodash macro (#3821)
+- [AIRFLOW-3403] Add AWS Athena Sensor (#4244)
+- [AIRFLOW-3323] Support HTTP basic authentication for Airflow Flower (#4166)
+- [AIRFLOW-3410] Add feature to allow Host Key Change for SSH Op (#4249)
+- [AIRFLOW-3275] Add Google Cloud SQL Query operator (#4170)
+- [AIRFLOW-2691] Manage JS dependencies via npm
+- [AIRFLOW-2795] Oracle to Oracle Transfer Operator (#3639)
+- [AIRFLOW-2596] Add Oracle to Azure Datalake Transfer Operator
+- [AIRFLOW-3220] Add Instance Group Manager Operators for GCE (#4167)
+- [AIRFLOW-2882] Add import and export for pool cli using JSON
+- [AIRFLOW-2965] CLI tool to show the next execution datetime (#3834)
+- [AIRFLOW-2874] Enables FAB's theme support (#3719)
+- [AIRFLOW-3336] Add new TriggerRule for 0 upstream failures (#4182)
+
+Improvements
+""""""""""""
+
+- [AIRFLOW-3680] Consistency update in tests for All GCP-related operators (#4493)
+- [AIRFLOW-3675] Use googlapiclient for google apis (#4484)
+- [AIRFLOW-3205] Support multipart uploads to GCS (#4084)
+- [AIRFLOW-2826] Add GoogleCloudKMSHook (#3677)
+- [AIRFLOW-3676] Add required permission to CloudSQL export/import example (#4489)
+- [AIRFLOW-3679] Added Google Cloud Base Hook to documentation (#4487)
+- [AIRFLOW-3594] Unify different License Header
+- [AIRFLOW-3197] Remove invalid parameter KeepJobFlowAliveWhenNoSteps in example DAG (#4404)
+- [AIRFLOW-3504] Refine the functionality of "/health" endpoint (#4309)
+- [AIRFLOW-3103][AIRFLOW-3147] Update flask-appbuilder (#3937)
+- [AIRFLOW-3168] More resillient database use in CI (#4014)
+- [AIRFLOW-3076] Remove preloading of MySQL testdata (#3911)
+- [AIRFLOW-3035] Allow custom 'job_error_states' in dataproc ops (#3884)
+- [AIRFLOW-3246] Make hmsclient optional in airflow.hooks.hive_hooks (#4080)
+- [AIRFLOW-3059] Log how many rows are read from Postgres (#3905)
+- [AIRFLOW-2463] Make task instance context available for hive queries
+- [AIRFLOW-3190] Make flake8 compliant (#4035)
+- [AIRFLOW-1998] Implemented DatabricksRunNowOperator for jobs/run-now … (#3813)
+- [AIRFLOW-2267] Airflow DAG level access (#3197)
+- [AIRFLOW-2359] Add set failed for DagRun and task in tree view (#3255)
+- [AIRFLOW-3008] Move Kubernetes example DAGs to contrib
+- [AIRFLOW-3402] Support global k8s affinity and toleration configs (#4247)
+- [AIRFLOW-3610] Add region param for EMR jobflow creation (#4418)
+- [AIRFLOW-3531] Fix test for GCS to GCS Transfer Hook (#4452)
+- [AIRFLOW-3531] Add gcs to gcs transfer operator. (#4331)
+- [AIRFLOW-3034]: Readme updates : Add Slack & Twitter, remove Gitter
+- [AIRFLOW-3028] Update Text & Images in Readme.md
+- [AIRFLOW-208] Add badge to show supported Python versions (#3839)
+- [AIRFLOW-2238] Update PR tool to push directly to Github
+- [AIRFLOW-2238] Flake8 fixes on dev/airflow-pr
+- [AIRFLOW-2238] Update PR tool to remove outdated info (#3978)
+- [AIRFLOW-3005] Replace 'Airbnb Airflow' with 'Apache Airflow' (#3845)
+- [AIRFLOW-3150] Make execution_date templated in TriggerDagRunOperator (#4359)
+- [AIRFLOW-1196][AIRFLOW-2399] Add templated field in TriggerDagRunOperator (#4228)
+- [AIRFLOW-3340] Placeholder support in connections form (#4185)
+- [AIRFLOW-3446] Add Google Cloud BigTable operators (#4354)
+- [AIRFLOW-1921] Add support for https and user auth (#2879)
+- [AIRFLOW-2770] Read `dags_in_image` config value as a boolean (#4319)
+- [AIRFLOW-3022] Add volume mount to KubernetesExecutorConfig (#3855)
+- [AIRFLOW-2917] Set AIRFLOW__CORE__SQL_ALCHEMY_CONN only when needed (#3766)
+- [AIRFLOW-2712] Pass annotations to KubernetesExecutorConfig
+- [AIRFLOW-461]  Support autodetected schemas in BigQuery run_load (#3880)
+- [AIRFLOW-2997] Support cluster fields in bigquery (#3838)
+- [AIRFLOW-2916] Arg `verify` for AwsHook() & S3 sensors/operators (#3764)
+- [AIRFLOW-491] Add feature to pass extra api configs to BQ Hook (#3733)
+- [AIRFLOW-2889] Fix typos detected by github.com/client9/misspell (#3732)
+- [AIRFLOW-850] Add a PythonSensor (#4349)
+- [AIRFLOW-2747] Explicit re-schedule of sensors (#3596)
+- [AIRFLOW-3392] Add index on dag_id in sla_miss table (#4235)
+- [AIRFLOW-3001] Add index 'ti_dag_date' to taskinstance (#3885)
+- [AIRFLOW-2861] Add index on log table (#3709)
+- [AIRFLOW-3518] Performance fixes for topological_sort of Tasks (#4322)
+- [AIRFLOW-3521] Fetch more than 50 items in `airflow-jira compare` script (#4300)
+- [AIRFLOW-1919] Add option to query for DAG runs given a DAG ID
+- [AIRFLOW-3444] Explicitly set transfer operator description. (#4279)
+- [AIRFLOW-3411]  Add OpenFaaS hook (#4267)
+- [AIRFLOW-2785] Add context manager entry points to mongoHook
+- [AIRFLOW-2524] Add SageMaker doc to AWS integration section (#4278)
+- [AIRFLOW-3479] Keeps records in Log Table when DAG is deleted (#4287)
+- [AIRFLOW-2948] Arg check & better doc - SSHOperator & SFTPOperator (#3793)
+- [AIRFLOW-2245] Add remote_host of SSH/SFTP operator as templated field (#3765)
+- [AIRFLOW-2670] Update SSH Operator's Hook to respect timeout (#3666)
+- [AIRFLOW-3380] Add metrics documentation (#4219)
+- [AIRFLOW-3361] Log the task_id in the PendingDeprecationWarning from BaseOperator (#4030)
+- [AIRFLOW-3213] Create ADLS to GCS operator (#4134)
+- [AIRFLOW-3395] added the REST API endpoints to the doc (#4236)
+- [AIRFLOW-3294] Update connections form and integration docs (#4129)
+- [AIRFLOW-3236] Create AzureDataLakeStorageListOperator (#4094)
+- [AIRFLOW-3062] Add Qubole in integration docs (#3946)
+- [AIRFLOW-3306] Disable flask-sqlalchemy modification tracking. (#4146)
+- [AIRFLOW-2867] Refactor Code to conform standards (#3714)
+- [AIRFLOW-2753] Add dataproc_job_id instance var holding actual DP jobId
+- [AIRFLOW-3132] Enable specifying auto_remove option for DockerOperator (#3977)
+- [AIRFLOW-2731] Raise psutil restriction to <6.0.0
+- [AIRFLOW-3384] Allow higher versions of Sqlalchemy and Jinja2 (#4227)
+- [Airflow-2760] Decouple DAG parsing loop from scheduler loop (#3873)
+- [AIRFLOW-3004] Add config disabling scheduler cron (#3899)
+- [AIRFLOW-3175] Fix docstring format in airflow/jobs.py (#4025)
+- [AIRFLOW-3589] Visualize reschedule state in all views (#4408)
+- [AIRFLOW-2698] Simplify Kerberos code (#3563)
+- [AIRFLOW-2499] Dockerise CI pipeline (#3393)
+- [AIRFLOW-3432] Add test for feature "Delete DAG in UI" (#4266)
+- [AIRFLOW-3301] Update DockerOperator CI test for PR #3977 (#4138)
+- [AIRFLOW-3478] Make sure that the session is closed
+- [AIRFLOW-3687] Add missing @apply_defaults decorators (#4498)
+- [AIRFLOW-3691] Update notice to 2019 (#4503)
+- [AIRFLOW-3689] Update pop-up message when deleting DAG in RBAC UI (#4505)
+- [AIRFLOW-2801] Skip test_mark_success_no_kill in PostgreSQL on CI (#3642)
+- [AIRFLOW-3693] Replace psycopg2-binary by psycopg2 (#4508)
+- [AIRFLOW-3700] Change the lowest allowed version of "requests" (#4517)
+- [AIRFLOW-3704] Support SSL Protection When Redis is Used as Broker for CeleryExecutor (#4521)
+- [AIRFLOW-3681] All GCP operators have now optional GCP Project ID (#4500)
+- [Airflow 2782] Upgrades Dagre D3 version to latest possible
+- [Airflow 2783] Implement eslint for JS code check (#3641)
+- [AIRFLOW-2805] Display multiple timezones on UI (#3687)
+- [AIRFLOW-3302] Small CSS fixes (#4140)
+- [Airflow-2766] Respect shared datetime across tabs
+- [AIRFLOW-2776] Compress tree view JSON
+- [AIRFLOW-2407] Use feature detection for reload() (#3298)
+- [AIRFLOW-3452] Removed an unused/dangerous display-none (#4295)
+- [AIRFLOW-3348] Update run statistics on dag refresh (#4197)
+- [AIRFLOW-3125] Monitor Task Instances creation rates (#3966)
+
+
+Bug fixes
+"""""""""
+
+- [AIRFLOW-3191] Fix not being able to specify execution_date when creating dagrun (#4037)
+- [AIRFLOW-3657] Fix zendesk integration (#4466)
+- [AIRFLOW-3605] Load plugins from entry_points (#4412)
+- [AIRFLOW-3646] Rename plugins_manager.py to test_xx to trigger tests (#4464)
+- [AIRFLOW-3655] Escape links generated in model views (#4463)
+- [AIRFLOW-3662] Add dependency for Enum (#4468)
+- [AIRFLOW-3630] Cleanup of GCP Cloud SQL Connection (#4451)
+- [AIRFLOW-1837] Respect task start_date when different from dag's (#4010)
+- [AIRFLOW-2829] Brush up the CI script for minikube
+- [AIRFLOW-3519] Fix example http operator (#4455)
+- [AIRFLOW-2811] Fix scheduler_ops_metrics.py to work (#3653)
+- [AIRFLOW-2751] add job properties update in hive to druid operator.
+- [AIRFLOW-2918] Remove unused imports
+- [AIRFLOW-2918] Fix Flake8 violations (#3931)
+- [AIRFLOW-2771] Add except type to broad S3Hook try catch clauses
+- [AIRFLOW-2918] Fix Flake8 violations (#3772)
+- [AIRFLOW-2099] Handle getsource() calls gracefully
+- [AIRFLOW-3397] Fix integrety error in rbac AirflowSecurityManager (#4305)
+- [AIRFLOW-3281] Fix Kubernetes operator with git-sync (#3770)
+- [AIRFLOW-2615] Limit DAGs parsing to once only
+- [AIRFLOW-2952] Fix Kubernetes CI (#3922)
+- [AIRFLOW-2933] Enable Codecov on Docker-CI Build (#3780)
+- [AIRFLOW-2082] Resolve a bug in adding password_auth to api as auth method (#4343)
+- [AIRFLOW-3612] Remove incubation/incubator mention (#4419)
+- [AIRFLOW-3581] Fix next_ds/prev_ds semantics for manual runs (#4385)
+- [AIRFLOW-3527] Update Cloud SQL Proxy to have shorter path for UNIX socket (#4350)
+- [AIRFLOW-3316] For gcs_to_bq: add missing init of schema_fields var (#4430)
+- [AIRFLOW-3583] Fix AirflowException import (#4389)
+- [AIRFLOW-3578] Fix Type Error for BigQueryOperator (#4384)
+- [AIRFLOW-2755] Added `kubernetes.worker_dags_folder` configuration (#3612)
+- [AIRFLOW-2655] Fix inconsistency of default config of kubernetes worker
+- [AIRFLOW-2645][AIRFLOW-2617] Add worker_container_image_pull_policy
+- [AIRFLOW-2661] fix config dags_volume_subpath and logs_volume_subpath
+- [AIRFLOW-3550] Standardize GKE hook (#4364)
+- [AIRFLOW-2863] Fix GKEClusterHook catching wrong exception (#3711)
+- [AIRFLOW-2939][AIRFLOW-3568] Fix TypeError in GCSToS3Op & S3ToGCSOp (#4371)
+- [AIRFLOW-3327] Add support for location in BigQueryHook (#4324)
+- [AIRFLOW-3438] Fix default values in BigQuery Hook & BigQueryOperator (…
+- [AIRFLOW-3355] Fix BigQueryCursor.execute to work with Python3 (#4198)
+- [AIRFLOW-3447] Add 2 options for ts_nodash Macro (#4323)
+- [AIRFLOW-1552] Airflow Filter_by_owner not working with password_auth (#4276)
+- [AIRFLOW-3484] Fix Over-logging in the k8s executor (#4296)
+- [AIRFLOW-3309] Add MongoDB connection (#4154)
+- [AIRFLOW-3414] Fix reload_module in DagFileProcessorAgent (#4253)
+- [AIRFLOW-1252] API accept JSON when invoking a trigger dag (#2334)
+- [AIRFLOW-3425] Fix setting default scope in hook (#4261)
+- [AIRFLOW-3416] Fixes Python 3 compatibility with CloudSqlQueryOperator (#4254)
+- [AIRFLOW-3263] Ignore exception when 'run' kills already killed job (#4108)
+- [AIRFLOW-3264] URL decoding when parsing URI for connection (#4109)
+- [AIRFLOW-3365][AIRFLOW-3366] Allow celery_broker_transport_options to be set with environment variables (#4211)
+- [AIRFLOW-2642] fix wrong value git-sync initcontainer env GIT_SYNC_ROOT (#3519)
+- [AIRFLOW-3353] Pin redis verison (#4195)
+- [AIRFLOW-3251] KubernetesPodOperator now uses 'image_pull_secrets' argument when creating Pods (#4188)
+- [AIRFLOW-2705] Move class-level moto decorator to method-level
+- [AIRFLOW-3233] Fix deletion of DAGs in the UI (#4069)
+- [AIRFLOW-2908] Allow retries with KubernetesExecutor. (#3758)
+- [AIRFLOW-1561] Fix scheduler to pick up example DAGs without other DAGs (#2635)
+- [AIRFLOW-3352] Fix expose_config not honoured on RBAC UI (#4194)
+- [AIRFLOW-3592] Fix logs when task is in rescheduled state (#4492)
+- [AIRFLOW-3634] Fix GCP Spanner Test (#4440)
+- [AIRFLOW-XXX] Fix PythonVirtualenvOperator tests (#3968)
+- [AIRFLOW-3239] Fix/refine tests for api/common/experimental/ (#4255)
+- [AIRFLOW-2951] Update dag_run table end_date when state change (#3798)
+- [AIRFLOW-2756] Fix bug in set DAG run state workflow (#3606)
+- [AIRFLOW-3690] Fix bug to set state of a task for manually-triggered DAGs (#4504)
+- [AIRFLOW-3319] KubernetsExecutor: Need in try_number in labels if getting them later (#4163)
+- [AIRFLOW-3724] Fix the broken refresh button on Graph View in RBAC UI
+- [AIRFLOW-3732] Fix issue when trying to edit connection in RBAC UI
+- [AIRFLOW-2866] Fix missing CSRF token head when using RBAC UI (#3804)
+- [AIRFLOW-3259] Fix internal server error when displaying charts (#4114)
+- [AIRFLOW-3271] Fix issue with persistence of RBAC Permissions modified via UI (#4118)
+- [AIRFLOW-3141] Handle duration View for missing dag (#3984)
+- [AIRFLOW-2766] Respect shared datetime across tabs
+- [AIRFLOW-1413] Fix FTPSensor failing on error message with unexpected (#2450)
+- [AIRFLOW-3378] KubernetesPodOperator does not delete on timeout failure (#4218)
+- [AIRFLOW-3245] Fix list processing in resolve_template_files (#4086)
+- [AIRFLOW-2703] Catch transient DB exceptions from scheduler's heartbeat it does not crash (#3650)
+- [AIRFLOW-1298] Clear UPSTREAM_FAILED using the clean cli (#3886)
+
+Doc-only changes
+""""""""""""""""
+
+- [AIRFLOW-XXX] GCP operators documentation clarifications (#4273)
+- [AIRFLOW-XXX] Docs: Fix paths to GCS transfer operator (#4479)
+- [AIRFLOW-XXX] Add missing GCP operators to Docs (#4260)
+- [AIRFLOW-XXX] Fix Docstrings for Operators (#3820)
+- [AIRFLOW-XXX] Fix inconsistent comment in example_python_operator.py (#4337)
+- [AIRFLOW-XXX] Fix incorrect parameter in SFTPOperator example (#4344)
+- [AIRFLOW-XXX] Add missing remote logging field (#4333)
+- [AIRFLOW-XXX] Revise template variables documentation (#4172)
+- [AIRFLOW-XXX] Fix typo in docstring of gcs_to_bq (#3833)
+- [AIRFLOW-XXX] Fix display of SageMaker operators/hook docs (#4263)
+- [AIRFLOW-XXX] Better instructions for airflow flower (#4214)
+- [AIRFLOW-XXX] Make pip install commands consistent (#3752)
+- [AIRFLOW-XXX] Add `BigQueryGetDataOperator` to Integration Docs (#4063)
+- [AIRFLOW-XXX] Don't spam test logs with "bad cron expression" messages (#3973)
+- [AIRFLOW-XXX] Update committer list based on latest TLP discussion (#4427)
+- [AIRFLOW-XXX] Fix incorrect statement in contributing guide (#4104)
+- [AIRFLOW-XXX] Fix Broken Link in CONTRIBUTING.md
+- [AIRFLOW-XXX] Update Contributing Guide - Git Hooks (#4120)
+- [AIRFLOW-3426] Correct Python Version Documentation Reference (#4259)
+- [AIRFLOW-2663] Add instructions to install SSH dependencies
+- [AIRFLOW-XXX] Clean up installation extra packages table (#3750)
+- [AIRFLOW-XXX] Remove redundant space in Kerberos (#3866)
+- [AIRFLOW-3086] Add extras group for google auth to setup.py (#3917)
+- [AIRFLOW-XXX] Add Kubernetes Dependency in Extra Packages Doc (#4281)
+- [AIRFLOW-3696] Add Version info to Airflow Documentation (#4512)
+- [AIRFLOW-XXX] Correct Typo in sensor's exception (#4545)
+- [AIRFLOW-XXX] Fix a typo of config (#4544)
+- [AIRFLOW-XXX] Fix BashOperator Docstring (#4052)
+- [AIRFLOW-3018] Fix Minor issues in Documentation
+- [AIRFLOW-XXX] Fix Minor issues with Azure Cosmos Operator (#4289)
+- [AIRFLOW-3382] Fix incorrect docstring in DatastoreHook (#4222)
+- [AIRFLOW-XXX] Fix copy&paste mistake (#4212)
+- [AIRFLOW-3260] Correct misleading BigQuery error (#4098)
+- [AIRFLOW-XXX] Fix Typo in SFTPOperator docstring (#4016)
+- [AIRFLOW-XXX] Fixing the issue in Documentation (#3998)
+- [AIRFLOW-XXX] Fix undocumented params in S3_hook
+- [AIRFLOW-XXX] Fix SlackWebhookOperator execute method comment (#3963)
+- [AIRFLOW-3070] Refine web UI authentication-related docs (#3863)
+
+Airflow 1.10.1, 2018-11-13
+--------------------------
+
+New features
+""""""""""""
+
+- [AIRFLOW-2524] Airflow integration with AWS Sagemaker
+- [AIRFLOW-2657] Add ability to delete DAG from web ui
+- [AIRFLOW-2780] Adds IMAP Hook to interact with a mail server
+- [AIRFLOW-2794] Add delete support for Azure blob
+- [AIRFLOW-2912] Add operators for Google Cloud Functions
+- [AIRFLOW-2974] Add Start/Restart/Terminate methods Databricks Hook
+- [AIRFLOW-2989] No Parameter to change bootDiskType for DataprocClusterCreateOperator
+- [AIRFLOW-3078] Basic operators for Google Compute Engine
+- [AIRFLOW-3147] Update Flask-AppBuilder version
+- [AIRFLOW-3231] Basic operators for Google Cloud SQL (deploy / patch / delete)
+- [AIRFLOW-3276] Google Cloud SQL database create / patch / delete operators
+
+Improvements
+""""""""""""
+
+- [AIRFLOW-393] Add progress callbacks for FTP downloads
+- [AIRFLOW-520] Show Airflow version on web page
+- [AIRFLOW-843] Exceptions now available in context during on_failure_callback
+- [AIRFLOW-2476] Update tabulate dependency to v0.8.2
+- [AIRFLOW-2592] Bump Bleach dependency
+- [AIRFLOW-2622] Add "confirm=False" option to SFTPOperator
+- [AIRFLOW-2662] support affinity & nodeSelector policies for kubernetes executor/operator
+- [AIRFLOW-2709] Improve error handling in Databricks hook
+- [AIRFLOW-2723] Update lxml dependancy to >= 4.0.
+- [AIRFLOW-2763] No precheck mechanism in place during worker initialisation for the connection to metadata database
+- [AIRFLOW-2789] Add ability to create single node cluster to DataprocClusterCreateOperator
+- [AIRFLOW-2797] Add ability to create Google Dataproc cluster with custom image
+- [AIRFLOW-2854] kubernetes_pod_operator add more configuration items
+- [AIRFLOW-2855] Need to Check Validity of Cron Expression When Process DAG File/Zip File
+- [AIRFLOW-2904] Clean an unnecessary line in airflow/executors/celery_executor.py
+- [AIRFLOW-2921] A trivial incorrectness in CeleryExecutor()
+- [AIRFLOW-2922] Potential deal-lock bug in CeleryExecutor()
+- [AIRFLOW-2932] GoogleCloudStorageHook - allow compression of file
+- [AIRFLOW-2949] Syntax Highlight for Single Quote
+- [AIRFLOW-2951] dag_run end_date Null after a dag is finished
+- [AIRFLOW-2956] Kubernetes tolerations for pod operator
+- [AIRFLOW-2997] Support for clustered tables in Bigquery hooks/operators
+- [AIRFLOW-3006] Fix error when schedule_interval="None"
+- [AIRFLOW-3008] Move Kubernetes related example DAGs to contrib/example_dags
+- [AIRFLOW-3025] Allow to specify dns and dns-search parameters for DockerOperator
+- [AIRFLOW-3067] (www_rbac) Flask flash messages are not displayed properly (no background color)
+- [AIRFLOW-3069] Decode output of S3 file transform operator
+- [AIRFLOW-3072] Assign permission get_logs_with_metadata to viewer role
+- [AIRFLOW-3090] INFO logs are too verbose
+- [AIRFLOW-3103] Update Flask-Login
+- [AIRFLOW-3112] Align SFTP hook with SSH hook
+- [AIRFLOW-3119] Enable loglevel on celery worker and inherit from airflow.cfg
+- [AIRFLOW-3137] Make ProxyFix middleware optional
+- [AIRFLOW-3173] Add _cmd options for more password config options
+- [AIRFLOW-3177] Change scheduler_heartbeat metric from gauge to counter
+- [AIRFLOW-3193] Pin docker requirement version to v3
+- [AIRFLOW-3195] Druid Hook: Log ingestion spec and task id
+- [AIRFLOW-3197] EMR Hook is missing some parameters to valid on the AWS API
+- [AIRFLOW-3232] Make documentation for GCF Functions operator more readable
+- [AIRFLOW-3262] Can't get log containing Response when using SimpleHttpOperator
+- [AIRFLOW-3265] Add support for "unix_socket" in connection extra for Mysql Hook
+
+Doc-only changes
+""""""""""""""""
+
+- [AIRFLOW-1441] Tutorial Inconsistencies Between Example Pipeline Definition and Recap
+- [AIRFLOW-2682] Add how-to guide(s) for how to use basic operators like BashOperator and PythonOperator
+- [AIRFLOW-3104] .airflowignore feature is not mentioned at all in documentation
+- [AIRFLOW-3237] Refactor example DAGs
+- [AIRFLOW-3187] Update airflow.gif file with a slower version
+- [AIRFLOW-3159] Update Airflow documentation on GCP Logging
+- [AIRFLOW-3030] Command Line docs incorrect subdir
+- [AIRFLOW-2990] Docstrings for Hooks/Operators are in incorrect format
+- [AIRFLOW-3127] Celery SSL Documentation is out-dated
+- [AIRFLOW-2779] Add license headers to doc files
+- [AIRFLOW-2779] Add project version to license
+
+Bug fixes
+"""""""""
+
+- [AIRFLOW-839] docker_operator.py attempts to log status key without first checking existence
+- [AIRFLOW-1104] Concurrency check in scheduler should count queued tasks as well as running
+- [AIRFLOW-1163] Add support for x-forwarded-* headers to support access behind AWS ELB
+- [AIRFLOW-1195] Cleared tasks in SubDagOperator do not trigger Parent dag_runs
+- [AIRFLOW-1508] Skipped state not part of State.task_states
+- [AIRFLOW-1762] Use key_file in SSHHook.create_tunnel()
+- [AIRFLOW-1837] Differing start_dates on tasks not respected by scheduler.
+- [AIRFLOW-1874] Support standard SQL in Check, ValueCheck and IntervalCheck BigQuery operators
+- [AIRFLOW-1917] print() from python operators end up with extra new line
+- [AIRFLOW-1970] Database cannot be initialized if an invalid fernet key is provided
+- [AIRFLOW-2145] Deadlock after clearing a running task
+- [AIRFLOW-2216] Cannot specify a profile for AWS Hook to load with s3 config file
+- [AIRFLOW-2574] initdb fails when mysql password contains percent sign
+- [AIRFLOW-2707] Error accessing log files from web UI
+- [AIRFLOW-2716] Replace new Python 3.7 keywords
+- [AIRFLOW-2744] RBAC app doesn't integrate plugins (blueprints etc)
+- [AIRFLOW-2772] BigQuery hook does not allow specifying both the partition field name and table name at the same time
+- [AIRFLOW-2778] Bad Import in collect_dag in DagBag
+- [AIRFLOW-2786] Variables view fails to render if a variable has an empty key
+- [AIRFLOW-2799] Filtering UI objects by datetime is broken
+- [AIRFLOW-2800] Remove airflow/ low-hanging linting errors
+- [AIRFLOW-2825] S3ToHiveTransfer operator may not may able to handle GZIP file with uppercase ext in S3
+- [AIRFLOW-2848] dag_id is missing in metadata table "job" for LocalTaskJob
+- [AIRFLOW-2860] DruidHook: time variable is not updated correctly when checking for timeout
+- [AIRFLOW-2865] Race condition between on_success_callback and LocalTaskJob's cleanup
+- [AIRFLOW-2893] Stuck dataflow job due to jobName mismatch.
+- [AIRFLOW-2895] Prevent scheduler from spamming heartbeats/logs
+- [AIRFLOW-2900] Code not visible for Packaged DAGs
+- [AIRFLOW-2905] Switch to regional dataflow job service.
+- [AIRFLOW-2907] Sendgrid - Attachments - ERROR - Object of type 'bytes' is not JSON serializable
+- [AIRFLOW-2938] Invalid 'extra' field in connection can raise an AttributeError when attempting to edit
+- [AIRFLOW-2979] Deprecated Celery Option not in Options list
+- [AIRFLOW-2981] TypeError in dataflow operators when using GCS jar or py_file
+- [AIRFLOW-2984] Cannot convert naive_datetime when task has a naive start_date/end_date
+- [AIRFLOW-2994] flatten_results in BigQueryOperator/BigQueryHook should default to None
+- [AIRFLOW-3002] ValueError in dataflow operators when using GCS jar or py_file
+- [AIRFLOW-3012] Email on sla miss is send only to first address on the list
+- [AIRFLOW-3046] ECS Operator mistakenly reports success when task is killed due to EC2 host termination
+- [AIRFLOW-3064] No output from `airflow test` due to default logging config
+- [AIRFLOW-3072] Only admin can view logs in RBAC UI
+- [AIRFLOW-3079] Improve initdb to support MSSQL Server
+- [AIRFLOW-3089] Google auth doesn't work under http
+- [AIRFLOW-3099] Errors raised when some blocs are missing in airflow.cfg
+- [AIRFLOW-3109] Default user permission should contain 'can_clear'
+- [AIRFLOW-3111] Confusing comments and instructions for log templates in UPDATING.md and default_airflow.cfg
+- [AIRFLOW-3124] Broken webserver debug mode (RBAC)
+- [AIRFLOW-3136] Scheduler Failing the Task retries run while processing Executor Events
+- [AIRFLOW-3138] Migration cc1e65623dc7 creates issues with postgres
+- [AIRFLOW-3161] Log Url link does not link to task instance logs in RBAC UI
+- [AIRFLOW-3162] HttpHook fails to parse URL when port is specified
+- [AIRFLOW-3183] Potential Bug in utils/dag_processing/DagFileProcessorManager.max_runs_reached()
+- [AIRFLOW-3203] Bugs in DockerOperator & Some operator test scripts were named incorrectly
+- [AIRFLOW-3238] Dags, removed from the filesystem, are not deactivated on initdb
+- [AIRFLOW-3268] Cannot pass SSL dictionary to mysql connection via URL
+- [AIRFLOW-3277] Invalid timezone transition handling for cron schedules
+- [AIRFLOW-3295] Require encryption in DaskExecutor when certificates are configured.
+- [AIRFLOW-3297] EmrStepSensor marks cancelled step as successful
+
+Airflow 1.10.0, 2018-08-03
+--------------------------
+- [AIRFLOW-2870] Use abstract TaskInstance for migration
+- [AIRFLOW-2859] Implement own UtcDateTime (#3708)
+- [AIRFLOW-2140] Don't require kubernetes for the SparkSubmit hook
+- [AIRFLOW-2869] Remove smart quote from default config
+- [AIRFLOW-2857] Fix Read the Docs env
+- [AIRFLOW-2817] Force explicit choice on GPL dependency
+- [AIRFLOW-2716] Replace async and await py3.7 keywords
+- [AIRFLOW-2810] Fix typo in Xcom model timestamp
+- [AIRFLOW-2710] Clarify fernet key value in documentation
+- [AIRFLOW-2606] Fix DB schema and SQLAlchemy model
+- [AIRFLOW-2646] Fix setup.py not to install snakebite on Python3
+- [AIRFLOW-2604] Add index to task_fail
+- [AIRFLOW-2650] Mark SchedulerJob as succeed when hitting Ctrl-c
+- [AIRFLOW-2678] Fix db schema unit test to remove checking fab models
+- [AIRFLOW-2624] Fix webserver login as anonymous
+- [AIRFLOW-2654] Fix incorret URL on refresh in Graph View of FAB UI
+- [AIRFLOW-2668] Handle missing optional cryptography dependency
+- [AIRFLOW-2681] Include last dag run of externally triggered DAGs in UI.
+- [AIRFLOW-1840] Support back-compat on old celery config
+- [AIRFLOW-2612][AIRFLOW-2534] Clean up Hive-related tests
+- [AIRFLOW-2608] Implements/Standardize custom exceptions for experimental APIs
+- [AIRFLOW-2607] Fix failing TestLocalClient
+- [AIRFLOW-2638] dbapi_hook: support REPLACE INTO
+- [AIRFLOW-2542][AIRFLOW-1790] Rename AWS Batch Operator queue to job_queue
+- [AIRFLOW-2567] Extract result from the kubernetes pod as Xcom
+- [AIRFLOW-XXX] Adding REA Group to readme
+- [AIRFLOW-2601] Allow user to specify k8s config
+- [AIRFLOW-2559] Azure Fileshare hook
+- [AIRFLOW-1786] Enforce correct behavior for soft-fail sensors
+- [AIRFLOW-2355] Airflow trigger tag parameters in subdag
+- [AIRFLOW-2613] Fix Airflow searching .zip bug
+- [AIRFLOW-2627] Add a sensor for Cassandra
+- [AIRFLOW-2634][AIRFLOW-2534] Remove dependency for impyla
+- [AIRFLOW-2611] Fix wrong dag volume mount path for kubernetes executor
+- [AIRFLOW-2562] Add Google Kubernetes Engine Operators
+- [AIRFLOW-2630] Fix classname in test_sql_sensor.py
+- [AIRFLOW-2534] Fix bug in HiveServer2Hook
+- [AIRFLOW-2586] Stop getting AIRFLOW_HOME value from config file in bash operator
+- [AIRFLOW-2605] Fix autocommit for MySqlHook
+- [AIRFLOW-2539][AIRFLOW-2359] Move remaing log config to configuration file
+- [AIRFLOW-1656] Tree view dags query changed
+- [AIRFLOW-2617] add imagePullPolicy config for kubernetes executor
+- [AIRFLOW-2429] Fix security/task/sensors/ti_deps folders flake8 error
+- [AIRFLOW-2550] Implements API endpoint to list DAG runs
+- [AIRFLOW-2512][AIRFLOW-2522] Use google-auth instead of oauth2client
+- [AIRFLOW-2429] Fix operators folder flake8 error
+- [AIRFLOW-2585] Fix several bugs in CassandraHook and CassandraToGCSOperator
+- [AIRFLOW-2597] Restore original dbapi.run() behavior
+- [AIRFLOW-2590] Fix commit in DbApiHook.run() for no-autocommit DB
+- [AIRFLOW-1115] fix github oauth api URL
+- [AIRFLOW-2587] Add TIMESTAMP type mapping to MySqlToHiveTransfer
+- [AIRFLOW-2591][AIRFLOW-2581] Set default value of autocommit to False in DbApiHook.run()
+- [AIRFLOW-59] Implement bulk_dump and bulk_load for the Postgres hook
+- [AIRFLOW-2533] Fix path to DAG's on kubernetes executor workers
+- [AIRFLOW-2581] RFLOW-2581] Fix DbApiHook autocommit
+- [AIRFLOW-2578] Add option to use proxies in JiraHook
+- [AIRFLOW-2575] Make gcs to gcs operator work with large files
+- [AIRFLOW-437] Send TI context in kill zombies
+- [AIRFLOW-2566] Change backfill to rerun failed tasks
+- [AIRFLOW-1021] Fix double login for new users with LDAP
+- [AIRFLOW-XXX] Typo fix
+- [AIRFLOW-2561] Fix typo in EmailOperator
+- [AIRFLOW-2573] Cast BigQuery TIMESTAMP field to float
+- [AIRFLOW-2560] Adding support for internalIpOnly to DataprocClusterCreateOperator
+- [AIRFLOW-2565] templatize cluster_label
+- [AIRFLOW-83] add mongo hook and operator
+- [AIRFLOW-2558] Clear task/dag is clearing all executions
+- [AIRFLOW-XXX] Fix doc typos
+- [AIRFLOW-2513] Change `bql` to `sql` for BigQuery Hooks & Ops
+- [AIRFLOW-2557] Fix pagination for s3
+- [AIRFLOW-2545] Eliminate DeprecationWarning
+- [AIRFLOW-2500] Fix MySqlToHiveTransfer to transfer unsigned type properly
+- [AIRFLOW-2462] Change PasswordUser setter to correct syntax
+- [AIRFLOW-2525] Fix a bug introduced by commit dabf1b9
+- [AIRFLOW-2553] Add webserver.pid to .gitignore
+- [AIRFLOW-1863][AIRFLOW-2529] Add dag run selection widgets to gantt view
+- [AIRFLOW-2504] Log username correctly and add extra to search columns
+- [AIRFLOW-2551] Encode binary data with base64 standard rather than base64 url
+- [AIRFLOW-2537] Add reset-dagrun option to backfill command
+- [AIRFLOW-2526] dag_run.conf can override params
+- [AIRFLOW-2544][AIRFLOW-1967] Guard against next major release of Celery, Flower
+- [AIRFLOW-XXX] Add Yieldr to who is using airflow
+- [AIRFLOW-2547] Describe how to run tests using Docker
+- [AIRFLOW-2538] Update faq doc on how to reduce airflow scheduler latency
+- [AIRFLOW-2529] Improve graph view performance and usability
+- [AIRFLOW-2517] backfill support passing key values through CLI
+- [AIRFLOW-2532] Support logs_volume_subpath for KubernetesExecutor
+- [AIRFLOW-2466] consider task_id in _change_state_for_tis_without_dagrun
+- [AIRFLOW-2519] Fix CeleryExecutor with SQLAlchemy
+- [AIRFLOW-2402] Fix RBAC task log
+- [AIRFLOW-XXX] Add M4U to user list
+- [AIRFLOW-2536] docs about how to deal with airflow initdb failure
+- [AIRFLOW-2530] KubernetesOperator supports multiple clusters
+- [AIRFLOW-1499] Eliminate duplicate and unneeded code
+- [AIRFLOW-2521] backfill - make variable name and logging messages more acurate
+- [AIRFLOW-2429] Fix hook, macros folder flake8 error
+- [Airflow-XXX] add Prime to company list
+- [AIRFLOW-2525] Fix PostgresHook.copy_expert to work with "COPY FROM"
+- [AIRFLOW-2515] Add dependency on thrift_sasl to hive extra
+- [AIRFLOW-2523] Add how-to for managing GCP connections
+- [AIRFLOW-2510] Introduce new macros: prev_ds and next_ds
+- [AIRFLOW-1730] Unpickle value of XCom queried from DB
+- [AIRFLOW-2518] Fix broken ToC links in integration.rst
+- [AIRFLOW-1472] Fix SLA misses triggering on skipped tasks.
+- [AIRFLOW-2520] CLI - make backfill less verbose
+- [AIRFLOW-2107] add time_partitioning to run_query on BigQueryBaseCursor
+- [AIRFLOW-1057][AIRFLOW-1380][AIRFLOW-2362][2362] AIRFLOW Update DockerOperator to new API
+- [AIRFLOW-2415] Make airflow DAG templating render numbers
+- [AIRFLOW-2473] Fix wrong skip condition for TransferTests
+- [AIRFLOW-2472] Implement MySqlHook.bulk_dump
+- [AIRFLOW-2419] Use default view for subdag operator
+- [AIRFLOW-2498] Fix Unexpected argument in SFTP Sensor
+- [AIRFLOW-2509] Separate config docs into how-to guides
+- [AIRFLOW-2429] Add BaseExecutor back
+- [AIRFLOW-2429] Fix dag, example_dags, executors flake8 error
+- [AIRFLOW-2502] Change Single triple quotes to double for docstrings
+- [AIRFLOW-2503] Fix broken links in CONTRIBUTING.md
+- [AIRFLOW-2501] Refer to devel instructions in docs contrib guide
+- [AIRFLOW-2429] Fix contrib folder's flake8 errors
+- [AIRFLOW-2471] Fix HiveCliHook.load_df to use unused parameters
+- [AIRFLOW-2495] Update celery to 4.1.1
+- [AIRFLOW-2429] Fix api, bin, config_templates folders flake8 error
+- [AIRFLOW-2493] Mark template_fields of all Operators in the API document as "templated"
+- [AIRFLOW-2489] Update FlaskAppBuilder to 1.11.1
+- [AIRFLOW-2448] Enhance HiveCliHook.load_df to work with datetime
+- [AIRFLOW-2487] Enhance druid ingestion hook
+- [AIRFLOW-2397] Support affinity policies for Kubernetes executor/operator
+- [AIRFLOW-2482] Add test for rewrite method in GCS Hook
+- [AIRFLOW-2481] Fix flaky Kubernetes test
+- [AIRFLOW-2479] Improve doc FAQ section
+- [AIRFLOW-2485] Fix Incorrect logging for Qubole Sensor
+- [AIRFLOW-2486] Remove unnecessary slash after port
+- [AIRFLOW-2429] Make Airflow flake8 compliant
+- [AIRFLOW-2491] Resolve flask version conflict
+- [AIRFLOW-2484] Remove duplicate key in MySQL to GCS Op
+- [ARIFLOW-2458] Add cassandra-to-gcs operator
+- [AIRFLOW-2477] Improve time units for task duration and landing times charts for RBAC UI
+- [AIRFLOW-2474] Only import snakebite if using py2
+- [AIRFLOW-48] Parse connection uri querystring
+- [AIRFLOW-2467][AIRFLOW-2] Update import direct warn message to use the module name
+- [AIRFLOW-XXX] Fix order of companies
+- [AIRFLOW-2452] Document field_dict must be OrderedDict
+- [AIRFLOW-2420] Azure Data Lake Hook
+- [AIRFLOW-2213] Add Quoble check operator
+- [AIRFLOW-2465] Fix wrong module names in the doc
+- [AIRFLOW-1929] Modifying TriggerDagRunOperator to accept execution_date
+- [AIRFLOW-2460] Users can now use volume mounts and volumes
+- [AIRFLOW-2110][AIRFLOW-2122] Enhance Http Hook
+- [AIRFLOW-XXX] Updated contributors list
+- [AIRFLOW-2435] Add launch_type to ECSOperator to allow FARGATE
+- [AIRFLOW-2451] Remove extra slash ('/') char when using wildcard in gcs_to_gcs operator
+- [AIRFLOW-2461] Add support for cluster scaling on dataproc operator
+- [AIRFLOW-2376] Fix no hive section error
+- [AIRFLOW-2425] Add lineage support
+- [AIRFLOW-2430] Extend query batching to additional slow queries
+- [AIRFLOW-2453] Add default nil value for kubernetes/git_subpath
+- [AIRFLOW-2396] Add support for resources in kubernetes operator
+- [AIRFLOW-2169] Encode binary data with base64 before importing to BigQuery
+- [AIRFLOW-XXX] Add spotahome in user list
+- [AIRFLOW-2457] Update FAB version requirement
+- [AIRFLOW-2454][Airflow 2454] Support imagePullPolicy for k8s
+- [AIRFLOW-2450] update supported k8s versions to 1.9 and 1.10
+- [AIRFLOW-2333] Add Segment Hook and TrackEventOperator
+- [AIRFLOW-2442][AIRFLOW-2] Airflow run command leaves database connections open
+- [AIRFLOW-2016] assign template_fields for Dataproc Workflow Template sub-classes, not base class
+- [AIRFLOW-2446] Add S3ToRedshiftTransfer into the "Integration" doc
+- [AIRFLOW-2449] Fix operators.py to run all test cases
+- [AIRFLOW-2424] Add dagrun status endpoint and increased k8s test coverage
+- [AIRFLOW-2441] Fix bugs in HiveCliHook.load_df
+- [AIRFLOW-2358][AIRFLOW-201804] Make the Kubernetes example optional
+- [AIRFLOW-2436] Remove cli_logger in initdb
+- [AIRFLOW-2444] Remove unused option(include_adhoc) in cli backfill command
+- [AIRFLOW-2447] Fix TestHiveMetastoreHook to run all cases
+- [AIRFLOW-2445] Allow templating in kubernetes operator
+- [AIRFLOW-2086][AIRFLOW-2393] Customize default dagrun number in tree view
+- [AIRFLOW-2437] Add PubNub to list of current airflow users
+- [AIRFLOW-XXX] Add Quantopian to list of Airflow users
+- [AIRFLOW-1978] Add WinRM windows operator and hook
+- [AIRFLOW-2427] Add tests to named hive sensor
+- [AIRFLOW-2412] Fix HiveCliHook.load_file to address HIVE-10541
+- [AIRFLOW-2431] Add the navigation bar color parameter for RBAC UI
+- [AIRFLOW-2407] Resolve Python undefined names
+- [AIRFLOW-1952] Add the navigation bar color parameter
+- [AIRFLOW-2222] Implement GoogleCloudStorageHook.rewrite
+- [AIRFLOW-2426] Add Google Cloud Storage Hook tests
+- [AIRFLOW-2418] Bump Flask-WTF
+- [AIRFLOW-2417] Wait for pod is not running to end task
+- [AIRFLOW-1914] Add other charset support to email utils
+- [AIRFLOW-XXX] Update README.md with Craig@Work
+- [AIRFLOW-1899] Fix Kubernetes tests
+- [AIRFLOW-1812] Update logging example
+- [AIRFLOW-2313] Add TTL parameters for Dataproc
+- [AIRFLOW-2411] add dataproc_jars to templated_fields
+- [AIRFLOW-XXX] Add Reddit to Airflow users
+- [AIRFLOW-XXX] Fix wrong table header in scheduler.rst
+- [AIRFLOW-2409] Supply password as a parameter
+- [AIRFLOW-2410][AIRFLOW-75] Set the timezone in the RBAC Web UI
+- [AIRFLOW-2394] default cmds and arguments in kubernetes operator
+- [AIRFLOW-2406] Add Apache2 License Shield to Readme
+- [AIRFLOW-2404] Add additional documentation for unqueued task
+- [AIRFLOW-2400] Add Ability to set Environment Variables for K8s
+- [AIRFLOW-XXX] Add Twine Labs as an Airflow user
+- [AIRFLOW-1853] Show only the desired number of runs in tree view
+- [AIRFLOW-2401] Document the use of variables in Jinja template
+- [AIRFLOW-2403] Fix License Headers
+- [AIRFLOW-1313] Fix license header
+- [AIRFLOW-2398] Add BounceX to list of current airflow users
+- [AIRFLOW-2363] Fix return type bug in TaskHandler
+- [AIRFLOW-2389] Create a pinot db api hook
+- [AIRFLOW-2390] Resolve FlaskWTFDeprecationWarning
+- [AIRFLOW-1933] Fix some typos
+- [AIRFLOW-1960] Add support for secrets in kubernetes operator
+- [AIRFLOW-1313] Add vertica_to_mysql operator
+- [AIRFLOW-1575] Add AWS Kinesis Firehose Hook for inserting batch records
+- [AIRFLOW-2266][AIRFLOW-2343] Remove google-cloud-dataflow dependency
+- [AIRFLOW-2370] Implement --use_random_password in create_user
+- [AIRFLOW-2348] Strip path prefix from the destination_object when source_object contains a wildcard[]
+- [AIRFLOW-2391] Fix to Flask 0.12.2
+- [AIRFLOW-2381] Fix the flaky ApiPasswordTests test
+- [AIRFLOW-2378] Add Groupon to list of current users
+- [AIRFLOW-2382] Fix wrong description for delimiter
+- [AIRFLOW-2380] Add support for environment variables in Spark submit operator.
+- [AIRFLOW-2377] Improve Sendgrid sender support
+- [AIRFLOW-2331] Support init action timeout on dataproc cluster create
+- [AIRFLOW-1835] Update docs: Variable file is json
+- [AIRFLOW-1781] Make search case-insensitive in LDAP group
+- [AIRFLOW-2042] Fix browser menu appearing over the autocomplete menu
+- [AIRFLOW-XXX] Remove wheelhouse files from travis not owned by travis
+- [AIRFLOW-2336] Use hmsclient in hive_hook
+- [AIRFLOW-2041] Correct Syntax in python examples
+- [AIRFLOW-74] SubdagOperators can consume all celeryd worker processes
+- [AIRFLOW-2369] Fix gcs tests
+- [AIRFLOW-2365] Fix autocommit attribute check
+- [AIRFLOW-2068] MesosExecutor allows optional Docker image
+- [AIRFLOW-1652] Push DatabricksRunSubmitOperator metadata into XCOM
+- [AIRFLOW-2234] Enable insert_rows for PrestoHook
+- [AIRFLOW-2208][Airflow-22208] Link to same DagRun graph from TaskInstance view
+- [AIRFLOW-1153] Allow HiveOperators to take hiveconfs
+- [AIRFLOW-775] Fix autocommit settings with Jdbc hook
+- [AIRFLOW-2364] Warn when setting autocommit on a connection which does not support it
+- [AIRFLOW-2357] Add persistent volume for the logs
+- [AIRFLOW-766] Skip conn.commit() when in Auto-commit
+- [AIRFLOW-2351] Check for valid default_args start_date
+- [AIRFLOW-1433] Set default rbac to initdb
+- [AIRFLOW-2270] Handle removed tasks in backfill
+- [AIRFLOW-2344] Fix `connections -l` to work with pipe/redirect
+- [AIRFLOW-2300] Add S3 Select functionarity to S3ToHiveTransfer
+- [AIRFLOW-1314] Cleanup the config
+- [AIRFLOW-1314] Polish some of the Kubernetes docs/config
+- [AIRFLOW-1314] Improve error handling
+- [AIRFLOW-1999] Add per-task GCP service account support
+- [AIRFLOW-1314] Rebasing against master
+- [AIRFLOW-1314] Small cleanup to address PR comments (#24)
+- [AIRFLOW-1314] Add executor_config and tests
+- [AIRFLOW-1314] Improve k8s support
+- [AIRFLOW-1314] Use VolumeClaim for transporting DAGs
+- [AIRFLOW-1314] Create integration testing environment
+- [AIRFLOW-1314] Git Mode to pull in DAGs for Kubernetes Executor
+- [AIRFLOW-1314] Add support for volume mounts & Secrets in Kubernetes Executor
+- [AIRFLOW=1314] Basic Kubernetes Mode
+- [AIRFLOW-2326][AIRFLOW-2222] remove contrib.gcs_copy_operator
+- [AIRFLOW-2328] Fix empty GCS blob in S3ToGoogleCloudStorageOperator
+- [AIRFLOW-2350] Fix grammar in UPDATING.md
+- [AIRFLOW-2302] Fix documentation
+- [AIRFLOW-2345] pip is not used in this setup.py
+- [AIRFLOW-2347] Add Banco de Formaturas to Readme
+- [AIRFLOW-2346] Add Investorise as official user of Airflow
+- [AIRFLOW-2330] Do not append destination prefix if not given
+- [AIRFLOW-2240][DASK] Added TLS/SSL support for the dask-distributed scheduler.
+- [AIRFLOW-2309] Fix duration calculation on TaskFail
+- [AIRFLOW-2335] fix issue with jdk8 download for ci
+- [AIRFLOW-2184] Add druid_checker_operator
+- [AIRFLOW-2299] Add S3 Select functionarity to S3FileTransformOperator
+- [AIRFLOW-2254] Put header as first row in unload
+- [AIRFLOW-610] Respect _cmd option in config before defaults
+- [AIRFLOW-2287] Fix incorrect ASF headers
+- [AIRFLOW-XXX] Add Zego as an Apache Airflow user
+- [AIRFLOW-952] fix save empty extra field in UI
+- [AIRFLOW-1325] Add ElasticSearch log handler and reader
+- [AIRFLOW-2301] Sync files of an S3 key with a GCS path
+- [AIRFLOW-2293] Fix S3FileTransformOperator to work with boto3
+- [AIRFLOW-3212][AIRFLOW-2314] Remove only leading slash in GCS path
+- [AIRFLOW-1509][AIRFLOW-442] SFTP Sensor
+- [AIRFLOW-2291] Add optional params to ML Engine
+- [AIRFLOW-1774] Allow consistent templating of arguments in MLEngineBatchPredictionOperator
+- [AIRFLOW-2302] Add missing operators and hooks
+- [AIRFLOW-2312] Docs Typo Correction: Corresponding
+- [AIRFLOW-1623] Trigger on_kill method in operators
+- [AIRFLOW-2162] When impersonating another user, pass env variables to sudo
+- [AIRFLOW-2304] Update quickstart doc to mention scheduler part
+- [AIRFLOW-1633] docker_operator needs a way to set shm_size
+- [AIRFLOW-1340] Add S3 to Redshift transfer operator
+- [AIRFLOW-2303] Lists the keys inside an S3 bucket
+- [AIRFLOW-2209] restore flask_login imports
+- [AIRFLOW-2306] Add Bonnier Broadcasting to list of current users
+- [AIRFLOW-2305][AIRFLOW-2027] Fix CI failure caused by []
+- [AIRFLOW-2281] Add support for Sendgrid categories
+- [AIRFLOW-2027] Only trigger sleep in scheduler after all files have parsed
+- [AIRFLOW-2256] SparkOperator: Add Client Standalone mode and retry mechanism
+- [AIRFLOW-2284] GCS to S3 operator
+- [AIRFLOW-2287] Update license notices
+- [AIRFLOW-2296] Add Cinimex DataLab to Readme
+- [AIRFLOW-2298] Add Kalibrr to who uses airflow
+- [AIRFLOW-2292] Fix docstring for S3Hook.get_wildcard_key
+- [AIRFLOW-XXX] Update PR template
+- [AIRFLOW-XXX] Remove outdated migrations.sql
+- [AIRFLOW-2287] Add license header to docs/Makefile
+- [AIRFLOW-2286] Add tokopedia to the readme
+- [AIRFLOW-2273] Add Discord webhook operator/hook
+- [AIRFLOW-2282] Fix grammar in UPDATING.md
+- [AIRFLOW-2200] Add snowflake operator with tests
+- [AIRFLOW-2178] Add handling on SLA miss errors
+- [AIRFLOW-2169] Fix type 'bytes' is not JSON serializable in python3
+- [AIRFLOW-2215] Pass environment to subproces.Popen in base_task_runner
+- [AIRFLOW-2253] Add Airflow CLI instrumentation
+- [AIRFLOW-2274] Fix Dataflow tests
+- [AIRFLOW-2269] Add Custom Ink as an Airflow user
+- [AIRFLOW-2259] Dataflow Hook Index out of range
+- [AIRFLOW-2233] Update updating.md to include the info of hdfs_sensors renaming
+- [AIRFLOW-2217] Add Slack webhook operator
+- [AIRFLOW-1729] improve dagBag time
+- [AIRFLOW-2264] Improve create_user cli help message
+- [AIRFLOW-2260] [AIRFLOW-2260] SSHOperator add command template .sh files
+- [AIRFLOW-2261] Check config/env for remote base log folder
+- [AIRFLOW-2258] Allow import of Parquet-format files into BigQuery
+- [AIRFLOW-1430] Include INSTALL instructions to avoid GPL
+- [AIRFLOW-1430] Solve GPL dependency
+- [AIRFLOW-2251] Add Thinknear as an Airflow user
+- [AIRFLOW-2244] bugfix: remove legacy LongText code from models.py
+- [AIRFLOW-2247] Fix RedshiftToS3Transfer not to fail with ValueError
+- [AIRFLOW-2249] Add side-loading support for Zendesk Hook
+- [AIRFLOW-XXX] Add Qplum to Airflow users
+- [AIRFLOW-2228] Enhancements in ValueCheckOperator
+- [AIRFLOW-1206] Typos
+- [AIRFLOW-2060] Update pendulum version to 1.4.4
+- [AIRFLOW-2248] Fix wrong param name in RedshiftToS3Transfer doc
+- [AIRFLOW-1433][AIRFLOW-85] New Airflow Webserver UI with RBAC support
+- [AIRFLOW-1235] Fix webserver's odd behaviour
+- [AIRFLOW-1460] Allow restoration of REMOVED TI's
+- [airflow-2235] Fix wrong docstrings in two operators
+- [AIRFLOW-XXX] Fix chronological order for companies using Airflow
+- [AIRFLOW-2124] Upload Python file to a bucket for Dataproc
+- [AIRFLOW-2212] Fix ungenerated sensor API reference
+- [AIRFLOW-2226] Rename google_cloud_storage_default to google_cloud_default
+- [AIRFLOW-2211] Rename hdfs_sensors.py to hdfs_sensor.py for consistency
+- [AIRFLOW-2225] Update document to include DruidDbApiHook
+- [Airflow-2202] Add filter support in HiveMetastoreHook().max_partition()
+- [AIRFLOW-2220] Remove duplicate numeric list entry in security.rst
+- [AIRFLOW-XXX] Update tutorial documentation
+- [AIRFLOW-2215] Update celery task to preserve environment variables and improve logging on exception
+- [AIRFLOW-2185] Use state instead of query param
+- [AIRFLOW-2183] Refactor DruidHook to enable sql
+- [AIRFLOW-2203] Defer cycle detection
+- [AIRFLOW-2203] Remove Useless Commands.
+- [AIRFLOW-2203] Cache signature in apply_defaults
+- [AIRFLOW-2203] Speed up Operator Resources
+- [AIRFLOW-2203] Cache static rules (trigger/weight)
+- [AIRFLOW-2203] Store task ids as sets not lists
+- [AIRFLOW-2205] Remove unsupported args from JdbcHook doc
+- [AIRFLOW-2207] Fix flaky test that uses app.cached_app()
+- [AIRFLOW-2206] Remove unsupported args from JdbcOperator doc
+- [AIRFLOW-2140] Add Kubernetes scheduler to SparkSubmitOperator
+- [AIRFLOW-XXX] Add Xero to list of users
+- [AIRFLOW-2204] Fix webserver debug mode
+- [AIRFLOW-102] Fix test_complex_template always succeeds
+- [AIRFLOW-442] Add SFTPHook
+- [AIRFLOW-2169] Add schema to MySqlToGoogleCloudStorageOperator
+- [AIRFLOW-2184][AIRFLOW-2138] Google Cloud Storage allow wildcards
+- [AIRFLOW-1588] Cast Variable value to string
+- [AIRFLOW-2199] Fix invalid reference to logger
+- [AIRFLOW-2191] Change scheduler heartbeat logs from info to debug
+- [AIRFLOW-2106] SalesForce hook sandbox option
+- [AIRFLOW-2197] Silence hostname_callable config error message
+- [AIRFLOW-2150] Use lighter call in HiveMetastoreHook().max_partition()
+- [AIRFLOW-2186] Change the way logging is carried out in few ops
+- [AIRFLOW-2181] Convert password_auth and test_password_endpoints from DOS to UNIX
+- [AIRFLOW-2187] Fix Broken Travis CI due to AIRFLOW-2123
+- [AIRFLOW-2175] Check that filepath is not None
+- [AIRFLOW-2173] Don't check task IDs for concurrency reached check
+- [AIRFLOW-2168] Remote logging for Azure Blob Storage
+- [AIRFLOW-XXX] Add DocuTAP to list of users
+- [AIRFLOW-2176] Change the way logging is carried out in BQ Get Data Operator
+- [AIRFLOW-2177] Add mock test for GCS Download op
+- [AIRFLOW-2123] Install CI dependencies from setup.py
+- [AIRFLOW-2129] Presto hook calls _parse_exception_message but defines _get_pretty_exception_message
+- [AIRFLOW-2174] Fix typos and wrongly rendered documents
+- [AIRFLOW-2171] Store delegated credentials
+- [AIRFLOW-2166] Restore BQ run_query dialect param
+- [AIRFLOW-2163] Add HBC Digital to users of airflow
+- [AIRFLOW-2065] Fix race-conditions when creating loggers
+- [AIRFLOW-2147] Plugin manager: added 'sensors' attribute
+- [AIRFLOW-2059] taskinstance query is awful, un-indexed, and does not scale
+- [AIRFLOW-2159] Fix a few typos in salesforce_hook
+- [AIRFLOW-2132] Add step to initialize database
+- [AIRFLOW-2160] Fix bad rowid deserialization
+- [AIRFLOW-2161] Add Vevo to list of companies using Airflow
+- [AIRFLOW-2149] Add link to apache Beam documentation to create self executing Jar
+- [AIRFLOW-2151] Allow getting the session from AwsHook
+- [AIRFLOW-2097] tz referenced before assignment
+- [AIRFLOW-2152] Add Multiply to list of companies using Airflow
+- [AIRFLOW-1551] Add operator to trigger Jenkins job
+- [AIRFLOW-2034] Fix mixup between %s and {} when using str.format Convention is to use .format for string formating oustide logging, else use lazy format See comment in related issue https://github.com/apache/airflow/pull/2823/files Identified problematic case using following command line .git/COMMIT_EDITMSG:`grep -r '%s'./* | grep '\.format('`
+- [AIRFLOW-2102] Add custom_args to Sendgrid personalizations
+- [AIRFLOW-1035][AIRFLOW-1053] import unicode_literals to parse Unicode in HQL
+- [AIRFLOW-2127] Keep loggers during DB migrations
+- [AIRFLOW-2146] Resolve issues with BQ using DbApiHook methods
+- [AIRFLOW-2087] Scheduler Report shows incorrect Total task number
+- [AIRFLOW-2139] Remove unncecessary boilerplate to get DataFrame using pandas_gbq
+- [AIRFLOW-2125] Using binary package psycopg2-binary
+- [AIRFLOW-2142] Include message on mkdir failure
+- [AIRFLOW-1615] SSHHook: use port specified by Connection
+- [AIRFLOW-2122] Handle boolean values in sshHook
+- [AIRFLOW-XXX] Add Tile to the list of users
+- [AIRFLOW-2130] Add missing Operators to API Reference docs
+- [AIRFLOW-XXX] Add timeout units (seconds)
+- [AIRFLOW-2134] Add Alan to the list of companies that use Airflow
+- [AIRFLOW-2133] Remove references to GitHub issues in CONTRIBUTING
+- [AIRFLOW-2131] Remove confusing AirflowImport docs
+- [AIRFLOW-1852] Allow hostname to be overridable.
+- [AIRFLOW-2126] Add Bluecore to active users
+- [AIRFLOW-1618] Add feature to create GCS bucket
+- [AIRFLOW-2108] Fix log indentation in BashOperator
+- [AIRFLOW-2115] Fix doc links to PythonHosted
+- [AIRFLOW-XXX] Add contributor from Easy company
+- [AIRFLOW-1882] Add ignoreUnknownValues option to gcs_to_bq operator
+- [AIRFLOW-2089] Add on kill for SparkSubmit in Standalone Cluster
+- [AIRFLOW-2113] Address missing DagRun callbacks Given that the handle_callback method belongs to the DAG object, we are able to get the list of task directly with get_task and reduce the communication with the database, making airflow more lightweight.
+- [AIRFLOW-2112] Fix svg width for Recent Tasks on UI.
+- [AIRFLOW-2116] Set CI Cloudant version to <2.0
+- [AIRFLOW-XXX] Add PMC to list of companies using Airflow
+- [AIRFLOW-2100] Fix Broken Documentation Links
+- [AIRFLOW-1404] Add 'flatten_results' & 'maximum_bytes_billed' to BQ Operator
+- [AIRFLOW-800] Initialize valid Google BigQuery Connection
+- [AIRFLOW-1319] Fix misleading SparkSubmitOperator and SparkSubmitHook docstring
+- [AIRFLOW-1983] Parse environment parameter as template
+- [AIRFLOW-2095] Add operator to create External BigQuery Table
+- [AIRFLOW-2085] Add SparkJdbc operator
+- [AIRFLOW-1002] Add ability to clean all dependencies of removed DAG
+- [AIRFLOW-2094] Jinjafied project_id, region & zone in DataProc{*} Operators
+- [AIRFLOW-2092] Fixed incorrect parameter in docstring for FTPHook
+- [AIRFLOW-XXX] Add SocialCops to Airflow users
+- [AIRFLOW-2088] Fix duplicate keys in MySQL to GCS Helper function
+- [AIRFLOW-2091] Fix incorrect docstring parameter in BigQuery Hook
+- [AIRFLOW-2090] Fix typo in DataStore Hook
+- [AIRFLOW-1157] Fix missing pools crashing the scheduler
+- [AIRFLOW-713] Jinjafy {EmrCreateJobFlow,EmrAddSteps}Operator attributes
+- [AIRFLOW-2083] Docs: Use "its" instead of "it's" where appropriate
+- [AIRFLOW-2066] Add operator to create empty BQ table
+- [AIRFLOW-XXX] add Karmic to list of companies
+- [AIRFLOW-2073] Make FileSensor fail when the file doesn't exist
+- [AIRFLOW-2078] Improve task_stats and dag_stats performance
+- [AIRFLOW-2080] Use a log-out icon instead of a power button
+- [AIRFLOW-2077] Fetch all pages of list_objects_v2 response
+- [AIRFLOW-XXX] Add TM to list of companies
+- [AIRFLOW-1985] Impersonation fixes for using `run_as_user`
+- [AIRFLOW-2018][AIRFLOW-2] Make Sensors backward compatible
+- [AIRFLOW-XXX] Fix typo in concepts doc (dag_md)
+- [AIRFLOW-2069] Allow Bytes to be uploaded to S3
+- [AIRFLOW-2074] Fix log var name in GHE auth
+- [AIRFLOW-1927] Convert naive datetimes for TaskInstances
+- [AIRFLOW-1760] Password auth for experimental API
+- [AIRFLOW-2038] Add missing kubernetes dependency for dev
+- [AIRFLOW-2040] Escape special chars in task instance logs URL
+- [AIRFLOW-1968][AIRFLOW-1520] Add role_arn and aws_account_id/aws_iam_role support back to aws hook
+- [AIRFLOW-2048] Fix task instance failure string formatting
+- [AIRFLOW-2046] Fix kerberos error to work with python 3.x
+- [AIRFLOW-2063] Add missing docs for GCP
+- [AIRFLOW-XXX] Fix typo in docs
+- [AIRFLOW-1793] Use docker_url instead of invalid base_url
+- [AIRFLOW-2055] Elaborate on slightly ambiguous documentation
+- [AIRFLOW-2039] BigQueryOperator supports priority property
+- [AIRFLOW-2053] Fix quote character bug in BQ hook
+- [AIRFLOW-2057] Add Overstock to list of companies
+- [AIRFLOW-XXX] Add Plaid to Airflow users
+- [AIRFLOW-2044] Add SparkSubmitOperator to documentation
+- [AIRFLOW-2037] Add methods to get Hash values of a GCS object
+- [AIRFLOW-2050] Fix Travis permission problem
+- [AIRFLOW-2043] Add Intercom to list of companies
+- [AIRFLOW-2023] Add debug logging around number of queued files
+- [AIRFLOW-XXX] Add Pernod-ricard as a airflow user
+- [AIRFLOW-1453] Add 'steps' into template_fields in EmrAddSteps
+- [AIRFLOW-2015] Add flag for interactive runs
+- [AIRFLOW-1895] Fix primary key integrity for mysql
+- [AIRFLOW-2030] Fix KeyError:`i` in DbApiHook for insert
+- [AIRFLOW-1943] Add External BigQuery Table feature
+- [AIRFLOW-2033] Add Google Cloud Storage List Operator
+- [AIRFLOW-2006] Add local log catching to kubernetes operator
+- [AIRFLOW-2031] Add missing gcp_conn_id in the example in DataFlow docstrings
+- [AIRFLOW-2029] Fix AttributeError in BigQueryPandasConnector
+- [AIRFLOW-2028] Add JobTeaser to official users list
+- [AIRFLOW-2016] Add support for Dataproc Workflow Templates
+- [AIRFLOW-2025] Reduced Logging verbosity
+- [AIRFLOW-1267][AIRFLOW-1874] Add dialect parameter to BigQueryHook
+- [AIRFLOW-XXX] Fixed a typo
+- [AIRFLOW-XXX] Typo node to nodes
+- [AIRFLOW-2019] Update DataflowHook for updating Streaming type job
+- [AIRFLOW-2017][Airflow 2017] adding query output to PostgresOperator
+- [AIRFLOW-1889] Split sensors into separate files
+- [AIRFLOW-1950] Optionally pass xcom_pull task_ids
+- [AIRFLOW-1755] Allow mount below root
+- [AIRFLOW-511][Airflow 511] add success/failure callbacks on dag level
+- [AIRFLOW-192] Add weight_rule param to BaseOperator
+- [AIRFLOW-2008] Use callable for python column defaults
+- [AIRFLOW-1984] Fix to AWS Batch operator
+- [AIRFLOW-2000] Support non-main dataflow job class
+- [AIRFLOW-2003] Use flask-caching instead of flask-cache
+- [AIRFLOW-2002] Do not swallow exception on logging import
+- [AIRFLOW-2004] Import flash from flask not flask.login
+- [AIRFLOW-1997] Fix GCP operator doc strings
+- [AIRFLOW-1996] Update DataflowHook waitfordone for Streaming type job[]
+- [AIRFLOW-1995][Airflow 1995] add on_kill method to SqoopOperator
+- [AIRFLOW-1770] Allow HiveOperator to take in a file
+- [AIRFLOW-1994] Change background color of Scheduled state Task Instances
+- [AIRFLOW-1436][AIRFLOW-1475] EmrJobFlowSensor considers Cancelled step as Successful
+- [AIRFLOW-1517] Kubernetes operator PR fixes
+- [AIRFLOW-1517] addressed PR comments
+- [AIRFLOW-1517] started documentation of k8s operator
+- [AIRFLOW-1517] Restore authorship of resources
+- [AIRFLOW-1517] Remove authorship of resources
+- [AIRFLOW-1517] Add minikube for kubernetes integration tests
+- [AIRFLOW-1517] Restore authorship of resources
+- [AIRFLOW-1517] fixed license issues
+- [AIRFLOW-1517] Created more accurate failures for kube cluster issues
+- [AIRFLOW-1517] Remove authorship of resources
+- [AIRFLOW-1517] Add minikube for kubernetes integration tests
+- [AIRFLOW-1988] Change BG color of None state TIs
+- [AIRFLOW-790] Clean up TaskInstances without DagRuns
+- [AIRFLOW-1949] Fix var upload, str() produces "b'...'" which is not json
+- [AIRFLOW-1930] Convert func.now() to timezone.utcnow()
+- [AIRFLOW-1688] Support load.time_partitioning in bigquery_hook
+- [AIRFLOW-1975] Make TriggerDagRunOperator callback optional
+- [AIRFLOW-1480] Render template attributes for ExternalTaskSensor fields
+- [AIRFLOW-1958] Add kwargs to send_email
+- [AIRFLOW-1976] Fix for missing log/logger attribute FileProcessHandler
+- [AIRFLOW-1982] Fix Executor event log formatting
+- [AIRFLOW-1971] Propagate hive config on impersonation
+- [AIRFLOW-1969] Always use HTTPS URIs for Google OAuth2
+- [AIRFLOW-1954] Add DataFlowTemplateOperator
+- [AIRFLOW-1963] Add config for HiveOperator mapred_queue
+- [AIRFLOW-1946][AIRFLOW-1855] Create a BigQuery Get Data Operator
+- [AIRFLOW-1953] Add labels to dataflow operators
+- [AIRFLOW-1967] Update Celery to 4.0.2
+- [AIRFLOW-1964] Add Upsight to list of Airflow users
+- [AIRFLOW-XXX] Changelog for 1.9.0
+- [AIRFLOW-1470] Implement BashSensor operator
+- [AIRFLOW-XXX] Pin sqlalchemy dependency
+- [AIRFLOW-1955] Do not reference unassigned variable
+- [AIRFLOW-1957] Add contributor to BalanceHero in Readme
+- [AIRFLOW-1517] Restore authorship of secrets and init container
+- [AIRFLOW-1517] Remove authorship of secrets and init container
+- [AIRFLOW-1935] Add BalanceHero to readme
+- [AIRFLOW-1939] add astronomer contributors
+- [AIRFLOW-1517] Kubernetes Operator
+- [AIRFLOW-1928] Fix @once with catchup=False
+- [AIRFLOW-1937] Speed up scheduling by committing in batch
+- [AIRFLOW-1821] Enhance default logging config by removing extra loggers
+- [AIRFLOW-1904] Correct DAG fileloc to the right filepath
+- [AIRFLOW-1909] Update docs with supported versions of MySQL server
+- [AIRFLOW-1915] Relax flask-wtf dependency specification
+- [AIRFLOW-1920] Update CONTRIBUTING.md to reflect enforced linting rules
+- [AIRFLOW-1942] Update Sphinx docs to remove deprecated import structure
+- [AIRFLOW-1846][AIRFLOW-1697] Hide Ad Hoc Query behind secure_mode config
+- [AIRFLOW-1948] Include details for on_kill failure
+- [AIRFLOW-1938] Clean up unused exception
+- [AIRFLOW-1932] Add GCP Pub/Sub Pull and Ack
+- [AIRFLOW-XXX] Purge coveralls
+- [AIRFLOW-XXX] Remove unused coveralls token
+- [AIRFLOW-1938] Remove tag version check in setup.py
+- [AIRFLOW-1916] Don't upload logs to remote from `run --raw`
+- [AIRFLOW-XXX] Fix failing PubSub tests on Python3
+- [AIRFLOW-XXX] Upgrade to python 3.5 and disable dask tests
+- [AIRFLOW-1913] Add new GCP PubSub operators
+- [AIRFLOW-1525] Fix minor LICENSE and NOTICE issues
+- [AIRFLOW-1687] fix fernet error without encryption
+- [AIRFLOW-1912] airflow.processor should not propagate logging
+- [AIRFLOW-1911] Rename celeryd_concurrency
+- [AIRFLOW-1885] Fix IndexError in ready_prefix_on_cmdline
+- [AIRFLOW-1854] Improve Spark Submit operator for standalone cluster mode
+- [AIRFLOW-1908] Fix celery broker options config load
+- [AIRFLOW-1907] Pass max_ingestion_time to Druid hook
+- [AIRFLOW-1909] Add away to list of users
+- [AIRFLOW-1893][AIRFLOW-1901] Propagate PYTHONPATH when using impersonation
+- [AIRFLOW-1892] Modify BQ hook to extract data filtered by column
+- [AIRFLOW-1829] Support for schema updates in query jobs
+- [AIRFLOW-1840] Make celery configuration congruent with Celery 4
+- [AIRFLOW-1878] Fix stderr/stdout redirection for tasks
+- [AIRFLOW-1897][AIRFLOW-1873] Task Logs for running instance not visible in WebUI
+- [AIRFLOW-1896] FIX bleach <> html5lib incompatibility
+- [AIRFLOW-1884][AIRFLOW-1059] Reset orphaned task state for external dagruns
+- [AIRFLOW-XXX] Fix typo in comment
+- [AIRFLOW-1869] Do not emit spurious warning on missing logs
+- [AIRFLOW-1888] Add AWS Redshift Cluster Sensor
+- [AIRFLOW-1887] Renamed endpoint url variable
+- [AIRFLOW-1873] Set TI.try_number to right value depending TI state
+- [AIRFLOW-1891] Fix non-ascii typo in default configuration template
+- [AIRFLOW-1879] Handle ti log entirely within ti
+- [AIRFLOW-1869] Write more error messages into gcs and file logs
+- [AIRFLOW-1876] Write subtask id to task log header
+- [AIRFLOW-1554] Fix wrong DagFileProcessor termination method call
+- [AIRFLOW-342] Do not use amqp, rpc as result backend
+- [AIRFLOW-966] Make celery broker_transport_options configurable
+- [AIRFLOW-1881] Make operator log in task log
+- [AIRFLOW-XXX] Added DataReply to the list of Airflow Users
+- [AIRFLOW-1883] Get File Size for objects in Google Cloud Storage
+- [AIRFLOW-1872] Set context for all handlers including parents
+- [AIRFLOW-1855][AIRFLOW-1866] Add GCS Copy Operator to copy multiple files
+- [AIRFLOW-1870] Enable flake8 tests
+- [AIRFLOW-1785] Enable Python 3 tests
+- [AIRFLOW-1850] Copy cmd before masking
+- [AIRFLOW-1665] Reconnect on database errors
+- [AIRFLOW-1559] Dispose SQLAlchemy engines on exit
+- [AIRFLOW-1559] Close file handles in subprocesses
+- [AIRFLOW-1559] Make database pooling optional
+- [AIRFLOW-1848][Airflow-1848] Fix DataFlowPythonOperator py_file extension doc comment
+- [AIRFLOW-1843] Add Google Cloud Storage Sensor with prefix
+- [AIRFLOW-1803] Time zone documentation
+- [AIRFLOW-1826] Update views to use timezone aware objects
+- [AIRFLOW-1827] Fix api endpoint date parsing
+- [AIRFLOW-1806] Use naive datetime when using cron
+- [AIRFLOW-1809] Update tests to use timezone aware objects
+- [AIRFLOW-1806] Use naive datetime for cron scheduling
+- [AIRFLOW-1807] Force use of time zone aware db fields
+- [AIRFLOW-1808] Convert all utcnow() to time zone aware
+- [AIRFLOW-1804] Add time zone configuration options
+- [AIRFLOW-1802] Convert database fields to timezone aware
+- [AIRFLOW-XXX] Add dask lock files to excludes
+- [AIRFLOW-1790] Add support for AWS Batch operator
+- [AIRFLOW-XXX] Update README.md
+- [AIRFLOW-1820] Remove timestamp from metric name
+- [AIRFLOW-1810] Remove unused mysql import in migrations.
+- [AIRFLOW-1838] Properly log collect_dags exception
+- [AIRFLOW-1842] Fixed Super class name for the gcs to gcs copy operator
+- [AIRFLOW-1845] Modal background now covers long or tall pages
+- [AIRFLOW-1229] Add link to Run Id, incl execution_date
+- [AIRFLOW-1842] Add gcs to gcs copy operator with renaming if required
+- [AIRFLOW-1841] change False to None in operator and hook
+- [AIRFLOW-1839] Fix more bugs in S3Hook boto -> boto3 migration
+- [AIRFLOW-1830] Support multiple domains in Google authentication backend
+- [AIRFLOW-1831] Add driver-classpath spark submit
+- [AIRFLOW-1795] Correctly call S3Hook after migration to boto3
+- [AIRFLOW-1811] Fix render Druid operator
+- [AIRFLOW-1819] Fix slack operator unittest bug
+- [AIRFLOW-1805] Allow Slack token to be passed through connection
+- [AIRFLOW-1816] Add region param to Dataproc operators
+- [AIRFLOW-868] Add postgres_to_gcs operator and unittests
+- [AIRFLOW-1613] make mysql_to_gcs_operator py3 compatible
+- [AIRFLOW-1817] use boto3 for s3 dependency
+- [AIRFLOW-1813] Bug SSH Operator empty buffer
+- [AIRFLOW-1801][AIRFLOW-288] Url encode execution dates
+- [AIRFLOW-1563] Catch OSError while symlinking the latest log directory
+- [AIRFLOW-1794] Remove uses of Exception.message for Python 3
+- [AIRFLOW-1799] Fix logging line which raises errors
+- [AIRFLOW-1102] Upgrade Gunicorn >=19.4.0
+- [AIRFLOW-1756] Fix S3TaskHandler to work with Boto3-based S3Hook
+- [AIRFLOW-1797] S3Hook.load_string didn't work on Python3
+- [AIRFLOW-646] Add docutils to setup_requires
+- [AIRFLOW-1792] Missing intervals DruidOperator
+- [AIRFLOW-1789][AIRFLOW-1712] Log SSHOperator stderr to log.warning
+- [AIRFLOW-1787] Fix task instance batch clear and set state bugs
+- [AIRFLOW-1780] Fix long output lines with unicode from hanging parent
+- [AIRFLOW-387] Close SQLAlchemy sessions properly
+- [AIRFLOW-1779] Add keepalive packets to ssh hook
+- [AIRFLOW-1669] Fix Docker and pin Moto to 1.1.19
+- [AIRFLOW-71] Add support for private Docker images
+- [AIRFLOW-XXX] Give a clue what the 'ds' variable is
+- [AIRFLOW-XXX] Correct typos in the faq docs page
+- [AIRFLOW-1571] Add AWS Lambda Hook
+- [AIRFLOW-1675] Fix docstrings for API docs
+- [AIRFLOW-1712][AIRFLOW-756][AIRFLOW-751] Log SSHOperator output
+- [AIRFLOW-1776] Capture stdout and stderr for logging
+- [AIRFLOW-1765] Make experimental API securable without needing Kerberos.
+- [AIRFLOW-1764] The web interface should not use the experimental API
+- [AIRFLOW-1771] Rename heartbeat to avoid confusion
+- [AIRFLOW-1769] Add support for templates in VirtualenvOperator
+- [AIRFLOW-1763] Fix S3TaskHandler unit tests
+- [AIRFLOW-1315] Add Qubole File & Partition Sensors
+- [AIRFLOW-1018] Make processor use logging framework
+- [AIRFLOW-1695] Add RedshiftHook using boto3
+- [AIRFLOW-1706] Fix query error for MSSQL backend
+- [AIRFLOW-1711] Use ldap3 dict for group membership
+- [AIRFLOW-1723] Make sendgrid a plugin
+- [AIRFLOW-1757] Add missing options to SparkSubmitOperator
+- [AIRFLOW-1734][Airflow 1734] Sqoop hook/operator enhancements
+- [AIRFLOW-1761] Fix type in scheduler.rst
+- [AIRFLOW-1731] Set pythonpath for logging
+- [AIRFLOW-1641] Handle executor events in the scheduler
+- [AIRFLOW-1744] Make sure max_tries can be set
+- [AIRFLOW-1732] Improve dataflow hook logging
+- [AIRFLOW-1736] Add HotelQuickly to Who Uses Airflow
+- [AIRFLOW-1657] Handle failing qubole operator
+- [AIRFLOW-1677] Fix typo in example_qubole_operator
+- [AIRFLOW-926] Fix JDBC Hook
+- [AIRFLOW-1520] Boto3 S3Hook, S3Log
+- [AIRFLOW-1716] Fix multiple __init__ def in SimpleDag
+- [AIRFLOW-XXX] Fix DateTime in Tree View
+- [AIRFLOW-1719] Fix small typo
+- [AIRFLOW-1432] Charts label for Y axis not visible
+- [AIRFLOW-1743] Verify ldap filters correctly
+- [AIRFLOW-1745] Restore default signal disposition
+- [AIRFLOW-1741] Correctly hide second chart on task duration page
+- [AIRFLOW-1728] Add networkUri, subnet, tags to Dataproc operator
+- [AIRFLOW-1726] Add copy_expert psycopg2 method to PostgresHook
+- [AIRFLOW-1330] Add conn_type argument to CLI when adding connection
+- [AIRFLOW-1698] Remove SCHEDULER_RUNS env var in systemd
+- [AIRFLOW-1694] Stop using itertools.izip
+- [AIRFLOW-1692] Change test_views filename to support Windows
+- [AIRFLOW-1722] Fix typo in scheduler autorestart output filename
+- [AIRFLOW-1723] Support sendgrid in email backend
+- [AIRFLOW-1718] Set num_retries on Dataproc job request execution
+- [AIRFLOW-1727] Add unit tests for DataProcHook
+- [AIRFLOW-1631] Fix timing issue in unit test
+- [AIRFLOW-1631] Fix local executor unbound parallelism
+- [AIRFLOW-1724] Add Fundera to Who uses Airflow?
+- [AIRFLOW-1683] Cancel BigQuery job on timeout.
+- [AIRFLOW-1714] Fix misspelling: s/seperate/separate/
+- [AIRFLOW-1681] Add batch clear in task instance view
+- [AIRFLOW-1696] Fix dataproc version label error
+- [AIRFLOW-1613] Handle binary field in MySqlToGoogleCloudStorageOperator
+- [AIRFLOW-1697] Mode to disable charts endpoint
+- [AIRFLOW-1691] Add better Google cloud logging documentation
+- [AIRFLOW-1690] Add detail to gcs error messages
+- [AIRFLOW-1682] Make S3TaskHandler write to S3 on close
+- [AIRFLOW-1634] Adds task_concurrency feature
+- [AIRFLOW-1676] Make GCSTaskHandler write to GCS on close
+- [AIRFLOW-1678] Fix erroneously repeated word in function docstrings
+- [AIRFLOW-1323] Made Dataproc operator parameter names consistent
+- [AIRFLOW-1590] fix unused module and variable
+- [AIRFLOW-1671] Add @apply_defaults back to gcs download operator
+- [AIRFLOW-988] Fix repeating SLA miss callbacks
+- [AIRFLOW-1611] Customize logging
+- [AIRFLOW-1668] Expose keepalives_idle for Postgres connections
+- [AIRFLOW-1658] Kill Druid task on timeout
+- [AIRFLOW-1669][AIRFLOW-1368] Fix Docker import
+- [AIRFLOW-891] Make webserver clock include date
+- [AIRFLOW-1560] Add AWS DynamoDB hook and operator for inserting batch items
+- [AIRFLOW-1654] Show tooltips for link icons in DAGs view
+- [AIRFLOW-1660] Change webpage width to full-width
+- [AIRFLOW-1664] write file as binary instead of str
+- [AIRFLOW-1659] Fix invalid obj attribute bug in file_task_handler.py
+- [AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file
+- [AIRFLOW-1650] Fix custom celery config loading
+- [AIRFLOW-1647] Fix Spark-sql hook
+- [AIRFLOW-1587] Fix CeleryExecutor import error
+- [Airflow-1640][AIRFLOW-1640] Add qubole default connection
+- [AIRFLOW-1576] Added region param to Dataproc{*}Operators
+- [AIRFLOW-1643] Add healthjump to officially using list
+- [AIRFLOW-1626] Add Azri Solutions to Airflow users
+- [AIRFLOW-1636] Add AWS and EMR connection type
+- [AIRFLOW-1527] Refactor celery config
+- [AIRFLOW-1639] Fix Fernet error handling
+- [AIRFLOW-1637] Fix Travis CI build status link
+- [AIRFLOW-1628] Fix docstring of sqlsensor
+- [AIRFLOW-1331] add SparkSubmitOperator option
+- [AIRFLOW-1627] Only query pool in SubDAG init when necessary
+- [AIRFLOW-1629] Make extra a textarea in edit connections form
+- [AIRFLOW-1368] Automatically remove Docker container on exit
+- [AIRFLOW-289] Make airflow timezone independent
+- [AIRFLOW-1356] Add `--celery_hostname` to `airflow worker`
+- [AIRFLOW-1247] Fix ignore_all_dependencies argument ignored
+- [AIRFLOW-1621] Add tests for server side paging
+- [AIRFLOW-1591] Avoid attribute error when rendering logging filename
+- [AIRFLOW-1031] Replace hard-code to DagRun.ID_PREFIX
+- [AIRFLOW-1604] Rename logger to log
+- [AIRFLOW-1512] Add PythonVirtualenvOperator
+- [AIRFLOW-1617] Fix XSS vulnerability in Variable endpoint
+- [AIRFLOW-1497] Reset hidden fields when changing connection type
+- [AIRFLOW-1619] Add poll_sleep parameter to GCP dataflow operator
+- [AIRFLOW-XXX] Remove landscape.io config
+- [AIRFLOW-XXX] Remove non working service badges
+- [AIRFLOW-1177] Fix Variable.setdefault w/existing JSON
+- [AIRFLOW-1600] Fix exception handling in get_fernet
+- [AIRFLOW-1614] Replace inspect.stack() with sys._getframe()
+- [AIRFLOW-1519] Add server side paging in DAGs list
+- [AIRFLOW-1309] Allow hive_to_druid to take tblproperties
+- [AIRFLOW-1613] Make MySqlToGoogleCloudStorageOperator compaitible with python3
+- [AIRFLOW-1603] add PAYMILL to companies list
+- [AIRFLOW-1609] Fix gitignore to ignore all venvs
+- [AIRFLOW-1601] Add configurable task cleanup time
+
+Airflow 1.9.0, 2018-01-02
 -------------------------
 
-[AIRFLOW-1525] Fix minor LICENSE and NOTICE issues
-[AIRFLOW-XXX] Bump version to 1.9.0
-[AIRFLOW-1897][AIRFLOW-1873] Task Logs for running instance not visible in WebUI
-[AIRFLOW-XXX] Make sure session is committed
-[AIRFLOW-1896] FIX bleach <> html5lib incompatibility
-[AIRFLOW-XXX] Fix log handler test
-[AIRFLOW-1873] Set TI.try_number to right value depending TI state
-[AIRFLOW-1554] Fix wrong DagFileProcessor termination method call
-[AIRFLOW-1872] Set context for all handlers including parents
-[AIRFLOW-XXX] Add dask lock files to excludes
-[AIRFLOW-1839] Fix more bugs in S3Hook boto -> boto3 migration
-[AIRFLOW-1795] Correctly call S3Hook after migration to boto3
-[AIRFLOW-1813] Bug SSH Operator empty buffer
-[AIRFLOW-1794] Remove uses of Exception.message for Python 3
-[AIRFLOW-1799] Fix logging line which raises errors
-[AIRFLOW-1102] Upgrade Gunicorn >=19.4.0
-[AIRFLOW-1756] Fix S3TaskHandler to work with Boto3-based S3Hook
-[AIRFLOW-1797] S3Hook.load_string didn't work on Python3
-[AIRFLOW-1792] Missing intervals DruidOperator
-[AIRFLOW-1789][AIRFLOW-1712] Log SSHOperator stderr to log.warning
-[AIRFLOW-1669] Fix Docker and pin Moto to 1.1.19
-[AIRFLOW-71] Add support for private Docker images
-[AIRFLOW-1779] Add keepalive packets to ssh hook
-[AIRFLOW-XXX] Give a clue what the 'ds' variable is
-[AIRFLOW-XXX] Correct typos in the faq docs page
-[AIRFLOW-1571] Add AWS Lambda Hook
-[AIRFLOW-1675] Fix docstrings for API docs
-[AIRFLOW-1712][AIRFLOW-756][AIRFLOW-751] Log SSHOperator output
-[AIRFLOW-1776] Capture stdout and stderr for logging
-[AIRFLOW-1765] Make experimental API securable without needing Kerberos.
-[AIRFLOW-1764] The web interface should not use the experimental API
-[AIRFLOW-1634] Adds task_concurrency feature
-[AIRFLOW-1018] Make processor use logging framework
-[AIRFLOW-1695] Add RedshiftHook using boto3
-[AIRFLOW-1706] Fix query error for MSSQL backend
-[AIRFLOW-1711] Use ldap3 dict for group membership
-[AIRFLOW-1757] Add missing options to SparkSubmitOperator
-[AIRFLOW-1734][Airflow 1734] Sqoop hook/operator enhancements
-[AIRFLOW-1731] Set pythonpath for logging
-[AIRFLOW-1641] Handle executor events in the scheduler
-[AIRFLOW-1744] Make sure max_tries can be set
-[AIRFLOW-1330] Add conn_type argument to CLI when adding connection
-[AIRFLOW-926] Fix JDBC Hook
-[AIRFLOW-1520] Boto3 S3Hook, S3Log
-[AIRFLOW-XXX] Fix DateTime in Tree View
-[AIRFLOW-1432] Charts label for Y axis not visible
-[AIRFLOW-1743] Verify ldap filters correctly
-[AIRFLOW-1745] Restore default signal disposition
-[AIRFLOW-1741] Correctly hide second chart on task duration page
-[AIRFLOW-1726] Add copy_expert psycopg2 method to PostgresHook
-[AIRFLOW-1698] Remove SCHEDULER_RUNS env var in systemd
-[AIRFLOW-1694] Stop using itertools.izip
-[AIRFLOW-1692] Change test_views filename to support Windows
-[AIRFLOW-1722] Fix typo in scheduler autorestart output filename
-[AIRFLOW-1691] Add better Google cloud logging documentation
-[AIRFLOW-1690] Add detail to gcs error messages
-[AIRFLOW-1682] Make S3TaskHandler write to S3 on close
-[AIRFLOW-1676] Make GCSTaskHandler write to GCS on close
-[AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file
-[AIRFLOW-1323] Made Dataproc operator parameter names consistent
-[AIRFLOW-1590] fix unused module and variable
-[AIRFLOW-988] Fix repeating SLA miss callbacks
-[AIRFLOW-1611] Customize logging
-[AIRFLOW-1668] Expose keepalives_idle for Postgres connections
-[AIRFLOW-1658] Kill Druid task on timeout
-[AIRFLOW-1669][AIRFLOW-1368] Fix Docker import
-[AIRFLOW-1560] Add AWS DynamoDB hook and operator for inserting batch items
-[AIRFLOW-1654] Show tooltips for link icons in DAGs view
-[AIRFLOW-1660] Change webpage width to full-width
-[AIRFLOW-1664] write file as binary instead of str
-[AIRFLOW-1659] Fix invalid obj attribute bug in file_task_handler.py
-[AIRFLOW-1650] Fix custom celery config loading
-[AIRFLOW-1647] Fix Spark-sql hook
-[AIRFLOW-1587] Fix CeleryExecutor import error
-[AIRFLOW-1636] Add AWS and EMR connection type
-[AIRFLOW-1527] Refactor celery config
-[AIRFLOW-1639] Fix Fernet error handling
-[AIRFLOW-1628] Fix docstring of sqlsensor
-[AIRFLOW-1331] add SparkSubmitOperator option
-[AIRFLOW-1627] Only query pool in SubDAG init when necessary
-[AIRFLOW-1629] Make extra a textarea in edit connections form
-[AIRFLOW-1621] Add tests for server side paging
-[AIRFLOW-1519] Add server side paging in DAGs list
-[AIRFLOW-289] Make airflow timezone independent
-[AIRFLOW-1356] Add `--celery_hostname` to `airflow worker`
-[AIRFLOW-1591] Avoid attribute error when rendering logging filename
-[AIRFLOW-1031] Replace hard-code to DagRun.ID_PREFIX
-[AIRFLOW-1604] Rename logger to log
-[AIRFLOW-1512] Add PythonVirtualenvOperator
-[AIRFLOW-1617] Fix XSS vulnerability in Variable endpoint
-[AIRFLOW-1497] Reset hidden fields when changing connection type
-[AIRFLOW-1177] Fix Variable.setdefault w/existing JSON
-[AIRFLOW-1600] Fix exception handling in get_fernet
-[AIRFLOW-1614] Replace inspect.stack() with sys._getframe()
-[AIRFLOW-1613] Make MySqlToGoogleCloudStorageOperator compaitible with python3
-[AIRFLOW-1609] Fix gitignore to ignore all venvs
-[AIRFLOW-1601] Add configurable task cleanup time
-[AIRFLOW-XXX] Bumping Airflow 1.9.0alpha0 version
-[AIRFLOW-1608] Handle pending job state in GCP Dataflow hook
-[AIRFLOW-1606] Use non static DAG.sync_to_db
-[AIRFLOW-1606][Airflow-1606][AIRFLOW-1605][AIRFLOW-160] DAG.sync_to_db is now a normal method
-[AIRFLOW-1602] LoggingMixin in DAG class
-[AIRFLOW-1593] expose load_string in WasbHook
-[AIRFLOW-1597] Add GameWisp as Airflow user
-[AIRFLOW-1594] Don't install test packages into python root.[]
-[AIRFLOW-1582] Improve logging within Airflow
-[AIRFLOW-1476] add INSTALL instruction for source releases
-[AIRFLOW-XXX] Save username and password in airflow-pr
-[AIRFLOW-1522] Increase text size for var field in variables for MySQL
-[AIRFLOW-950] Missing AWS integrations on documentation::integrations
-[AIRFLOW-XXX] 1.8.2 release notes
-[AIRFLOW-1573] Remove `thrift < 0.10.0` requirement
-[AIRFLOW-1584] Remove insecure /headers endpoint
-[AIRFLOW-1586] Add mapping for date type to mysql_to_gcs operator
-[AIRFLOW-1579] Adds support for jagged rows in Bigquery hook for BQ load jobs
-[AIRFLOW-1577] Add token support to DatabricksHook
-[AIRFLOW-1580] Error in string formating
-[AIRFLOW-1567] Updated docs for Google ML Engine operators/hooks
-[AIRFLOW-1574] add 'to' attribute to templated vars of email operator
-[AIRFLOW-1572] add carbonite to company list
-[AIRFLOW-1568] Fix typo in BigQueryHook
-[AIRFLOW-1493][AIRFLOW-XXXX][WIP] fixed dumb thing
-[AIRFLOW-1567][Airflow-1567] Renamed cloudml hook and operator to mlengine
-[AIRFLOW-1568] Add datastore export/import operators
-[AIRFLOW-1564] Use Jinja2 to render logging filename
-[AIRFLOW-1562] Spark-sql logging contains deadlock
-[AIRFLOW-1556][Airflow 1556] Add support for SQL parameters in BigQueryBaseCursor
-[AIRFLOW-108] Add CreditCards.com to companies list
-[AIRFLOW-1541] Add channel to template fields of slack_operator
-[AIRFLOW-1535] Add service account/scopes in dataproc
-[AIRFLOW-1384] Add to README.md CaDC/ARGO
-[AIRFLOW-1546] add Zymergen 80to org list in README
-[AIRFLOW-1545] Add Nextdoor to companies list
-[AIRFLOW-1544] Add DataFox to companies list
-[AIRFLOW-1529] Add logic supporting quoted newlines in Google BigQuery load jobs
-[AIRFLOW-1521] Fix emplate rendering for BigqueryTableDeleteOperator
-[AIRFLOW-1324] Generalize Druid operator and hook
-[AIRFLOW-1516] Fix error handling getting fernet
-[AIRFLOW-1420][AIRFLOW-1473] Fix deadlock check
-[AIRFLOW-1495] Fix migration on index on job_id
-[AIRFLOW-1483] Making page size consistent in list
-[AIRFLOW-1495] Add TaskInstance index on job_id
-[AIRFLOW-855] Replace PickleType with LargeBinary in XCom
-[AIRFLOW-1505] Document when Jinja substitution occurs
-[AIRFLOW-1504] Log dataproc cluster name
-[AIRFLOW-1239] Fix unicode error for logs in base_task_runner
-[AIRFLOW-1280] Fix Gantt chart height
-[AIRFLOW-1507] Template parameters in file_to_gcs operator
-[AIRFLOW-1452] workaround lock on method
-[AIRFLOW-1385] Make Airflow task logging configurable
-[AIRFLOW-940] Handle error on variable decrypt
-[AIRFLOW-1492] Add gauge for task successes/failures
-[AIRFLOW-1443] Update Airflow configuration documentation
-[AIRFLOW-1486] Unexpected S3 writing log error
-[AIRFLOW-1487] Added links to all companies officially using Airflow
-[AIRFLOW-1489] Fix typo in BigQueryCheckOperator
-[AIRFLOW-1349] Fix backfill to respect limits
-[AIRFLOW-1478] Chart owner column should be sortable
-[AIRFLOW-1397][AIRFLOW-1] No Last Run column data displyed in Airflow UI 1.8.1
-[AIRFLOW-1474] Add dag_id regex feature for `airflow clear` command
-[AIRFLOW-1445] Changing HivePartitionSensor UI color to lighter shade
-[AIRFLOW-1359] Use default_args in Cloud ML eval
-[AIRFLOW-1389] Support createDisposition in BigQueryOperator
-[AIRFLOW-1349] Refactor BackfillJob _execute
-[AIRFLOW-1459] Fixed broken integration .rst formatting
-[AIRFLOW-1448] Revert "Fix cli reading logfile in memory"
-[AIRFLOW-1398] Allow ExternalTaskSensor to wait on multiple runs of a task
-[AIRFLOW-1399] Fix cli reading logfile in memory
-[AIRFLOW-1442] Remove extra space from ignore_all_deps generated command
-[AIRFLOW-1438] Change batch size per query in scheduler
-[AIRFLOW-1439] Add max billing tier for the BQ Hook and Operator
-[AIRFLOW-1437] Modify BigQueryTableDeleteOperator
-[Airflow 1332] Split logs based on try number
-[AIRFLOW-1385] Create abstraction for Airflow task logging
-[AIRFLOW-756][AIRFLOW-751] Replace ssh hook, operator & sftp operator with paramiko based
-[AIRFLOW-1393][[AIRFLOW-1393] Enable Py3 tests in contrib/spark_submit_hook[
-[AIRFLOW-1345] Dont expire TIs on each scheduler loop
-[AIRFLOW-1059] Reset orphaned tasks in batch for scheduler
-[AIRFLOW-1255] Fix SparkSubmitHook output deadlock
-[AIRFLOW-1359] Add Google CloudML utils for model evaluation
-[AIRFLOW-1247] Fix ignore all dependencies argument ignored
-[AIRFLOW-1401] Standardize cloud ml operator arguments
-[AIRFLOW-1394] Add quote_character param to GCS hook and operator
-[AIRFLOW-1402] Cleanup SafeConfigParser DeprecationWarning
-[AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array -- it's already an array.[
-[AIRFLOW-1384] Add ARGO/CaDC as a Airflow user
-[AIRFLOW-1357] Fix scheduler zip file support
-[AIRFLOW-1382] Add working dir option to DockerOperator
-[AIRFLOW-1388] Add Cloud ML Engine operators to integration doc
-[AIRFLOW-1387] Add unicode string prefix
-[AIRFLOW-1366] Add max_tries to task instance
-[AIRFLOW-1300] Enable table creation with TBLPROPERTIES
-[AIRFLOW-1271] Add Google CloudML Training Operator
-[AIRFLOW-300] Add Google Pubsub hook and operator
-[AIRFLOW-1343] Fix dataproc label format
-[AIRFLOW-1367] Pass Content-ID To reference inline images in an email, we need to be able to add <img src="cid:{}"/> to the HTML. However currently the Content-ID (cid) is not passed, so we need to add it
-[AIRFLOW-1265] Fix celery executor parsing CELERY_SSL_ACTIVE
-[AIRFLOW-1272] Google Cloud ML Batch Prediction Operator
-[AIRFLOW-1352][AIRFLOW-1335] Revert MemoryHandler change ()[]
-[AIRFLOW-1350] Add query_uri param to Hive/SparkSQL DataProc operator
-[AIRFLOW-1334] Check if tasks are backfill on scheduler in a join
-[AIRFLOW-1343] Add Airflow default label to the dataproc operator
-[AIRFLOW-1273] Add Google Cloud ML version and model operators
-[AIRFLOW-1273]AIRFLOW-1273] Add Google Cloud ML version and model operators
-[AIRFLOW-1321] Fix hidden field key to ignore case
-[AIRFLOW-1337] Make log_format key names lowercase
-[AIRFLOW-1338][AIRFLOW-782] Add GCP dataflow hook runner change to UPDATING.md
-[AIRFLOW-801] Remove outdated docstring on BaseOperator
-[AIRFLOW-1344] Fix text encoding bug when reading logs for Python 3.5
-[AIRFLOW-1338] Fix incompatible GCP dataflow hook
-[AIRFLOW-1333] Enable copy function for Google Cloud Storage Hook
-[AIRFLOW-1337] Allow log format customization via airflow.cfg
-[AIRFLOW-1320] Update LetsBonus users in README
-[AIRFLOW-1335] Use MemoryHandler for buffered logging
-[AIRFLOW-1339] Add Drivy to the list of users
-[AIRFLOW-1275] Put 'airflow pool' into API
-[AIRFLOW-1296] Propagate SKIPPED to all downstream tasks
-[AIRFLOW-1317] Fix minor issues in API reference
-[AIRFLOW-1308] Disable nanny usage for Dask
-[AIRFLOW-1172] Support nth weekday of the month cron expression
-[AIRFLOW-936] Add clear/mark success for DAG in the UI
-[AIRFLOW-1294] Backfills can loose tasks to execute
-[AIRFLOW-1299] Support imageVersion in Google Dataproc cluster
-[AIRFLOW-1291] Update NOTICE and LICENSE files to match ASF requirements
-[AIRFLOW-1301] Add New Relic to list of companies
-[AIRFLOW-1289] Removes restriction on number of scheduler threads
-[AIRFLOW-1024] Ignore celery executor errors (#49)
-[AIRFLOW-1265] Fix exception while loading celery configurations
-[AIRFLOW-1290] set docs author to 'Apache Airflow'
-[AIRFLOW-1242] Allowing project_id to have a colon in it.
-[AIRFLOW-1282] Fix known event column sorting
-[AIRFLOW-1166] Speed up _change_state_for_tis_without_dagrun
-[AIRFLOW-1208] Speed-up cli tests
-[AIRFLOW-1192] Some enhancements to qubole_operator
-[AIRFLOW-1281] Sort variables by key field by default
-[AIRFLOW-1277] Forbid KE creation with empty fields
-[AIRFLOW-1276] Forbid event creation with end_data earlier than start_date
-[AIRFLOW-1263] Dynamic height for charts
-[AIRFLOW-1266] Increase width of gantt y axis
-[AIRFLOW-1244] Forbid creation of a pool with empty name
-[AIRFLOW-1274][HTTPSENSOR] Rename parameter params to data
-[AIRFLOW-654] Add SSL Config Option for CeleryExecutor w/ RabbitMQ - Add BROKER_USE_SSL config to give option to send AMQP messages over SSL - Can be set using usual airflow options (e.g. airflow.cfg, env vars, etc.)
-[AIRFLOW-1256] Add United Airlines to readme
-[AIRFLOW-1251] Add eRevalue to Airflow users
-[AIRFLOW-908] Print hostname at the start of cli run
-[AIRFLOW-1237] Fix IN-predicate sqlalchemy warning
-[AIRFLOW-1243] DAGs table has no default entries to show
-[AIRFLOW-1245] Fix random failure in test_trigger_dag_for_date
-[AIRFLOW-1248] Fix wrong conf name for worker timeout
-[AIRFLOW-1197] : SparkSubmitHook on_kill error
-[AIRFLOW-1191] : SparkSubmitHook custom cmd
-[AIRFLOW-1234] Cover utils.operator_helpers with UTs
-[AIRFLOW-1217] Enable Sqoop logging
-[AIRFLOW-645] Support HTTPS connections in HttpHook
-[AIRFLOW-1231] Use flask_wtf.CSRFProtect
-[AIRFLOW-1232] Remove deprecated readfp warning
-[AIRFLOW-1233] Cover utils.json with unit tests
-[AIRFLOW-1227] Remove empty column on the Logs view
-[AIRFLOW-1226] Remove empty column on the Jobs view
-[AIRFLOW-1221] Fix templating bug with DatabricksSubmitRunOperator
-[AIRFLOW-1210] Enable DbApiHook unit tests
-[AIRFLOW-1199] Fix create modal
-[AIRFLOW-1200] Forbid creation of a variable with an empty key
-[AIRFLOW-1207] Enable utils.helpers unit tests
-[AIRFLOW-1213] Add hcatalog parameters to sqoop
-[AIRFLOW-1201] Update deprecated 'nose-parameterized'
-[AIRFLOW-1186] Sort dag.get_task_instances by execution_date
-[AIRFLOW-1203] Pin Google API client version to fix OAuth issue
-[AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before.
-[AIRFLOW-1180] Fix flask-wtf version for test_csrf_rejection
-[AIRFLOW-993] Update date inference logic
-[AIRFLOW-1170] DbApiHook insert_rows inserts parameters separately
-[AIRFLOW-1041] Do not shadow xcom_push method[]
-[AIRFLOW-860][AIRFLOW-935] Fix plugin executor import cycle and executor selection
-[AIRFLOW-1189] Fix get a DataFrame using BigQueryHook failing
-[AIRFLOW-1184] SparkSubmitHook does not split args
-[AIRFLOW-1182] SparkSubmitOperator template field
-[AIRFLOW-823] Allow specifying execution date in task_info API
-[AIRFLOW-1175] Add Pronto Tools to Airflow user list
-[AIRFLOW-1150] Fix scripts execution in sparksql hook[]
-[AIRFLOW-1141] remove crawl_for_tasks
-[AIRFLOW-1193] Add Checkr to company using Airflow
-[AIRFLOW-1168] Add closing() to all connections and cursors
-[AIRFLOW-1188] Add max_bad_records param to GoogleCloudStorageToBigQueryOperator
-[AIRFLOW-1187][AIRFLOW-1185] Fix PyPi package names in documents
-[AIRFLOW-1185] Fix PyPi URL in templates
-[AIRFLOW-XXX] Updating CHANGELOG, README, and UPDATING after 1.8.1 release
-[AIRFLOW-1181] Add delete and list functionality to gcs_hook
-[AIRFLOW-1179] Fix Pandas 0.2x breaking Google BigQuery change
-[AIRFLOW-1167] Support microseconds in FTPHook modification time
-[AIRFLOW-1173] Add Robinhood to who uses Airflow
-[AIRFLOW-945][AIRFLOW-941] Remove psycopg2 connection workaround
-[AIRFLOW-1140] DatabricksSubmitRunOperator should template the "json" field.
-[AIRFLOW-1160] Update Spark parameters for Mesos
-[AIRFLOW 1149][AIRFLOW-1149] Allow for custom filters in Jinja2 templates
-[AIRFLOW-1036] Randomize exponential backoff
-[AIRFLOW-1155] Add Tails.com to community
-[AIRFLOW-1142] Do not reset orphaned state for backfills
-[AIRFLOW-492] Make sure stat updates cannot fail a task
-[AIRFLOW-1119] Fix unload query so headers are on first row[]
-[AIRFLOW-1089] Add Spark application arguments
-[AIRFLOW-1125] Document encrypted connections
-[AIRFLOW-1122] Increase stroke width in UI
-[AIRFLOW-1138] Add missing licenses to files in scripts directory
-(AIRFLOW-11-38) [AIRFLOW-1136] Capture invalid arguments for Sqoop
-[AIRFLOW-1127] Move license notices to LICENSE
-[AIRFLOW-1118] Add evo.company to Airflow users
-[AIRFLOW-1121][AIRFLOW-1004] Fix `airflow webserver --pid` to write out pid file
-[AIRFLOW-1124] Do not set all tasks to scheduled in backfill
-[AIRFLOW-1120] Update version view to include Apache prefix
-[AIRFLOW-1091] Add script that can compare jira target against merges
-[AIRFLOW-1107] Add support for ftps non-default port
-[AIRFLOW-1000] Rebrand distribution to Apache Airflow
-[AIRFLOW-1094] Run unit tests under contrib in Travis
-[AIRFLOW-1112] Log which pool when pool is full in scheduler
-[AIRFLOW-1106] Add Groupalia/Letsbonus to the ReadMe
-[AIRFLOW-1109] Use kill signal to kill processes and log results
-[AIRFLOW-1074] Don't count queued tasks for concurrency limits
-[AIRFLOW-1095] Make ldap_auth memberOf come from configuration
-[AIRFLOW-1090] Add HBO
-[AIRFLOW-1035] Use binary exponential backoff
-[AIRFLOW-1081] Improve performance of duration chart
-[AIRFLOW-1078] Fix latest_runs endpoint for old flask versions
-[AIRFLOW-1085] Enhance the SparkSubmitOperator
-[AIRFLOW-1050] Do not count up_for_retry as not ready
-[AIRFLOW-1028] Databricks Operator for Airflow
-[AIRFLOW-1075] Security docs cleanup
-[AIRFLOW-1033][AIFRLOW-1033] Fix ti_deps for no schedule dags
-[AIRFLOW-1016] Allow HTTP HEAD request method on HTTPSensor
-[AIRFLOW-970] Load latest_runs on homepage async
-[AIRFLOW-111] Include queued tasks in scheduler concurrency check
-[AIRFLOW-1001] Fix landing times if there is no following schedule
-[AIRFLOW-1065] Add functionality for Azure Blob Storage over wasb://
-[AIRFLOW-947] Improve exceptions for unavailable Presto cluster
-[AIRFLOW-1067] use example.com in examples
-[AIRFLOW-1064] Change default sort to job_id for TaskInstanceModelView
-[AIRFLOW-1030][AIRFLOW-1] Fix hook import for HttpSensor
-[AIRFLOW-1051] Add a test for resetdb to CliTests
-[AIRFLOW-1004][AIRFLOW-276] Fix `airflow webserver -D` to run in background
-[AIRFLOW-1062] Fix DagRun#find to return correct result
-[AIRFLOW-1011] Fix bug in BackfillJob._execute() for SubDAGs
-[AIRFLOW-1038] Specify celery serialization options explicitly
-[AIRFLOW-1054] Fix broken import in test_dag
-[AIRFLOW-1007] Use Jinja sandbox for chart_data endpoint
-[AIRFLOW-719] Fix race condition in ShortCircuit, Branch and LatestOnly
-[AIRFLOW-1043] Fix doc strings of operators
-[AIRFLOW-840] Make ticket renewer python3 compatible
-[AIRFLOW-985] Extend the sqoop operator and hook
-[AIRFLOW-1034] Make it possible to connect to S3 in sigv4 regions
-[AIRFLOW-1045] Make log level configurable via airflow.cfg
-[AIRFLOW-1047] Sanitize strings passed to Markup
-[AIRFLOW-1040] Fix some small typos in comments and docstrings
-[AIRFLOW-1017] get_task_instance shouldn't throw exception when no TI
-[AIRFLOW-1006] Add config_templates to MANIFEST
-[AIRFLOW-999] Add support for Redis database
-[AIRFLOW-1009] Remove SQLOperator from Concepts page
-[AIRFLOW-1006] Move config templates to separate files
-[AIRFLOW-1005] Improve Airflow startup time
-[AIRFLOW-1010] Add convenience script for signing releases
-[AIRFLOW-995] Remove reference to actual Airflow issue
-[AIRFLOW-681] homepage doc link should pointing to apache repo not airbnb repo
-[AIRFLOW-705][AIRFLOW-706] Fix run_command bugs
-[AIRFLOW-990] Fix Py27 unicode logging in DockerOperator
-[AIRFLOW-963] Fix non-rendered code examples
-[AIRFLOW-969] Catch bad python_callable argument
-[AIRFLOW-984] Enable subclassing of SubDagOperator
-[AIRFLOW-997] Update setup.cfg to point to Apache
-[AIRFLOW-994] Add MiNODES to the official airflow user list
-[AIRFLOW-995][AIRFLOW-1] Update GitHub PR Template
-[AIRFLOW-989] Do not mark dag run successful if unfinished tasks
-[AIRFLOW-903] New configuration setting for the default dag view
-[AIRFLOW-979] Add GovTech GDS
-[AIRFLOW-933] Replace eval with literal_eval to prevent RCE
-[AIRFLOW-974] Fix mkdirs race condition
-[AIRFLOW-917] Fix formatting of error message
-[AIRFLOW-770] Refactor BaseHook so env vars are always read
-[AIRFLOW-900] Double trigger should not kill original task instance
-[AIRFLOW-900] Fixes bugs in LocalTaskJob for double run protection
-[AIRFLOW-932][AIRFLOW-932][AIRFLOW-921][AIRFLOW-910] Do not mark tasks removed when backfilling[
-[AIRFLOW-961] run onkill when SIGTERMed
-[AIRFLOW-910] Use parallel task execution for backfills
-[AIRFLOW-967] Wrap strings in native for py2 ldap compatibility
-[AIRFLOW-958] Improve tooltip readability
-AIRFLOW-959 Cleanup and reorganize .gitignore
-AIRFLOW-960 Add .editorconfig file
-[AIRFLOW-931] Do not set QUEUED in TaskInstances
-[AIRFLOW-956] Get docs working on readthedocs.org
-[AIRFLOW-954] Fix configparser ImportError
-[AIRFLOW-941] Use defined parameters for psycopg2
-[AIRFLOW-943] Update Digital First Media in users list
-[AIRFLOW-942] Add mytaxi to Airflow users
-[AIRFLOW-939] add .swp to gitginore
-[AIRFLOW-719] Prevent DAGs from ending prematurely
-[AIRFLOW-938] Use test for True in task_stats queries
-[AIRFLOW-937] Improve performance of task_stats
-[AIRFLOW-933] use ast.literal_eval rather eval because ast.literal_eval does not execute input.
-[AIRFLOW-925] Revert airflow.hooks change that cherry-pick picked
-[AIRFLOW-919] Running tasks with no start date shouldn't break a DAGs UI
-[AIRFLOW-802][AIRFLOW-1] Add spark-submit operator/hook
-[AIRFLOW-725] Use keyring to store credentials for JIRA
-[AIRFLOW-916] Remove deprecated readfp function
-[AIRFLOW-911] Add coloring and timing to tests
-[AIRFLOW-906] Update Code icon from lightning bolt to file
-[AIRFLOW-897] Prevent dagruns from failing with unfinished tasks
-[AIRFLOW-896] Remove unicode to 8-bit conversion in BigQueryOperator
-[AIRFLOW-899] Tasks in SCHEDULED state should be white in the UI instead of black
-[AIRFLOW-895] Address Apache release incompliancies
-[AIRFLOW-893][AIRFLOW-510] Fix crashing webservers when a dagrun has no start date
-[AIRFLOW-880] Make webserver serve logs in a sane way for remote logs
-[AIRFLOW-889] Fix minor error in the docstrings for BaseOperator
-[AIRFLOW-809][AIRFLOW-1] Use __eq__ ColumnOperator When Testing Booleans
-[AIRFLOW-875] Add template to HttpSensor params
-[AIRFLOW-866] Add FTPSensor
-[AIRFLOW-881] Check if SubDagOperator is in DAG context manager
-[AIRFLOW-885] Add change.org to the users list
-[AIRFLOW-836] Use POST and CSRF for state changing endpoints
-[AIRFLOW-862] Fix Unit Tests for DaskExecutor
-[AIRFLOW-887] Support future v0.16
-[AIRFLOW-886] Pass result to post_execute() hook
-[AIRFLOW-871] change logging.warn() into warning()
-[AIRFLOW-882] Remove unnecessary dag>>op assignment in docs
-[AIRFLOW-861] make pickle_info endpoint be login_required
-[AIRFLOW-869] Refactor mark success functionality
-[AIRFLOW-877] Remove .sql template extension from GCS download operator
-[AIRFLOW-826] Add Zendesk hook
-[AIRFLOW-842] do not query the DB with an empty IN clause
-[AIRFLOW-834] change raise StopIteration into return
-[AIRFLOW-832] Let debug server run without SSL
-[AIRFLOW-862] Add DaskExecutor
-[AIRFLOW-858] Configurable database name for DB operators
-[AIRFLOW-863] Example DAGs should have recent start dates
-[AIRFLOW-853] use utf8 encoding for stdout line decode
-[AIRFLOW-857] Use library assert statements instead of conditionals
-[AIRFLOW-856] Make sure execution date is set for local client
-[AIRFLOW-854] Add OKI as Airflow user
-[AIRFLOW-830][AIRFLOW-829][AIRFLOW-88] Reduce Travis log verbosity
-[AIRFLOW-814] Fix Presto*CheckOperator.__init__
-[AIRFLOW-793] Enable compressed loading in S3ToHiveTransfer
-[AIRFLOW-844] Fix cgroups directory creation
-[AIRFLOW-831] Restore import to fix broken tests
-[AIRFLOW-794] Access DAGS_FOLDER and SQL_ALCHEMY_CONN exclusively from settings
-[AIRFLOW-694] Fix config behaviour for empty envvar
-[AIRFLOW-365] Set dag.fileloc explicitly and use for Code view
-[AIRFLOW-781] Allow DataFlowOperators to accept jobs stored in GCS
+- [AIRFLOW-1525] Fix minor LICENSE and NOTICE issues
+- [AIRFLOW-XXX] Bump version to 1.9.0
+- [AIRFLOW-1897][AIRFLOW-1873] Task Logs for running instance not visible in WebUI
+- [AIRFLOW-XXX] Make sure session is committed
+- [AIRFLOW-1896] FIX bleach <> html5lib incompatibility
+- [AIRFLOW-XXX] Fix log handler test
+- [AIRFLOW-1873] Set TI.try_number to right value depending TI state
+- [AIRFLOW-1554] Fix wrong DagFileProcessor termination method call
+- [AIRFLOW-1872] Set context for all handlers including parents
+- [AIRFLOW-XXX] Add dask lock files to excludes
+- [AIRFLOW-1839] Fix more bugs in S3Hook boto -> boto3 migration
+- [AIRFLOW-1795] Correctly call S3Hook after migration to boto3
+- [AIRFLOW-1813] Bug SSH Operator empty buffer
+- [AIRFLOW-1794] Remove uses of Exception.message for Python 3
+- [AIRFLOW-1799] Fix logging line which raises errors
+- [AIRFLOW-1102] Upgrade Gunicorn >=19.4.0
+- [AIRFLOW-1756] Fix S3TaskHandler to work with Boto3-based S3Hook
+- [AIRFLOW-1797] S3Hook.load_string didn't work on Python3
+- [AIRFLOW-1792] Missing intervals DruidOperator
+- [AIRFLOW-1789][AIRFLOW-1712] Log SSHOperator stderr to log.warning
+- [AIRFLOW-1669] Fix Docker and pin Moto to 1.1.19
+- [AIRFLOW-71] Add support for private Docker images
+- [AIRFLOW-1779] Add keepalive packets to ssh hook
+- [AIRFLOW-XXX] Give a clue what the 'ds' variable is
+- [AIRFLOW-XXX] Correct typos in the faq docs page
+- [AIRFLOW-1571] Add AWS Lambda Hook
+- [AIRFLOW-1675] Fix docstrings for API docs
+- [AIRFLOW-1712][AIRFLOW-756][AIRFLOW-751] Log SSHOperator output
+- [AIRFLOW-1776] Capture stdout and stderr for logging
+- [AIRFLOW-1765] Make experimental API securable without needing Kerberos.
+- [AIRFLOW-1764] The web interface should not use the experimental API
+- [AIRFLOW-1634] Adds task_concurrency feature
+- [AIRFLOW-1018] Make processor use logging framework
+- [AIRFLOW-1695] Add RedshiftHook using boto3
+- [AIRFLOW-1706] Fix query error for MSSQL backend
+- [AIRFLOW-1711] Use ldap3 dict for group membership
+- [AIRFLOW-1757] Add missing options to SparkSubmitOperator
+- [AIRFLOW-1734][Airflow 1734] Sqoop hook/operator enhancements
+- [AIRFLOW-1731] Set pythonpath for logging
+- [AIRFLOW-1641] Handle executor events in the scheduler
+- [AIRFLOW-1744] Make sure max_tries can be set
+- [AIRFLOW-1330] Add conn_type argument to CLI when adding connection
+- [AIRFLOW-926] Fix JDBC Hook
+- [AIRFLOW-1520] Boto3 S3Hook, S3Log
+- [AIRFLOW-XXX] Fix DateTime in Tree View
+- [AIRFLOW-1432] Charts label for Y axis not visible
+- [AIRFLOW-1743] Verify ldap filters correctly
+- [AIRFLOW-1745] Restore default signal disposition
+- [AIRFLOW-1741] Correctly hide second chart on task duration page
+- [AIRFLOW-1726] Add copy_expert psycopg2 method to PostgresHook
+- [AIRFLOW-1698] Remove SCHEDULER_RUNS env var in systemd
+- [AIRFLOW-1694] Stop using itertools.izip
+- [AIRFLOW-1692] Change test_views filename to support Windows
+- [AIRFLOW-1722] Fix typo in scheduler autorestart output filename
+- [AIRFLOW-1691] Add better Google cloud logging documentation
+- [AIRFLOW-1690] Add detail to gcs error messages
+- [AIRFLOW-1682] Make S3TaskHandler write to S3 on close
+- [AIRFLOW-1676] Make GCSTaskHandler write to GCS on close
+- [AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file
+- [AIRFLOW-1323] Made Dataproc operator parameter names consistent
+- [AIRFLOW-1590] fix unused module and variable
+- [AIRFLOW-988] Fix repeating SLA miss callbacks
+- [AIRFLOW-1611] Customize logging
+- [AIRFLOW-1668] Expose keepalives_idle for Postgres connections
+- [AIRFLOW-1658] Kill Druid task on timeout
+- [AIRFLOW-1669][AIRFLOW-1368] Fix Docker import
+- [AIRFLOW-1560] Add AWS DynamoDB hook and operator for inserting batch items
+- [AIRFLOW-1654] Show tooltips for link icons in DAGs view
+- [AIRFLOW-1660] Change webpage width to full-width
+- [AIRFLOW-1664] write file as binary instead of str
+- [AIRFLOW-1659] Fix invalid obj attribute bug in file_task_handler.py
+- [AIRFLOW-1650] Fix custom celery config loading
+- [AIRFLOW-1647] Fix Spark-sql hook
+- [AIRFLOW-1587] Fix CeleryExecutor import error
+- [AIRFLOW-1636] Add AWS and EMR connection type
+- [AIRFLOW-1527] Refactor celery config
+- [AIRFLOW-1639] Fix Fernet error handling
+- [AIRFLOW-1628] Fix docstring of sqlsensor
+- [AIRFLOW-1331] add SparkSubmitOperator option
+- [AIRFLOW-1627] Only query pool in SubDAG init when necessary
+- [AIRFLOW-1629] Make extra a textarea in edit connections form
+- [AIRFLOW-1621] Add tests for server side paging
+- [AIRFLOW-1519] Add server side paging in DAGs list
+- [AIRFLOW-289] Make airflow timezone independent
+- [AIRFLOW-1356] Add `--celery_hostname` to `airflow worker`
+- [AIRFLOW-1591] Avoid attribute error when rendering logging filename
+- [AIRFLOW-1031] Replace hard-code to DagRun.ID_PREFIX
+- [AIRFLOW-1604] Rename logger to log
+- [AIRFLOW-1512] Add PythonVirtualenvOperator
+- [AIRFLOW-1617] Fix XSS vulnerability in Variable endpoint
+- [AIRFLOW-1497] Reset hidden fields when changing connection type
+- [AIRFLOW-1177] Fix Variable.setdefault w/existing JSON
+- [AIRFLOW-1600] Fix exception handling in get_fernet
+- [AIRFLOW-1614] Replace inspect.stack() with sys._getframe()
+- [AIRFLOW-1613] Make MySqlToGoogleCloudStorageOperator compaitible with python3
+- [AIRFLOW-1609] Fix gitignore to ignore all venvs
+- [AIRFLOW-1601] Add configurable task cleanup time
+- [AIRFLOW-XXX] Bumping Airflow 1.9.0alpha0 version
+- [AIRFLOW-1608] Handle pending job state in GCP Dataflow hook
+- [AIRFLOW-1606] Use non static DAG.sync_to_db
+- [AIRFLOW-1606][Airflow-1606][AIRFLOW-1605][AIRFLOW-160] DAG.sync_to_db is now a normal method
+- [AIRFLOW-1602] LoggingMixin in DAG class
+- [AIRFLOW-1593] expose load_string in WasbHook
+- [AIRFLOW-1597] Add GameWisp as Airflow user
+- [AIRFLOW-1594] Don't install test packages into python root.[]
+- [AIRFLOW-1582] Improve logging within Airflow
+- [AIRFLOW-1476] add INSTALL instruction for source releases
+- [AIRFLOW-XXX] Save username and password in airflow-pr
+- [AIRFLOW-1522] Increase text size for var field in variables for MySQL
+- [AIRFLOW-950] Missing AWS integrations on documentation::integrations
+- [AIRFLOW-XXX] 1.8.2 release notes
+- [AIRFLOW-1573] Remove `thrift < 0.10.0` requirement
+- [AIRFLOW-1584] Remove insecure /headers endpoint
+- [AIRFLOW-1586] Add mapping for date type to mysql_to_gcs operator
+- [AIRFLOW-1579] Adds support for jagged rows in Bigquery hook for BQ load jobs
+- [AIRFLOW-1577] Add token support to DatabricksHook
+- [AIRFLOW-1580] Error in string formating
+- [AIRFLOW-1567] Updated docs for Google ML Engine operators/hooks
+- [AIRFLOW-1574] add 'to' attribute to templated vars of email operator
+- [AIRFLOW-1572] add carbonite to company list
+- [AIRFLOW-1568] Fix typo in BigQueryHook
+- [AIRFLOW-1493][AIRFLOW-XXXX][WIP] fixed dumb thing
+- [AIRFLOW-1567][Airflow-1567] Renamed cloudml hook and operator to mlengine
+- [AIRFLOW-1568] Add datastore export/import operators
+- [AIRFLOW-1564] Use Jinja2 to render logging filename
+- [AIRFLOW-1562] Spark-sql logging contains deadlock
+- [AIRFLOW-1556][Airflow 1556] Add support for SQL parameters in BigQueryBaseCursor
+- [AIRFLOW-108] Add CreditCards.com to companies list
+- [AIRFLOW-1541] Add channel to template fields of slack_operator
+- [AIRFLOW-1535] Add service account/scopes in dataproc
+- [AIRFLOW-1384] Add to README.md CaDC/ARGO
+- [AIRFLOW-1546] add Zymergen 80to org list in README
+- [AIRFLOW-1545] Add Nextdoor to companies list
+- [AIRFLOW-1544] Add DataFox to companies list
+- [AIRFLOW-1529] Add logic supporting quoted newlines in Google BigQuery load jobs
+- [AIRFLOW-1521] Fix emplate rendering for BigqueryTableDeleteOperator
+- [AIRFLOW-1324] Generalize Druid operator and hook
+- [AIRFLOW-1516] Fix error handling getting fernet
+- [AIRFLOW-1420][AIRFLOW-1473] Fix deadlock check
+- [AIRFLOW-1495] Fix migration on index on job_id
+- [AIRFLOW-1483] Making page size consistent in list
+- [AIRFLOW-1495] Add TaskInstance index on job_id
+- [AIRFLOW-855] Replace PickleType with LargeBinary in XCom
+- [AIRFLOW-1505] Document when Jinja substitution occurs
+- [AIRFLOW-1504] Log dataproc cluster name
+- [AIRFLOW-1239] Fix unicode error for logs in base_task_runner
+- [AIRFLOW-1280] Fix Gantt chart height
+- [AIRFLOW-1507] Template parameters in file_to_gcs operator
+- [AIRFLOW-1452] workaround lock on method
+- [AIRFLOW-1385] Make Airflow task logging configurable
+- [AIRFLOW-940] Handle error on variable decrypt
+- [AIRFLOW-1492] Add gauge for task successes/failures
+- [AIRFLOW-1443] Update Airflow configuration documentation
+- [AIRFLOW-1486] Unexpected S3 writing log error
+- [AIRFLOW-1487] Added links to all companies officially using Airflow
+- [AIRFLOW-1489] Fix typo in BigQueryCheckOperator
+- [AIRFLOW-1349] Fix backfill to respect limits
+- [AIRFLOW-1478] Chart owner column should be sortable
+- [AIRFLOW-1397][AIRFLOW-1] No Last Run column data displyed in Airflow UI 1.8.1
+- [AIRFLOW-1474] Add dag_id regex feature for `airflow clear` command
+- [AIRFLOW-1445] Changing HivePartitionSensor UI color to lighter shade
+- [AIRFLOW-1359] Use default_args in Cloud ML eval
+- [AIRFLOW-1389] Support createDisposition in BigQueryOperator
+- [AIRFLOW-1349] Refactor BackfillJob _execute
+- [AIRFLOW-1459] Fixed broken integration .rst formatting
+- [AIRFLOW-1448] Revert "Fix cli reading logfile in memory"
+- [AIRFLOW-1398] Allow ExternalTaskSensor to wait on multiple runs of a task
+- [AIRFLOW-1399] Fix cli reading logfile in memory
+- [AIRFLOW-1442] Remove extra space from ignore_all_deps generated command
+- [AIRFLOW-1438] Change batch size per query in scheduler
+- [AIRFLOW-1439] Add max billing tier for the BQ Hook and Operator
+- [AIRFLOW-1437] Modify BigQueryTableDeleteOperator
+- [Airflow 1332] Split logs based on try number
+- [AIRFLOW-1385] Create abstraction for Airflow task logging
+- [AIRFLOW-756][AIRFLOW-751] Replace ssh hook, operator & sftp operator with paramiko based
+- [AIRFLOW-1393][[AIRFLOW-1393] Enable Py3 tests in contrib/spark_submit_hook[
+- [AIRFLOW-1345] Dont expire TIs on each scheduler loop
+- [AIRFLOW-1059] Reset orphaned tasks in batch for scheduler
+- [AIRFLOW-1255] Fix SparkSubmitHook output deadlock
+- [AIRFLOW-1359] Add Google CloudML utils for model evaluation
+- [AIRFLOW-1247] Fix ignore all dependencies argument ignored
+- [AIRFLOW-1401] Standardize cloud ml operator arguments
+- [AIRFLOW-1394] Add quote_character param to GCS hook and operator
+- [AIRFLOW-1402] Cleanup SafeConfigParser DeprecationWarning
+- [AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array -- it's already an array.[
+- [AIRFLOW-1384] Add ARGO/CaDC as a Airflow user
+- [AIRFLOW-1357] Fix scheduler zip file support
+- [AIRFLOW-1382] Add working dir option to DockerOperator
+- [AIRFLOW-1388] Add Cloud ML Engine operators to integration doc
+- [AIRFLOW-1387] Add unicode string prefix
+- [AIRFLOW-1366] Add max_tries to task instance
+- [AIRFLOW-1300] Enable table creation with TBLPROPERTIES
+- [AIRFLOW-1271] Add Google CloudML Training Operator
+- [AIRFLOW-300] Add Google Pubsub hook and operator
+- [AIRFLOW-1343] Fix dataproc label format
+- [AIRFLOW-1367] Pass Content-ID To reference inline images in an email, we need to be able to add <img src="cid:{}"/> to the HTML. However currently the Content-ID (cid) is not passed, so we need to add it
+- [AIRFLOW-1265] Fix celery executor parsing CELERY_SSL_ACTIVE
+- [AIRFLOW-1272] Google Cloud ML Batch Prediction Operator
+- [AIRFLOW-1352][AIRFLOW-1335] Revert MemoryHandler change ()[]
+- [AIRFLOW-1350] Add query_uri param to Hive/SparkSQL DataProc operator
+- [AIRFLOW-1334] Check if tasks are backfill on scheduler in a join
+- [AIRFLOW-1343] Add Airflow default label to the dataproc operator
+- [AIRFLOW-1273] Add Google Cloud ML version and model operators
+- [AIRFLOW-1273]AIRFLOW-1273] Add Google Cloud ML version and model operators
+- [AIRFLOW-1321] Fix hidden field key to ignore case
+- [AIRFLOW-1337] Make log_format key names lowercase
+- [AIRFLOW-1338][AIRFLOW-782] Add GCP dataflow hook runner change to UPDATING.md
+- [AIRFLOW-801] Remove outdated docstring on BaseOperator
+- [AIRFLOW-1344] Fix text encoding bug when reading logs for Python 3.5
+- [AIRFLOW-1338] Fix incompatible GCP dataflow hook
+- [AIRFLOW-1333] Enable copy function for Google Cloud Storage Hook
+- [AIRFLOW-1337] Allow log format customization via airflow.cfg
+- [AIRFLOW-1320] Update LetsBonus users in README
+- [AIRFLOW-1335] Use MemoryHandler for buffered logging
+- [AIRFLOW-1339] Add Drivy to the list of users
+- [AIRFLOW-1275] Put 'airflow pool' into API
+- [AIRFLOW-1296] Propagate SKIPPED to all downstream tasks
+- [AIRFLOW-1317] Fix minor issues in API reference
+- [AIRFLOW-1308] Disable nanny usage for Dask
+- [AIRFLOW-1172] Support nth weekday of the month cron expression
+- [AIRFLOW-936] Add clear/mark success for DAG in the UI
+- [AIRFLOW-1294] Backfills can loose tasks to execute
+- [AIRFLOW-1299] Support imageVersion in Google Dataproc cluster
+- [AIRFLOW-1291] Update NOTICE and LICENSE files to match ASF requirements
+- [AIRFLOW-1301] Add New Relic to list of companies
+- [AIRFLOW-1289] Removes restriction on number of scheduler threads
+- [AIRFLOW-1024] Ignore celery executor errors (#49)
+- [AIRFLOW-1265] Fix exception while loading celery configurations
+- [AIRFLOW-1290] set docs author to 'Apache Airflow'
+- [AIRFLOW-1242] Allowing project_id to have a colon in it.
+- [AIRFLOW-1282] Fix known event column sorting
+- [AIRFLOW-1166] Speed up _change_state_for_tis_without_dagrun
+- [AIRFLOW-1208] Speed-up cli tests
+- [AIRFLOW-1192] Some enhancements to qubole_operator
+- [AIRFLOW-1281] Sort variables by key field by default
+- [AIRFLOW-1277] Forbid KE creation with empty fields
+- [AIRFLOW-1276] Forbid event creation with end_data earlier than start_date
+- [AIRFLOW-1263] Dynamic height for charts
+- [AIRFLOW-1266] Increase width of gantt y axis
+- [AIRFLOW-1244] Forbid creation of a pool with empty name
+- [AIRFLOW-1274][HTTPSENSOR] Rename parameter params to data
+- [AIRFLOW-654] Add SSL Config Option for CeleryExecutor w/ RabbitMQ - Add BROKER_USE_SSL config to give option to send AMQP messages over SSL - Can be set using usual airflow options (e.g. airflow.cfg, env vars, etc.)
+- [AIRFLOW-1256] Add United Airlines to readme
+- [AIRFLOW-1251] Add eRevalue to Airflow users
+- [AIRFLOW-908] Print hostname at the start of cli run
+- [AIRFLOW-1237] Fix IN-predicate sqlalchemy warning
+- [AIRFLOW-1243] DAGs table has no default entries to show
+- [AIRFLOW-1245] Fix random failure in test_trigger_dag_for_date
+- [AIRFLOW-1248] Fix wrong conf name for worker timeout
+- [AIRFLOW-1197] : SparkSubmitHook on_kill error
+- [AIRFLOW-1191] : SparkSubmitHook custom cmd
+- [AIRFLOW-1234] Cover utils.operator_helpers with UTs
+- [AIRFLOW-1217] Enable Sqoop logging
+- [AIRFLOW-645] Support HTTPS connections in HttpHook
+- [AIRFLOW-1231] Use flask_wtf.CSRFProtect
+- [AIRFLOW-1232] Remove deprecated readfp warning
+- [AIRFLOW-1233] Cover utils.json with unit tests
+- [AIRFLOW-1227] Remove empty column on the Logs view
+- [AIRFLOW-1226] Remove empty column on the Jobs view
+- [AIRFLOW-1221] Fix templating bug with DatabricksSubmitRunOperator
+- [AIRFLOW-1210] Enable DbApiHook unit tests
+- [AIRFLOW-1199] Fix create modal
+- [AIRFLOW-1200] Forbid creation of a variable with an empty key
+- [AIRFLOW-1207] Enable utils.helpers unit tests
+- [AIRFLOW-1213] Add hcatalog parameters to sqoop
+- [AIRFLOW-1201] Update deprecated 'nose-parameterized'
+- [AIRFLOW-1186] Sort dag.get_task_instances by execution_date
+- [AIRFLOW-1203] Pin Google API client version to fix OAuth issue
+- [AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before.
+- [AIRFLOW-1180] Fix flask-wtf version for test_csrf_rejection
+- [AIRFLOW-993] Update date inference logic
+- [AIRFLOW-1170] DbApiHook insert_rows inserts parameters separately
+- [AIRFLOW-1041] Do not shadow xcom_push method[]
+- [AIRFLOW-860][AIRFLOW-935] Fix plugin executor import cycle and executor selection
+- [AIRFLOW-1189] Fix get a DataFrame using BigQueryHook failing
+- [AIRFLOW-1184] SparkSubmitHook does not split args
+- [AIRFLOW-1182] SparkSubmitOperator template field
+- [AIRFLOW-823] Allow specifying execution date in task_info API
+- [AIRFLOW-1175] Add Pronto Tools to Airflow user list
+- [AIRFLOW-1150] Fix scripts execution in sparksql hook[]
+- [AIRFLOW-1141] remove crawl_for_tasks
+- [AIRFLOW-1193] Add Checkr to company using Airflow
+- [AIRFLOW-1168] Add closing() to all connections and cursors
+- [AIRFLOW-1188] Add max_bad_records param to GoogleCloudStorageToBigQueryOperator
+- [AIRFLOW-1187][AIRFLOW-1185] Fix PyPi package names in documents
+- [AIRFLOW-1185] Fix PyPi URL in templates
+- [AIRFLOW-XXX] Updating CHANGELOG, README, and UPDATING after 1.8.1 release
+- [AIRFLOW-1181] Add delete and list functionality to gcs_hook
+- [AIRFLOW-1179] Fix Pandas 0.2x breaking Google BigQuery change
+- [AIRFLOW-1167] Support microseconds in FTPHook modification time
+- [AIRFLOW-1173] Add Robinhood to who uses Airflow
+- [AIRFLOW-945][AIRFLOW-941] Remove psycopg2 connection workaround
+- [AIRFLOW-1140] DatabricksSubmitRunOperator should template the "json" field.
+- [AIRFLOW-1160] Update Spark parameters for Mesos
+- [AIRFLOW 1149][AIRFLOW-1149] Allow for custom filters in Jinja2 templates
+- [AIRFLOW-1036] Randomize exponential backoff
+- [AIRFLOW-1155] Add Tails.com to community
+- [AIRFLOW-1142] Do not reset orphaned state for backfills
+- [AIRFLOW-492] Make sure stat updates cannot fail a task
+- [AIRFLOW-1119] Fix unload query so headers are on first row[]
+- [AIRFLOW-1089] Add Spark application arguments
+- [AIRFLOW-1125] Document encrypted connections
+- [AIRFLOW-1122] Increase stroke width in UI
+- [AIRFLOW-1138] Add missing licenses to files in scripts directory
+- (AIRFLOW-11-38) [AIRFLOW-1136] Capture invalid arguments for Sqoop
+- [AIRFLOW-1127] Move license notices to LICENSE
+- [AIRFLOW-1118] Add evo.company to Airflow users
+- [AIRFLOW-1121][AIRFLOW-1004] Fix `airflow webserver --pid` to write out pid file
+- [AIRFLOW-1124] Do not set all tasks to scheduled in backfill
+- [AIRFLOW-1120] Update version view to include Apache prefix
+- [AIRFLOW-1091] Add script that can compare jira target against merges
+- [AIRFLOW-1107] Add support for ftps non-default port
+- [AIRFLOW-1000] Rebrand distribution to Apache Airflow
+- [AIRFLOW-1094] Run unit tests under contrib in Travis
+- [AIRFLOW-1112] Log which pool when pool is full in scheduler
+- [AIRFLOW-1106] Add Groupalia/Letsbonus to the ReadMe
+- [AIRFLOW-1109] Use kill signal to kill processes and log results
+- [AIRFLOW-1074] Don't count queued tasks for concurrency limits
+- [AIRFLOW-1095] Make ldap_auth memberOf come from configuration
+- [AIRFLOW-1090] Add HBO
+- [AIRFLOW-1035] Use binary exponential backoff
+- [AIRFLOW-1081] Improve performance of duration chart
+- [AIRFLOW-1078] Fix latest_runs endpoint for old flask versions
+- [AIRFLOW-1085] Enhance the SparkSubmitOperator
+- [AIRFLOW-1050] Do not count up_for_retry as not ready
+- [AIRFLOW-1028] Databricks Operator for Airflow
+- [AIRFLOW-1075] Security docs cleanup
+- [AIRFLOW-1033][AIFRLOW-1033] Fix ti_deps for no schedule dags
+- [AIRFLOW-1016] Allow HTTP HEAD request method on HTTPSensor
+- [AIRFLOW-970] Load latest_runs on homepage async
+- [AIRFLOW-111] Include queued tasks in scheduler concurrency check
+- [AIRFLOW-1001] Fix landing times if there is no following schedule
+- [AIRFLOW-1065] Add functionality for Azure Blob Storage over wasb://
+- [AIRFLOW-947] Improve exceptions for unavailable Presto cluster
+- [AIRFLOW-1067] use example.com in examples
+- [AIRFLOW-1064] Change default sort to job_id for TaskInstanceModelView
+- [AIRFLOW-1030][AIRFLOW-1] Fix hook import for HttpSensor
+- [AIRFLOW-1051] Add a test for resetdb to CliTests
+- [AIRFLOW-1004][AIRFLOW-276] Fix `airflow webserver -D` to run in background
+- [AIRFLOW-1062] Fix DagRun#find to return correct result
+- [AIRFLOW-1011] Fix bug in BackfillJob._execute() for SubDAGs
+- [AIRFLOW-1038] Specify celery serialization options explicitly
+- [AIRFLOW-1054] Fix broken import in test_dag
+- [AIRFLOW-1007] Use Jinja sandbox for chart_data endpoint
+- [AIRFLOW-719] Fix race condition in ShortCircuit, Branch and LatestOnly
+- [AIRFLOW-1043] Fix doc strings of operators
+- [AIRFLOW-840] Make ticket renewer python3 compatible
+- [AIRFLOW-985] Extend the sqoop operator and hook
+- [AIRFLOW-1034] Make it possible to connect to S3 in sigv4 regions
+- [AIRFLOW-1045] Make log level configurable via airflow.cfg
+- [AIRFLOW-1047] Sanitize strings passed to Markup
+- [AIRFLOW-1040] Fix some small typos in comments and docstrings
+- [AIRFLOW-1017] get_task_instance shouldn't throw exception when no TI
+- [AIRFLOW-1006] Add config_templates to MANIFEST
+- [AIRFLOW-999] Add support for Redis database
+- [AIRFLOW-1009] Remove SQLOperator from Concepts page
+- [AIRFLOW-1006] Move config templates to separate files
+- [AIRFLOW-1005] Improve Airflow startup time
+- [AIRFLOW-1010] Add convenience script for signing releases
+- [AIRFLOW-995] Remove reference to actual Airflow issue
+- [AIRFLOW-681] homepage doc link should pointing to apache repo not airbnb repo
+- [AIRFLOW-705][AIRFLOW-706] Fix run_command bugs
+- [AIRFLOW-990] Fix Py27 unicode logging in DockerOperator
+- [AIRFLOW-963] Fix non-rendered code examples
+- [AIRFLOW-969] Catch bad python_callable argument
+- [AIRFLOW-984] Enable subclassing of SubDagOperator
+- [AIRFLOW-997] Update setup.cfg to point to Apache
+- [AIRFLOW-994] Add MiNODES to the official airflow user list
+- [AIRFLOW-995][AIRFLOW-1] Update GitHub PR Template
+- [AIRFLOW-989] Do not mark dag run successful if unfinished tasks
+- [AIRFLOW-903] New configuration setting for the default dag view
+- [AIRFLOW-979] Add GovTech GDS
+- [AIRFLOW-933] Replace eval with literal_eval to prevent RCE
+- [AIRFLOW-974] Fix mkdirs race condition
+- [AIRFLOW-917] Fix formatting of error message
+- [AIRFLOW-770] Refactor BaseHook so env vars are always read
+- [AIRFLOW-900] Double trigger should not kill original task instance
+- [AIRFLOW-900] Fixes bugs in LocalTaskJob for double run protection
+- [AIRFLOW-932][AIRFLOW-932][AIRFLOW-921][AIRFLOW-910] Do not mark tasks removed when backfilling[
+- [AIRFLOW-961] run onkill when SIGTERMed
+- [AIRFLOW-910] Use parallel task execution for backfills
+- [AIRFLOW-967] Wrap strings in native for py2 ldap compatibility
+- [AIRFLOW-958] Improve tooltip readability
+- AIRFLOW-959 Cleanup and reorganize .gitignore
+- AIRFLOW-960 Add .editorconfig file
+- [AIRFLOW-931] Do not set QUEUED in TaskInstances
+- [AIRFLOW-956] Get docs working on readthedocs.org
+- [AIRFLOW-954] Fix configparser ImportError
+- [AIRFLOW-941] Use defined parameters for psycopg2
+- [AIRFLOW-943] Update Digital First Media in users list
+- [AIRFLOW-942] Add mytaxi to Airflow users
+- [AIRFLOW-939] add .swp to gitginore
+- [AIRFLOW-719] Prevent DAGs from ending prematurely
+- [AIRFLOW-938] Use test for True in task_stats queries
+- [AIRFLOW-937] Improve performance of task_stats
+- [AIRFLOW-933] use ast.literal_eval rather eval because ast.literal_eval does not execute input.
+- [AIRFLOW-925] Revert airflow.hooks change that cherry-pick picked
+- [AIRFLOW-919] Running tasks with no start date shouldn't break a DAGs UI
+- [AIRFLOW-802][AIRFLOW-1] Add spark-submit operator/hook
+- [AIRFLOW-725] Use keyring to store credentials for JIRA
+- [AIRFLOW-916] Remove deprecated readfp function
+- [AIRFLOW-911] Add coloring and timing to tests
+- [AIRFLOW-906] Update Code icon from lightning bolt to file
+- [AIRFLOW-897] Prevent dagruns from failing with unfinished tasks
+- [AIRFLOW-896] Remove unicode to 8-bit conversion in BigQueryOperator
+- [AIRFLOW-899] Tasks in SCHEDULED state should be white in the UI instead of black
+- [AIRFLOW-895] Address Apache release incompliancies
+- [AIRFLOW-893][AIRFLOW-510] Fix crashing webservers when a dagrun has no start date
+- [AIRFLOW-880] Make webserver serve logs in a sane way for remote logs
+- [AIRFLOW-889] Fix minor error in the docstrings for BaseOperator
+- [AIRFLOW-809][AIRFLOW-1] Use __eq__ ColumnOperator When Testing Booleans
+- [AIRFLOW-875] Add template to HttpSensor params
+- [AIRFLOW-866] Add FTPSensor
+- [AIRFLOW-881] Check if SubDagOperator is in DAG context manager
+- [AIRFLOW-885] Add change.org to the users list
+- [AIRFLOW-836] Use POST and CSRF for state changing endpoints
+- [AIRFLOW-862] Fix Unit Tests for DaskExecutor
+- [AIRFLOW-887] Support future v0.16
+- [AIRFLOW-886] Pass result to post_execute() hook
+- [AIRFLOW-871] change logging.warn() into warning()
+- [AIRFLOW-882] Remove unnecessary dag>>op assignment in docs
+- [AIRFLOW-861] make pickle_info endpoint be login_required
+- [AIRFLOW-869] Refactor mark success functionality
+- [AIRFLOW-877] Remove .sql template extension from GCS download operator
+- [AIRFLOW-826] Add Zendesk hook
+- [AIRFLOW-842] do not query the DB with an empty IN clause
+- [AIRFLOW-834] change raise StopIteration into return
+- [AIRFLOW-832] Let debug server run without SSL
+- [AIRFLOW-862] Add DaskExecutor
+- [AIRFLOW-858] Configurable database name for DB operators
+- [AIRFLOW-863] Example DAGs should have recent start dates
+- [AIRFLOW-853] use utf8 encoding for stdout line decode
+- [AIRFLOW-857] Use library assert statements instead of conditionals
+- [AIRFLOW-856] Make sure execution date is set for local client
+- [AIRFLOW-854] Add OKI as Airflow user
+- [AIRFLOW-830][AIRFLOW-829][AIRFLOW-88] Reduce Travis log verbosity
+- [AIRFLOW-814] Fix Presto*CheckOperator.__init__
+- [AIRFLOW-793] Enable compressed loading in S3ToHiveTransfer
+- [AIRFLOW-844] Fix cgroups directory creation
+- [AIRFLOW-831] Restore import to fix broken tests
+- [AIRFLOW-794] Access DAGS_FOLDER and SQL_ALCHEMY_CONN exclusively from settings
+- [AIRFLOW-694] Fix config behaviour for empty envvar
+- [AIRFLOW-365] Set dag.fileloc explicitly and use for Code view
+- [AIRFLOW-781] Allow DataFlowOperators to accept jobs stored in GCS
 
-AIRFLOW 1.8.2, 2017-09-04
+Airflow 1.8.2, 2017-09-04
 -------------------------
 
-9a53e66 [AIRFLOW-809][AIRFLOW-1] Use __eq__ ColumnOperator When Testing Booleans
-333e0b3 [AIRFLOW-1296] Propagate SKIPPED to all downstream tasks
-93825d5 Re-enable caching for hadoop components
-33a9dcb Pin Hive and Hadoop to a specific version and create writable warehouse dir
-7cff6cd [AIRFLOW-1308] Disable nanny usage for Dask
-c6a09c4 Updating CHANGELOG for 1.8.2rc1
-570b2ed [AIRFLOW-1294] Backfills can loose tasks to execute
-3f48d48 [AIRFLOW-1291] Update NOTICE and LICENSE files to match ASF requirements
-e10af9a [AIRFLOW-XXX] Set version to 1.8.2rc1
-69bd269 [AIRFLOW-1160] Update Spark parameters for Mesos
-9692510 [AIRFLOW 1149][AIRFLOW-1149] Allow for custom filters in Jinja2 templates
-6de5330 [AIRFLOW-1119] Fix unload query so headers are on first row[]
-b4e9eb8 [AIRFLOW-1089] Add Spark application arguments
-a4083f3 [AIRFLOW-1078] Fix latest_runs endpoint for old flask versions
-7a02841 [AIRFLOW-1074] Don't count queued tasks for concurrency limits
-a2c18a5 [AIRFLOW-1064] Change default sort to job_id for TaskInstanceModelView
-d1c64ab [AIRFLOW-1038] Specify celery serialization options explicitly
-b4ee88a [AIRFLOW-1036] Randomize exponential backoff
-9fca409 [AIRFLOW-993] Update date inference logic
-272c2f5 [AIRFLOW-1167] Support microseconds in FTPHook modification time
-c7c0b72 [AIRFLOW-1179] Fix Pandas 0.2x breaking Google BigQuery change
-acd0166 [AIRFLOW-1263] Dynamic height for charts
-7f33f6e [AIRFLOW-1266] Increase width of gantt y axis
-fc33c04 [AIRFLOW-1290] set docs author to 'Apache Airflow'
-2e9eee3 [AIRFLOW-1282] Fix known event column sorting
-2389a8a [AIRFLOW-1166] Speed up _change_state_for_tis_without_dagrun
-bf966e6 [AIRFLOW-1192] Some enhancements to qubole_operator
-57d5bcd [AIRFLOW-1281] Sort variables by key field by default
-802fc15 [AIRFLOW-1244] Forbid creation of a pool with empty name
-1232b6a [AIRFLOW-1243] DAGs table has no default entries to show
-b0ba3c9 [AIRFLOW-1227] Remove empty column on the Logs view
-c406652 [AIRFLOW-1226] Remove empty column on the Jobs view
-51a83cc [AIRFLOW-1199] Fix create modal
-cac7d4c [AIRFLOW-1200] Forbid creation of a variable with an empty key
-5f3ee52 [AIRFLOW-1186] Sort dag.get_task_instances by execution_date
-f446c08 [AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before.
-93b8e96 [AIRFLOW-1180] Fix flask-wtf version for test_csrf_rejection
-bb56805 [AIRFLOW-1170] DbApiHook insert_rows inserts parameters separately
-093b2f0 [AIRFLOW-1150] Fix scripts execution in sparksql hook[]
-777f181 [AIRFLOW-1168] Add closing() to all connections and cursors
-bc8e912 [AIRFLOW-XXX] Updating CHANGELOG, README, and UPDATING after 1.8.1 release
+- [AIRFLOW-809][AIRFLOW-1] Use __eq__ ColumnOperator When Testing Booleans
+- [AIRFLOW-1296] Propagate SKIPPED to all downstream tasks
+- Re-enable caching for hadoop components
+- Pin Hive and Hadoop to a specific version and create writable warehouse dir
+- [AIRFLOW-1308] Disable nanny usage for Dask
+- Updating CHANGELOG for 1.8.2rc1
+- [AIRFLOW-1294] Backfills can loose tasks to execute
+- [AIRFLOW-1291] Update NOTICE and LICENSE files to match ASF requirements
+- [AIRFLOW-XXX] Set version to 1.8.2rc1
+- [AIRFLOW-1160] Update Spark parameters for Mesos
+- [AIRFLOW 1149][AIRFLOW-1149] Allow for custom filters in Jinja2 templates
+- [AIRFLOW-1119] Fix unload query so headers are on first row[]
+- [AIRFLOW-1089] Add Spark application arguments
+- [AIRFLOW-1078] Fix latest_runs endpoint for old flask versions
+- [AIRFLOW-1074] Don't count queued tasks for concurrency limits
+- [AIRFLOW-1064] Change default sort to job_id for TaskInstanceModelView
+- [AIRFLOW-1038] Specify celery serialization options explicitly
+- [AIRFLOW-1036] Randomize exponential backoff
+- [AIRFLOW-993] Update date inference logic
+- [AIRFLOW-1167] Support microseconds in FTPHook modification time
+- [AIRFLOW-1179] Fix Pandas 0.2x breaking Google BigQuery change
+- [AIRFLOW-1263] Dynamic height for charts
+- [AIRFLOW-1266] Increase width of gantt y axis
+- [AIRFLOW-1290] set docs author to 'Apache Airflow'
+- [AIRFLOW-1282] Fix known event column sorting
+- [AIRFLOW-1166] Speed up _change_state_for_tis_without_dagrun
+- [AIRFLOW-1192] Some enhancements to qubole_operator
+- [AIRFLOW-1281] Sort variables by key field by default
+- [AIRFLOW-1244] Forbid creation of a pool with empty name
+- [AIRFLOW-1243] DAGs table has no default entries to show
+- [AIRFLOW-1227] Remove empty column on the Logs view
+- [AIRFLOW-1226] Remove empty column on the Jobs view
+- [AIRFLOW-1199] Fix create modal
+- [AIRFLOW-1200] Forbid creation of a variable with an empty key
+- [AIRFLOW-1186] Sort dag.get_task_instances by execution_date
+- [AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before.
+- [AIRFLOW-1180] Fix flask-wtf version for test_csrf_rejection
+- [AIRFLOW-1170] DbApiHook insert_rows inserts parameters separately
+- [AIRFLOW-1150] Fix scripts execution in sparksql hook[]
+- [AIRFLOW-1168] Add closing() to all connections and cursors
+- [AIRFLOW-XXX] Updating CHANGELOG, README, and UPDATING after 1.8.1 release
 
-AIRFLOW 1.8.1, 2017-05-09
+Airflow 1.8.1, 2017-05-09
 -------------------------
 
-[AIRFLOW-1142] SubDAG Tasks Not Executed Even Though All Dependencies Met
-[AIRFLOW-1138] Add licenses to files in scripts directory
-[AIRFLOW-1127] Move license notices to LICENSE instead of NOTICE
-[AIRFLOW-1124] Do not set all task instances to scheduled on backfill
-[AIRFLOW-1120] Update version view to include Apache prefix
-[AIRFLOW-1062] DagRun#find returns wrong result if external_trigger=False is specified
-[AIRFLOW-1054] Fix broken import on test_dag
-[AIRFLOW-1050] Retries ignored - regression
-[AIRFLOW-1033] TypeError: can't compare datetime.datetime to None
-[AIRFLOW-1017] get_task_instance should return None instead of throw an exception for non-existent TIs
-[AIRFLOW-1011] Fix bug in BackfillJob._execute() for SubDAGs
-[AIRFLOW-1004] `airflow webserver -D` runs in foreground
-[AIRFLOW-1001] Landing Time shows "unsupported operand type(s) for -: 'datetime.datetime' and 'NoneType'" on example_subdag_operator
-[AIRFLOW-1000] Rebrand to Apache Airflow instead of Airflow
-[AIRFLOW-989] Clear Task Regression
-[AIRFLOW-974] airflow.util.file mkdir has a race condition
-[AIRFLOW-906] Update Code icon from lightning bolt to file
-[AIRFLOW-858] Configurable database name for DB operators
-[AIRFLOW-853] ssh_execute_operator.py stdout decode default to ASCII
-[AIRFLOW-832] Fix debug server
-[AIRFLOW-817] Trigger dag fails when using CLI + API
-[AIRFLOW-816] Make sure to pull nvd3 from local resources
-[AIRFLOW-815] Add previous/next execution dates to available default variables.
-[AIRFLOW-813] Fix unterminated unit tests in tests.job (tests/job.py)
-[AIRFLOW-812] Scheduler job terminates when there is no dag file
-[AIRFLOW-806] UI should properly ignore DAG doc when it is None
-[AIRFLOW-794] Consistent access to DAGS_FOLDER and SQL_ALCHEMY_CONN
-[AIRFLOW-785] ImportError if cgroupspy is not installed
-[AIRFLOW-784] Cannot install with funcsigs > 1.0.0
-[AIRFLOW-780] The UI no longer shows broken DAGs
-[AIRFLOW-777] dag_is_running is initlialized to True instead of False
-[AIRFLOW-719] Skipped operations make DAG finish prematurely
-[AIRFLOW-694] Empty env vars do not overwrite non-empty config values
-[AIRFLOW-492] Insert into dag_stats table results into failed task while task itself succeeded
-[AIRFLOW-139] Executing VACUUM with PostgresOperator
-[AIRFLOW-111] DAG concurrency is not honored
-[AIRFLOW-88] Improve clarity Travis CI reports
+- [AIRFLOW-1142] SubDAG Tasks Not Executed Even Though All Dependencies Met
+- [AIRFLOW-1138] Add licenses to files in scripts directory
+- [AIRFLOW-1127] Move license notices to LICENSE instead of NOTICE
+- [AIRFLOW-1124] Do not set all task instances to scheduled on backfill
+- [AIRFLOW-1120] Update version view to include Apache prefix
+- [AIRFLOW-1062] DagRun#find returns wrong result if external_trigger=False is specified
+- [AIRFLOW-1054] Fix broken import on test_dag
+- [AIRFLOW-1050] Retries ignored - regression
+- [AIRFLOW-1033] TypeError: can't compare datetime.datetime to None
+- [AIRFLOW-1017] get_task_instance should return None instead of throw an exception for non-existent TIs
+- [AIRFLOW-1011] Fix bug in BackfillJob._execute() for SubDAGs
+- [AIRFLOW-1004] `airflow webserver -D` runs in foreground
+- [AIRFLOW-1001] Landing Time shows "unsupported operand type(s) for -: 'datetime.datetime' and 'NoneType'" on example_subdag_operator
+- [AIRFLOW-1000] Rebrand to Apache Airflow instead of Airflow
+- [AIRFLOW-989] Clear Task Regression
+- [AIRFLOW-974] airflow.util.file mkdir has a race condition
+- [AIRFLOW-906] Update Code icon from lightning bolt to file
+- [AIRFLOW-858] Configurable database name for DB operators
+- [AIRFLOW-853] ssh_execute_operator.py stdout decode default to ASCII
+- [AIRFLOW-832] Fix debug server
+- [AIRFLOW-817] Trigger dag fails when using CLI + API
+- [AIRFLOW-816] Make sure to pull nvd3 from local resources
+- [AIRFLOW-815] Add previous/next execution dates to available default variables.
+- [AIRFLOW-813] Fix unterminated unit tests in tests.job (tests/job.py)
+- [AIRFLOW-812] Scheduler job terminates when there is no dag file
+- [AIRFLOW-806] UI should properly ignore DAG doc when it is None
+- [AIRFLOW-794] Consistent access to DAGS_FOLDER and SQL_ALCHEMY_CONN
+- [AIRFLOW-785] ImportError if cgroupspy is not installed
+- [AIRFLOW-784] Cannot install with funcsigs > 1.0.0
+- [AIRFLOW-780] The UI no longer shows broken DAGs
+- [AIRFLOW-777] dag_is_running is initlialized to True instead of False
+- [AIRFLOW-719] Skipped operations make DAG finish prematurely
+- [AIRFLOW-694] Empty env vars do not overwrite non-empty config values
+- [AIRFLOW-492] Insert into dag_stats table results into failed task while task itself succeeded
+- [AIRFLOW-139] Executing VACUUM with PostgresOperator
+- [AIRFLOW-111] DAG concurrency is not honored
+- [AIRFLOW-88] Improve clarity Travis CI reports
 
-AIRFLOW 1.8.0, 2017-03-12
+Airflow 1.8.0, 2017-03-12
 -------------------------
 
-[AIRFLOW-900] Double trigger should not kill original task instance
-[AIRFLOW-900] Fixes bugs in LocalTaskJob for double run protection
-[AIRFLOW-932] Do not mark tasks removed when backfilling
-[AIRFLOW-961] run onkill when SIGTERMed
-[AIRFLOW-910] Use parallel task execution for backfills
-[AIRFLOW-967] Wrap strings in native for py2 ldap compatibility
-[AIRFLOW-941] Use defined parameters for psycopg2
-[AIRFLOW-719] Prevent DAGs from ending prematurely
-[AIRFLOW-938] Use test for True in task_stats queries
-[AIRFLOW-937] Improve performance of task_stats
-[AIRFLOW-933] use ast.literal_eval rather eval because ast.literal_eval does not execute input.
-[AIRFLOW-925] Revert airflow.hooks change that cherry-pick picked
-[AIRFLOW-919] Running tasks with no start date shouldn't break a DAGs UI
-[AIRFLOW-802] Add spark-submit operator/hook
-[AIRFLOW-897] Prevent dagruns from failing with unfinished tasks
-[AIRFLOW-861] make pickle_info endpoint be login_required
-[AIRFLOW-853] use utf8 encoding for stdout line decode
-[AIRFLOW-856] Make sure execution date is set for local client
-[AIRFLOW-830][AIRFLOW-829][AIRFLOW-88] Reduce Travis log verbosity
-[AIRFLOW-831] Restore import to fix broken tests
-[AIRFLOW-794] Access DAGS_FOLDER and SQL_ALCHEMY_CONN exclusively from settings
-[AIRFLOW-694] Fix config behaviour for empty envvar
-[AIRFLOW-365] Set dag.fileloc explicitly and use for Code view
-[AIRFLOW-931] Do not set QUEUED in TaskInstances
-[AIRFLOW-899] Tasks in SCHEDULED state should be white in the UI instead of black
-[AIRFLOW-895] Address Apache release incompliancies
-[AIRFLOW-893][AIRFLOW-510] Fix crashing webservers when a dagrun has no start date
-[AIRFLOW-793] Enable compressed loading in S3ToHiveTransfer
-[AIRFLOW-863] Example DAGs should have recent start dates
-[AIRFLOW-869] Refactor mark success functionality
-[AIRFLOW-856] Make sure execution date is set for local client
-[AIRFLOW-814] Fix Presto*CheckOperator.__init__
-[AIRFLOW-844] Fix cgroups directory creation
-[AIRFLOW-816] Use static nvd3 and d3
-[AIRFLOW-821] Fix py3 compatibility
-[AIRFLOW-817] Check for None value of execution_date in endpoint
-[AIRFLOW-822] Close db before exception
-[AIRFLOW-815] Add prev/next execution dates to template variables
-[AIRFLOW-813] Fix unterminated unit tests in SchedulerJobTest
-[AIRFLOW-813] Fix unterminated scheduler unit tests
-[AIRFLOW-806] UI should properly ignore DAG doc when it is None
-[AIRFLOW-812] Fix the scheduler termination bug.
-[AIRFLOW-780] Fix dag import errors no longer working
-[AIRFLOW-783] Fix py3 incompatibility in BaseTaskRunner
-[AIRFLOW-810] Correct down_revision dag_id/state index creation
-[AIRFLOW-807] Improve scheduler performance for large DAGs
-[AIRFLOW-798] Check return_code before forcing termination
-[AIRFLOW-139] Let psycopg2 handle autocommit for PostgresHook
-[AIRFLOW-776] Add missing cgroups devel dependency
-[AIRFLOW-777] Fix expression to check if a DagRun is in running state
-[AIRFLOW-785] Don't import CgroupTaskRunner at global scope
-[AIRFLOW-784] Pin funcsigs to 1.0.0
-[AIRFLOW-624] Fix setup.py to not import airflow.version as version
-[AIRFLOW-779] Task should fail with specific message when deleted
-[AIRFLOW-778] Fix completey broken MetastorePartitionSensor
-[AIRFLOW-739] Set pickle_info log to debug
-[AIRFLOW-771] Make S3 logs append instead of clobber
-[AIRFLOW-773] Fix flaky datetime addition in api test
-[AIRFLOW-219][AIRFLOW-398] Cgroups + impersonation
-[AIRFLOW-683] Add jira hook, operator and sensor
-[AIRFLOW-762] Add Google DataProc delete operator
-[AIRFLOW-760] Update systemd config
-[AIRFLOW-759] Use previous dag_run to verify depend_on_past
-[AIRFLOW-757] Set child_process_log_directory default more sensible
-[AIRFLOW-692] Open XCom page to super-admins only
-[AIRFLOW-737] Fix HDFS Sensor directory.
-[AIRFLOW-747] Fix retry_delay not honoured
-[AIRFLOW-558] Add Support for dag.catchup=(True|False) Option
-[AIRFLOW-489] Allow specifying execution date in trigger_dag API
-[AIRFLOW-738] Commit deleted xcom items before insert
-[AIRFLOW-729] Add Google Cloud Dataproc cluster creation operator
-[AIRFLOW-728] Add Google BigQuery table sensor
-[AIRFLOW-741] Log to debug instead of info for app.py
-[AIRFLOW-731] Fix period bug for NamedHivePartitionSensor
-[AIRFLOW-740] Pin jinja2 to < 2.9.0
-[AIRFLOW-663] Improve time units for task performance charts
-[AIRFLOW-665] Fix email attachments
-[AIRFLOW-734] Fix SMTP auth regression when not using user/pass
-[AIRFLOW-702] Fix LDAP Regex Bug
-[AIRFLOW-717] Add Cloud Storage updated sensor
-[AIRFLOW-695] Retries do not execute because dagrun is in FAILED state
-[AIRFLOW-673] Add operational metrics test for SchedulerJob
-[AIRFLOW-727] try_number is not increased
-[AIRFLOW-715] A more efficient HDFS Sensor:
-[AIRFLOW-716] Allow AVRO BigQuery load-job without schema
-[AIRFLOW-718] Allow the query URI for DataProc Pig
-Log needs to be part of try/catch block
-[AIRFLOW-721] Descendant process can disappear before termination
-[AIRFLOW-403] Bash operator's kill method leaves underlying processes running
-[AIRFLOW-657] Add AutoCommit Parameter for MSSQL
-[AIRFLOW-641] Improve pull request instructions
-[AIRFLOW-685] Add test for MySqlHook.bulk_load()
-[AIRFLOW-686] Match auth backend config section
-[AIRFLOW-691] Add SSH KeepAlive option to SSH_hook
-[AIRFLOW-709] Use same engine for migrations and reflection
-[AIRFLOW-700] Update to reference to web authentication documentation
-[AIRFLOW-649] Support non-sched DAGs in LatestOnlyOp
-[AIRFLOW-712] Fix AIRFLOW-667 to use proper HTTP error properties
-[AIRFLOW-710] Add OneFineStay as official user
-[AIRFLOW-703][AIRFLOW-1] Stop Xcom being cleared too early
-[AIRFLOW-679] Stop concurrent task instances from running
-[AIRFLOW-704][AIRFLOW-1] Fix invalid syntax in BQ hook
-[AIRFLOW-667] Handle BigQuery 503 error
-[AIRFLOW-680] Disable connection pool for commands
-[AIRFLOW-678] Prevent scheduler from double triggering TIs
-[AIRFLOW-677] Kill task if it fails to heartbeat
-[AIRFLOW-674] Ability to add descriptions for DAGs
-[AIRFLOW-682] Bump MAX_PERIODS to make mark_success work for large DAGs
-Use jdk selector to set required jdk
-[AIRFLOW-647] Restore dag.get_active_runs
-[AIRFLOW-662] Change seasons to months in project description
-[AIRFLOW-656] Add dag/task/date index to xcom table
-[AIRFLOW-658] Improve schema_update_options in GCP
-[AIRFLOW-41] Fix pool oversubscription
-[AIRFLOW-489] Add API Framework
-[AIRFLOW-653] Add some missing endpoint tests
-[AIRFLOW-652] Remove obsolete endpoint
-[AIRFLOW-345] Add contrib ECSOperator
-[AIRFLOW-650] Adding Celect to user list
-[AIRFLOW-510] Filter Paused Dags, show Last Run & Trigger Dag
-[AIRFLOW-643] Improve date handling for sf_hook
-[AIRFLOW-638] Add schema_update_options to GCP ops
-[AIRFLOW-640] Install and enable nose-ignore-docstring
-[AIRFLOW-639]AIRFLOW-639] Alphasort package names
-[AIRFLOW-375] Fix pylint errors
-[AIRFLOW-347] Show empty DAG runs in tree view
-[AIRFLOW-628] Adding SalesforceHook to contrib/hooks
-[AIRFLOW-514] hive hook loads data from pandas DataFrame into hive and infers types
-[AIRFLOW-565] Fixes DockerOperator on Python3.x
-[AIRFLOW-635] Encryption option for S3 hook
-[AIRFLOW-137] Fix max_active_runs on clearing tasks
-[AIRFLOW-343] Fix schema plumbing in HiveServer2Hook
-[AIRFLOW-130] Fix ssh operator macosx
-[AIRFLOW-633] Show TI attributes in TI view
-[AIRFLOW-626][AIRFLOW-1] HTML Content does not show up when sending email with attachment
-[AIRFLOW-533] Set autocommit via set_autocommit
-[AIRFLOW-629] stop pinning lxml
-[AIRFLOW-464] Add setdefault method to Variable
-[AIRFLOW-626][AIRFLOW-1] HTML Content does not show up when sending email with attachment
-[AIRFLOW-591] Add datadog hook & sensor
-[AIRFLOW-561] Add RedshiftToS3Transfer operator
-[AIRFLOW-570] Pass root to date form on gantt
-[AIRFLOW-504] Store fractional seconds in MySQL tables
-[AIRFLOW-623] LDAP attributes not always a list
-[AIRFLOW-611] source_format in BigQueryBaseCursor
-[AIRFLOW-619] Fix exception in Gannt chart
-[AIRFLOW-618] Cast DateTimes to avoid sqllite errors
-[AIRFLOW-422] Add JSON endpoint for task info
-[AIRFLOW-616][AIRFLOW-617] Minor fixes to PR tool UX
-[AIRFLOW-179] Fix DbApiHook with non-ASCII chars
-[AIRFLOW-566] Add timeout while fetching logs
-[AIRFLOW-615] Set graph glyphicon first
-[AIRFLOW-609] Add application_name to PostgresHook
-[AIRFLOW-604] Revert .first() to .one()
-[AIRFLOW-370] Create AirflowConfigException in exceptions.py
-[AIRFLOW-582] Fixes TI.get_dagrun filter (removes start_date)
-[AIRFLOW-568] Fix double task_stats count if a DagRun is active
-[AIRFLOW-585] Fix race condition in backfill execution loop
-[AIRFLOW-580] Prevent landscape warning on .format
-[AIRFLOW-597] Check if content is None, not false-equivalent
-[AIRFLOW-586] test_dag_v1 fails from 0 to 3 a.m.
-[AIRFLOW-453] Add XCom Admin Page
-[AIRFLOW-588] Add Google Cloud Storage Object sensor[]
-[AIRFLOW-592] example_xcom import Error
-[AIRFLOW-587] Fix incorrect scope for Google Auth[]
-[AIRFLOW-589] Add templatable job_name[]
-[AIRFLOW-227] Show running config in config view
-[AIRFLOW-319]AIRFLOW-319] xcom push response in HTTP Operator
-[AIRFLOW-385] Add symlink to latest scheduler log directory
-[AIRFLOW-583] Fix decode error in gcs_to_bq
-[AIRFLOW-96] s3_conn_id using environment variable
-[AIRFLOW-575] Clarify tutorial and FAQ about `schedule_interval` always inheriting from DAG object
-[AIRFLOW-577] Output BigQuery job for improved debugging
-[AIRFLOW-560] Get URI & SQLA engine from Connection
-[AIRFLOW-518] Require DataProfilingMixin for Variables CRUD
-[AIRFLOW-553] Fix load path for filters.js
-[AIRFLOW-554] Add Jinja support to Spark-sql
-[AIRFLOW-550] Make ssl config check empty string safe
-[AIRFLOW-500] Use id for github allowed teams
-[AIRFLOW-556] Add UI PR guidelines
-[AIRFLOW-358][AIRFLOW-430] Add `connections` cli
-[AIRFLOW-548] Load DAGs immediately & continually
-[AIRFLOW-539] Updated BQ hook and BQ operator to support Standard SQL.
-[AIRFLOW-378] Add string casting to params of spark-sql operator
-[AIRFLOW-544] Add Pause/Resume toggle button
-[AIRFLOW-333][AIRFLOW-258] Fix non-module plugin components
-[AIRFLOW-542] Add tooltip to DAGs links icons
-[AIRFLOW-530] Update docs to reflect connection environment var has to be in uppercase
-[AIRFLOW-525] Update template_fields in Qubole Op
-[AIRFLOW-480] Support binary file download from GCS
-[AIRFLOW-198] Implement latest_only_operator
-[AIRFLOW-91] Add SSL config option for the webserver
-[AIRFLOW-191] Fix connection leak with PostgreSQL backend
-[AIRFLOW-512] Fix 'bellow' typo in docs & comments
-[AIRFLOW-509][AIRFLOW-1] Create operator to delete tables in BigQuery
-[AIRFLOW-498] Remove hard-coded gcp project id
-[AIRFLOW-505] Support unicode characters in authors' names
-[AIRFLOW-494] Add per-operator success/failure metrics
-[AIRFLOW-488] Fix test_simple fail
-[AIRFLOW-468] Update Panda requirement to 0.17.1
-[AIRFLOW-159] Add cloud integration section + GCP documentation
-[AIRFLOW-477][AIRFLOW-478] Restructure security section for clarity
-[AIRFLOW-467] Allow defining of project_id in BigQueryHook
-[AIRFLOW-483] Change print to logging statement
-[AIRFLOW-475] make the segment granularity in Druid hook configurable
+- [AIRFLOW-900] Double trigger should not kill original task instance
+- [AIRFLOW-900] Fixes bugs in LocalTaskJob for double run protection
+- [AIRFLOW-932] Do not mark tasks removed when backfilling
+- [AIRFLOW-961] run onkill when SIGTERMed
+- [AIRFLOW-910] Use parallel task execution for backfills
+- [AIRFLOW-967] Wrap strings in native for py2 ldap compatibility
+- [AIRFLOW-941] Use defined parameters for psycopg2
+- [AIRFLOW-719] Prevent DAGs from ending prematurely
+- [AIRFLOW-938] Use test for True in task_stats queries
+- [AIRFLOW-937] Improve performance of task_stats
+- [AIRFLOW-933] use ast.literal_eval rather eval because ast.literal_eval does not execute input.
+- [AIRFLOW-925] Revert airflow.hooks change that cherry-pick picked
+- [AIRFLOW-919] Running tasks with no start date shouldn't break a DAGs UI
+- [AIRFLOW-802] Add spark-submit operator/hook
+- [AIRFLOW-897] Prevent dagruns from failing with unfinished tasks
+- [AIRFLOW-861] make pickle_info endpoint be login_required
+- [AIRFLOW-853] use utf8 encoding for stdout line decode
+- [AIRFLOW-856] Make sure execution date is set for local client
+- [AIRFLOW-830][AIRFLOW-829][AIRFLOW-88] Reduce Travis log verbosity
+- [AIRFLOW-831] Restore import to fix broken tests
+- [AIRFLOW-794] Access DAGS_FOLDER and SQL_ALCHEMY_CONN exclusively from settings
+- [AIRFLOW-694] Fix config behaviour for empty envvar
+- [AIRFLOW-365] Set dag.fileloc explicitly and use for Code view
+- [AIRFLOW-931] Do not set QUEUED in TaskInstances
+- [AIRFLOW-899] Tasks in SCHEDULED state should be white in the UI instead of black
+- [AIRFLOW-895] Address Apache release incompliancies
+- [AIRFLOW-893][AIRFLOW-510] Fix crashing webservers when a dagrun has no start date
+- [AIRFLOW-793] Enable compressed loading in S3ToHiveTransfer
+- [AIRFLOW-863] Example DAGs should have recent start dates
+- [AIRFLOW-869] Refactor mark success functionality
+- [AIRFLOW-856] Make sure execution date is set for local client
+- [AIRFLOW-814] Fix Presto*CheckOperator.__init__
+- [AIRFLOW-844] Fix cgroups directory creation
+- [AIRFLOW-816] Use static nvd3 and d3
+- [AIRFLOW-821] Fix py3 compatibility
+- [AIRFLOW-817] Check for None value of execution_date in endpoint
+- [AIRFLOW-822] Close db before exception
+- [AIRFLOW-815] Add prev/next execution dates to template variables
+- [AIRFLOW-813] Fix unterminated unit tests in SchedulerJobTest
+- [AIRFLOW-813] Fix unterminated scheduler unit tests
+- [AIRFLOW-806] UI should properly ignore DAG doc when it is None
+- [AIRFLOW-812] Fix the scheduler termination bug.
+- [AIRFLOW-780] Fix dag import errors no longer working
+- [AIRFLOW-783] Fix py3 incompatibility in BaseTaskRunner
+- [AIRFLOW-810] Correct down_revision dag_id/state index creation
+- [AIRFLOW-807] Improve scheduler performance for large DAGs
+- [AIRFLOW-798] Check return_code before forcing termination
+- [AIRFLOW-139] Let psycopg2 handle autocommit for PostgresHook
+- [AIRFLOW-776] Add missing cgroups devel dependency
+- [AIRFLOW-777] Fix expression to check if a DagRun is in running state
+- [AIRFLOW-785] Don't import CgroupTaskRunner at global scope
+- [AIRFLOW-784] Pin funcsigs to 1.0.0
+- [AIRFLOW-624] Fix setup.py to not import airflow.version as version
+- [AIRFLOW-779] Task should fail with specific message when deleted
+- [AIRFLOW-778] Fix completey broken MetastorePartitionSensor
+- [AIRFLOW-739] Set pickle_info log to debug
+- [AIRFLOW-771] Make S3 logs append instead of clobber
+- [AIRFLOW-773] Fix flaky datetime addition in api test
+- [AIRFLOW-219][AIRFLOW-398] Cgroups + impersonation
+- [AIRFLOW-683] Add jira hook, operator and sensor
+- [AIRFLOW-762] Add Google DataProc delete operator
+- [AIRFLOW-760] Update systemd config
+- [AIRFLOW-759] Use previous dag_run to verify depend_on_past
+- [AIRFLOW-757] Set child_process_log_directory default more sensible
+- [AIRFLOW-692] Open XCom page to super-admins only
+- [AIRFLOW-737] Fix HDFS Sensor directory.
+- [AIRFLOW-747] Fix retry_delay not honoured
+- [AIRFLOW-558] Add Support for dag.catchup=(True|False) Option
+- [AIRFLOW-489] Allow specifying execution date in trigger_dag API
+- [AIRFLOW-738] Commit deleted xcom items before insert
+- [AIRFLOW-729] Add Google Cloud Dataproc cluster creation operator
+- [AIRFLOW-728] Add Google BigQuery table sensor
+- [AIRFLOW-741] Log to debug instead of info for app.py
+- [AIRFLOW-731] Fix period bug for NamedHivePartitionSensor
+- [AIRFLOW-740] Pin jinja2 to < 2.9.0
+- [AIRFLOW-663] Improve time units for task performance charts
+- [AIRFLOW-665] Fix email attachments
+- [AIRFLOW-734] Fix SMTP auth regression when not using user/pass
+- [AIRFLOW-702] Fix LDAP Regex Bug
+- [AIRFLOW-717] Add Cloud Storage updated sensor
+- [AIRFLOW-695] Retries do not execute because dagrun is in FAILED state
+- [AIRFLOW-673] Add operational metrics test for SchedulerJob
+- [AIRFLOW-727] try_number is not increased
+- [AIRFLOW-715] A more efficient HDFS Sensor:
+- [AIRFLOW-716] Allow AVRO BigQuery load-job without schema
+- [AIRFLOW-718] Allow the query URI for DataProc Pig
+- Log needs to be part of try/catch block
+- [AIRFLOW-721] Descendant process can disappear before termination
+- [AIRFLOW-403] Bash operator's kill method leaves underlying processes running
+- [AIRFLOW-657] Add AutoCommit Parameter for MSSQL
+- [AIRFLOW-641] Improve pull request instructions
+- [AIRFLOW-685] Add test for MySqlHook.bulk_load()
+- [AIRFLOW-686] Match auth backend config section
+- [AIRFLOW-691] Add SSH KeepAlive option to SSH_hook
+- [AIRFLOW-709] Use same engine for migrations and reflection
+- [AIRFLOW-700] Update to reference to web authentication documentation
+- [AIRFLOW-649] Support non-sched DAGs in LatestOnlyOp
+- [AIRFLOW-712] Fix AIRFLOW-667 to use proper HTTP error properties
+- [AIRFLOW-710] Add OneFineStay as official user
+- [AIRFLOW-703][AIRFLOW-1] Stop Xcom being cleared too early
+- [AIRFLOW-679] Stop concurrent task instances from running
+- [AIRFLOW-704][AIRFLOW-1] Fix invalid syntax in BQ hook
+- [AIRFLOW-667] Handle BigQuery 503 error
+- [AIRFLOW-680] Disable connection pool for commands
+- [AIRFLOW-678] Prevent scheduler from double triggering TIs
+- [AIRFLOW-677] Kill task if it fails to heartbeat
+- [AIRFLOW-674] Ability to add descriptions for DAGs
+- [AIRFLOW-682] Bump MAX_PERIODS to make mark_success work for large DAGs
+- Use jdk selector to set required jdk
+- [AIRFLOW-647] Restore dag.get_active_runs
+- [AIRFLOW-662] Change seasons to months in project description
+- [AIRFLOW-656] Add dag/task/date index to xcom table
+- [AIRFLOW-658] Improve schema_update_options in GCP
+- [AIRFLOW-41] Fix pool oversubscription
+- [AIRFLOW-489] Add API Framework
+- [AIRFLOW-653] Add some missing endpoint tests
+- [AIRFLOW-652] Remove obsolete endpoint
+- [AIRFLOW-345] Add contrib ECSOperator
+- [AIRFLOW-650] Adding Celect to user list
+- [AIRFLOW-510] Filter Paused Dags, show Last Run & Trigger Dag
+- [AIRFLOW-643] Improve date handling for sf_hook
+- [AIRFLOW-638] Add schema_update_options to GCP ops
+- [AIRFLOW-640] Install and enable nose-ignore-docstring
+- [AIRFLOW-639]AIRFLOW-639] Alphasort package names
+- [AIRFLOW-375] Fix pylint errors
+- [AIRFLOW-347] Show empty DAG runs in tree view
+- [AIRFLOW-628] Adding SalesforceHook to contrib/hooks
+- [AIRFLOW-514] hive hook loads data from pandas DataFrame into hive and infers types
+- [AIRFLOW-565] Fixes DockerOperator on Python3.x
+- [AIRFLOW-635] Encryption option for S3 hook
+- [AIRFLOW-137] Fix max_active_runs on clearing tasks
+- [AIRFLOW-343] Fix schema plumbing in HiveServer2Hook
+- [AIRFLOW-130] Fix ssh operator macosx
+- [AIRFLOW-633] Show TI attributes in TI view
+- [AIRFLOW-626][AIRFLOW-1] HTML Content does not show up when sending email with attachment
+- [AIRFLOW-533] Set autocommit via set_autocommit
+- [AIRFLOW-629] stop pinning lxml
+- [AIRFLOW-464] Add setdefault method to Variable
+- [AIRFLOW-626][AIRFLOW-1] HTML Content does not show up when sending email with attachment
+- [AIRFLOW-591] Add datadog hook & sensor
+- [AIRFLOW-561] Add RedshiftToS3Transfer operator
+- [AIRFLOW-570] Pass root to date form on gantt
+- [AIRFLOW-504] Store fractional seconds in MySQL tables
+- [AIRFLOW-623] LDAP attributes not always a list
+- [AIRFLOW-611] source_format in BigQueryBaseCursor
+- [AIRFLOW-619] Fix exception in Gannt chart
+- [AIRFLOW-618] Cast DateTimes to avoid sqllite errors
+- [AIRFLOW-422] Add JSON endpoint for task info
+- [AIRFLOW-616][AIRFLOW-617] Minor fixes to PR tool UX
+- [AIRFLOW-179] Fix DbApiHook with non-ASCII chars
+- [AIRFLOW-566] Add timeout while fetching logs
+- [AIRFLOW-615] Set graph glyphicon first
+- [AIRFLOW-609] Add application_name to PostgresHook
+- [AIRFLOW-604] Revert .first() to .one()
+- [AIRFLOW-370] Create AirflowConfigException in exceptions.py
+- [AIRFLOW-582] Fixes TI.get_dagrun filter (removes start_date)
+- [AIRFLOW-568] Fix double task_stats count if a DagRun is active
+- [AIRFLOW-585] Fix race condition in backfill execution loop
+- [AIRFLOW-580] Prevent landscape warning on .format
+- [AIRFLOW-597] Check if content is None, not false-equivalent
+- [AIRFLOW-586] test_dag_v1 fails from 0 to 3 a.m.
+- [AIRFLOW-453] Add XCom Admin Page
+- [AIRFLOW-588] Add Google Cloud Storage Object sensor[]
+- [AIRFLOW-592] example_xcom import Error
+- [AIRFLOW-587] Fix incorrect scope for Google Auth[]
+- [AIRFLOW-589] Add templatable job_name[]
+- [AIRFLOW-227] Show running config in config view
+- [AIRFLOW-319]AIRFLOW-319] xcom push response in HTTP Operator
+- [AIRFLOW-385] Add symlink to latest scheduler log directory
+- [AIRFLOW-583] Fix decode error in gcs_to_bq
+- [AIRFLOW-96] s3_conn_id using environment variable
+- [AIRFLOW-575] Clarify tutorial and FAQ about `schedule_interval` always inheriting from DAG object
+- [AIRFLOW-577] Output BigQuery job for improved debugging
+- [AIRFLOW-560] Get URI & SQLA engine from Connection
+- [AIRFLOW-518] Require DataProfilingMixin for Variables CRUD
+- [AIRFLOW-553] Fix load path for filters.js
+- [AIRFLOW-554] Add Jinja support to Spark-sql
+- [AIRFLOW-550] Make ssl config check empty string safe
+- [AIRFLOW-500] Use id for github allowed teams
+- [AIRFLOW-556] Add UI PR guidelines
+- [AIRFLOW-358][AIRFLOW-430] Add `connections` cli
+- [AIRFLOW-548] Load DAGs immediately & continually
+- [AIRFLOW-539] Updated BQ hook and BQ operator to support Standard SQL.
+- [AIRFLOW-378] Add string casting to params of spark-sql operator
+- [AIRFLOW-544] Add Pause/Resume toggle button
+- [AIRFLOW-333][AIRFLOW-258] Fix non-module plugin components
+- [AIRFLOW-542] Add tooltip to DAGs links icons
+- [AIRFLOW-530] Update docs to reflect connection environment var has to be in uppercase
+- [AIRFLOW-525] Update template_fields in Qubole Op
+- [AIRFLOW-480] Support binary file download from GCS
+- [AIRFLOW-198] Implement latest_only_operator
+- [AIRFLOW-91] Add SSL config option for the webserver
+- [AIRFLOW-191] Fix connection leak with PostgreSQL backend
+- [AIRFLOW-512] Fix 'bellow' typo in docs & comments
+- [AIRFLOW-509][AIRFLOW-1] Create operator to delete tables in BigQuery
+- [AIRFLOW-498] Remove hard-coded gcp project id
+- [AIRFLOW-505] Support unicode characters in authors' names
+- [AIRFLOW-494] Add per-operator success/failure metrics
+- [AIRFLOW-488] Fix test_simple fail
+- [AIRFLOW-468] Update Panda requirement to 0.17.1
+- [AIRFLOW-159] Add cloud integration section + GCP documentation
+- [AIRFLOW-477][AIRFLOW-478] Restructure security section for clarity
+- [AIRFLOW-467] Allow defining of project_id in BigQueryHook
+- [AIRFLOW-483] Change print to logging statement
+- [AIRFLOW-475] make the segment granularity in Druid hook configurable
 
 
-AIRFLOW 1.7.2
+Airflow 1.7.2
 -------------
 
-[AIRFLOW-463] Link Airflow icon to landing page
-[AIRFLOW-149] Task Dependency Engine + Why Isn't My Task Running View
-[AIRFLOW-361] Add default failure handler for the Qubole Operator
-[AIRFLOW-353] Fix dag run status update failure
-[AIRFLOW-447] Store source URIs in Python 3 compatible list
-[AIRFLOW-443] Make module names unique when importing
-[AIRFLOW-444] Add Google authentication backend
-[AIRFLOW-446][AIRFLOW-445] Adds missing dataproc submit options
-[AIRFLOW-431] Add CLI for CRUD operations on pools
-[AIRFLOW-329] Update Dag Overview Page with Better Status Columns
-[AIRFLOW-360] Fix style warnings in models.py
-[AIRFLOW-425] Add white fill for null state tasks in tree view.
-[AIRFLOW-69] Use dag runs in backfill jobs
-[AIRFLOW-415] Make dag_id not found error clearer
-[AIRFLOW-416] Use ordinals in README's company list
-[AIRFLOW-369] Allow setting default DAG orientation
-[AIRFLOW-410] Add 2 Q/A to the FAQ in the docs
-[AIRFLOW-407] Add different colors for some sensors
-[AIRFLOW-414] Improve error message for missing FERNET_KEY
-[AIRFLOW-406] Sphinx/rst fixes
-[AIRFLOW-412] Fix lxml dependency
-[AIRFLOW-413] Fix unset path bug when backfilling via pickle
-[AIRFLOW-78] Airflow clear leaves dag_runs
-[AIRFLOW-402] Remove NamedHivePartitionSensor static check, add docs
-[AIRFLOW-394] Add an option to the Task Duration graph to show cumulative times
-[AIRFLOW-404] Retry download if unpacking fails for hive
-[AIRFLOW-276] Gunicorn rolling restart
-[AIRFLOW-399] Remove dags/testdruid.py
-[AIRFLOW-400] models.py/DAG.set_dag_runs_state() does not correctly set state
-[AIRFLOW-395] Fix colon/equal signs typo for resources in default config
-[AIRFLOW-397] Documentation: Fix typo "instatiating" to "instantiating"
-[AIRFLOW-395] Remove trailing commas from resources in config
-[AIRFLOW-388] Add a new chart for Task_Tries for each DAG
-[AIRFLOW-322] Fix typo in FAQ section
-[AIRFLOW-375] Pylint fixes
-limit scope to user email only AIRFLOW-386
-[AIRFLOW-383] Cleanup example qubole operator dag
-[AIRFLOW-160] Parse DAG files through child processes
-[AIRFLOW-381] Manual UI Dag Run creation: require dag_id field
-[AIRFLOW-373] Enhance CLI variables functionality
-[AIRFLOW-379] Enhance Variables page functionality: import/export variables
-[AIRFLOW-331] modify the LDAP authentication config lines in  'Security' sample codes
-[AIRFLOW-356][AIRFLOW-355][AIRFLOW-354] Replace nobr, enable DAG only exists locally message, change edit DAG icon
-[AIRFLOW-362] Import __future__ division
-[AIRFLOW-359] Pin flask-login to 0.2.11
-[AIRFLOW-261] Add bcc and cc fields to EmailOperator
-[AIRFLOW-348] Fix code style warnings
-[AIRFLOW-349] Add metric for number of zombies killed
-[AIRFLOW-340] Remove unused dependency on Babel
-[AIRFLOW-339]: Ability to pass a flower conf file
-[AIRFLOW-341][operators] Add resource requirement attributes to operators
-[AIRFLOW-335] Fix simple style errors/warnings
-[AIRFLOW-337] Add __repr__ to VariableAccessor and VariableJsonAccessor
-[AIRFLOW-334] Fix using undefined variable
-[AIRFLOW-315] Fix blank lines code style warnings
-[AIRFLOW-306] Add Spark-sql Hook and Operator
-[AIRFLOW-327] Add rename method to the FTPHook
-[AIRFLOW-321] Fix a wrong code example about tests/dags
-[AIRFLOW-316] Always check DB state for Backfill Job execution
-[AIRFLOW-264] Adding workload management for Hive
-[AIRFLOW-297] support exponential backoff option for retry delay
-[AIRFLOW-31][AIRFLOW-200] Add note to updating.md
-[AIRFLOW-307] There is no __neq__ python magic method.
-[AIRFLOW-309] Add requirements of develop dependencies to docs
-[AIRFLOW-307] Rename __neq__ to __ne__ python magic method.
-[AIRFLOW-313] Fix code style for sqoop_hook.py
-[AIRFLOW-311] Fix wrong path in CONTRIBUTING.md
-[AIRFLOW-24] DataFlow Java Operator
-[AIRFLOW-308] Add link to refresh DAG within DAG view header
-[AIRFLOW-314] Fix BigQuery cursor run_table_upsert method
-[AIRFLOW-298] fix incubator diclaimer in docs
-[AIRFLOW-284] HiveServer2Hook fix for cursor scope for get_results
-[AIRFLOW-260] More graceful exit when issues can't be closed
-[AIRFLOW-260] Handle case when no version is found
-[AIRFLOW-228] Handle empty version list in PR tool
-[AIRFLOW-302] Improve default squash commit message
-[AIRFLOW-187] Improve prompt styling
-[AIRFLOW-187] Fix typo in argument name
-[AIRFLOW-187] Move "Close XXX" message to end of squash commit
-[AIRFLOW-247] Add EMR hook, operators and sensors. Add AWS base hook
-[AIRFLOW-301] Fix broken unit test
-[AIRFLOW-100] Add execution_date_fn to ExternalTaskSensor
-[AIRFLOW-282] Remove PR Tool logic that depends on version formatting
-[AIRFLOW-291] Add index for state in TI table
-[AIRFLOW-269] Add some unit tests for PostgreSQL
-[AIRFLOW-296] template_ext is being treated as a string rather than a tuple in qubole operator
-[AIRFLOW-286] Improve FTPHook to implement context manager interface
-[AIRFLOW-243] Create NamedHivePartitionSensor
-[AIRFLOW-246] Improve dag_stats endpoint query
-[AIRFLOW-189] Highlighting of Parent/Child nodes in Graphs
-[ARFLOW-255] Check dagrun timeout when comparing active runs
-[AIRFLOW-281] Add port to mssql_hook
-[AIRFLOW-285] Use Airflow 2.0 style imports for all remaining hooks/operators
-[AIRFLOW-40] Add LDAP group filtering feature.
-[AIRFLOW-277] Multiple deletions does not work in Task Instances view if using SQLite backend
-[AIRFLOW-200] Make hook/operator imports lazy, and print proper exceptions
-[AIRFLOW-283] Make store_to_xcom_key a templated field in GoogleCloudStorageDownloadOperator
-[AIRFLOW-278] Support utf-8 ecoding for SQL
-[AIRFLOW-280] clean up tmp druid table no matter if an ingestion job succeeds or not
-[AIRFLOW-274] Add XCom functionality to GoogleCloudStorageDownloadOperator
-[AIRFLOW-273] Create an svg version of the airflow logo.
-[AIRFLOW-275] Update contributing guidelines
-[AIRFLOW-244] Modify hive operator to inject analysis data
-[AIRFLOW-162] Allow variable to be accessible into templates
-[AIRFLOW-248] Add Apache license header to all files
-[AIRFLOW-263] Remove temp backtick file
-[AIRFLOW-252] Raise Sqlite exceptions when deleting tasks instance in WebUI
-[AIRFLOW-180] Fix timeout behavior for sensors
-[AIRFLOW-262] Simplify commands in MANIFEST.in
-[AIRFLOW-31] Add zope dependency
-[AIRFLOW-6] Remove dependency on Highcharts
-[AIRFLOW-234] make task that aren't `running` self-terminate
-[AIRFLOW-256] Fix test_scheduler_reschedule heartrate
-Add Python 3 compatibility fix
-[AIRFLOW-31] Use standard imports for hooks/operators
-[AIRFLOW-173] Initial implementation of FileSensor
-[AIRFLOW-224] Collect orphaned tasks and reschedule them
-[AIRFLOW-239] Fix tests indentation
-[AIRFLOW-225] Better units for task duration graph
-[AIRFLOW-241] Add testing done section to PR template
-[AIRFLOW-222] Show duration of task instances in ui
-[AIRFLOW-231] Do not eval user input in PrestoHook
-[AIRFLOW-216] Add Sqoop Hook and Operator
-[AIRFLOW-171] Add upgrade notes on email and S3 to 1.7.1.2
-[AIRFLOW-238] Make compatible with flask-admin 1.4.1
-[AIRFLOW-230] [HiveServer2Hook] adding multi statements support
-[AIRFLOW-142] setup_env.sh doesn't download hive tarball if hdp is specified as distro
-[AIRFLOW-223] Make parametrable the IP on which Flower binds to
-[AIRFLOW-218] Added option to enable webserver gunicorn access/err logs
-[AIRFLOW-213] Add "Closes #X" phrase to commit messages
-[AIRFLOW-68] Align start_date with the schedule_interval
-[AIRFLOW-9] Improving docs to meet Apache's standards
-[AIRFLOW-131] Make XCom.clear more selective
-[AIRFLOW-214] Fix occasion of detached taskinstance
-[AIRFLOW-206] Add commit to close PR
-[AIRFLOW-206] Always load local log files if they exist
-[AIRFLOW-211] Fix JIRA "resolve" vs "close" behavior
-[AIRFLOW-64] Add note about relative DAGS_FOLDER
-[AIRFLOW-114] Sort plugins dropdown
-[AIRFLOW-209] Add scheduler tests and improve lineage handling
-[AIRFLOW-207] Improve JIRA auth workflow
-[AIRFLOW-187] Improve PR tool UX
-[AIRFLOW-155] Documentation of Qubole Operator
-Optimize and refactor process_dag
-[AIRFLOW-185] Handle empty versions list
-[AIRFLOW-201] Fix for HiveMetastoreHook + kerberos
-[AIRFLOW-202]: Fixes stray print line
-[AIRFLOW-196] Fix bug that exception is not handled in HttpSensor
-[AIRFLOW-195] : Add toggle support to subdag clearing in the CLI
-[AIRFLOW-23] Support for Google Cloud DataProc
-[AIRFLOW-25] Configuration for Celery always required
-[AIRFLOW-190] Add codecov and remove download count
-[AIRFLOW-168] Correct evaluation of @once schedule
-[AIRFLOW-183] Fetch log from remote when worker returns 4xx/5xx response
-[AIRFLOW-181] Fix failing unpacking of hadoop by redownloading
-[AIRFLOW-176] remove unused formatting key
-[AIRFLOW-167]: Add dag_state option in cli
-[AIRFLOW-178] Fix bug so that zip file is detected in DAG folder
-[AIRFLOW-176] Improve PR Tool JIRA workflow
-AIRFLOW-45: Support Hidden Airflow Variables
-[AIRFLOW-175] Run git-reset before checkout in PR tool
-[AIRFLOW-157] Make PR tool Py3-compat; add JIRA command
-[AIRFLOW-170] Add missing @apply_defaults
-
+- [AIRFLOW-463] Link Airflow icon to landing page
+- [AIRFLOW-149] Task Dependency Engine + Why Isn't My Task Running View
+- [AIRFLOW-361] Add default failure handler for the Qubole Operator
+- [AIRFLOW-353] Fix dag run status update failure
+- [AIRFLOW-447] Store source URIs in Python 3 compatible list
+- [AIRFLOW-443] Make module names unique when importing
+- [AIRFLOW-444] Add Google authentication backend
+- [AIRFLOW-446][AIRFLOW-445] Adds missing dataproc submit options
+- [AIRFLOW-431] Add CLI for CRUD operations on pools
+- [AIRFLOW-329] Update Dag Overview Page with Better Status Columns
+- [AIRFLOW-360] Fix style warnings in models.py
+- [AIRFLOW-425] Add white fill for null state tasks in tree view.
+- [AIRFLOW-69] Use dag runs in backfill jobs
+- [AIRFLOW-415] Make dag_id not found error clearer
+- [AIRFLOW-416] Use ordinals in README's company list
+- [AIRFLOW-369] Allow setting default DAG orientation
+- [AIRFLOW-410] Add 2 Q/A to the FAQ in the docs
+- [AIRFLOW-407] Add different colors for some sensors
+- [AIRFLOW-414] Improve error message for missing FERNET_KEY
+- [AIRFLOW-406] Sphinx/rst fixes
+- [AIRFLOW-412] Fix lxml dependency
+- [AIRFLOW-413] Fix unset path bug when backfilling via pickle
+- [AIRFLOW-78] Airflow clear leaves dag_runs
+- [AIRFLOW-402] Remove NamedHivePartitionSensor static check, add docs
+- [AIRFLOW-394] Add an option to the Task Duration graph to show cumulative times
+- [AIRFLOW-404] Retry download if unpacking fails for hive
+- [AIRFLOW-276] Gunicorn rolling restart
+- [AIRFLOW-399] Remove dags/testdruid.py
+- [AIRFLOW-400] models.py/DAG.set_dag_runs_state() does not correctly set state
+- [AIRFLOW-395] Fix colon/equal signs typo for resources in default config
+- [AIRFLOW-397] Documentation: Fix typo "instatiating" to "instantiating"
+- [AIRFLOW-395] Remove trailing commas from resources in config
+- [AIRFLOW-388] Add a new chart for Task_Tries for each DAG
+- [AIRFLOW-322] Fix typo in FAQ section
+- [AIRFLOW-375] Pylint fixes
+- limit scope to user email only AIRFLOW-386
+- [AIRFLOW-383] Cleanup example qubole operator dag
+- [AIRFLOW-160] Parse DAG files through child processes
+- [AIRFLOW-381] Manual UI Dag Run creation: require dag_id field
+- [AIRFLOW-373] Enhance CLI variables functionality
+- [AIRFLOW-379] Enhance Variables page functionality: import/export variables
+- [AIRFLOW-331] modify the LDAP authentication config lines in  'Security' sample codes
+- [AIRFLOW-356][AIRFLOW-355][AIRFLOW-354] Replace nobr, enable DAG only exists locally message, change edit DAG icon
+- [AIRFLOW-362] Import __future__ division
+- [AIRFLOW-359] Pin flask-login to 0.2.11
+- [AIRFLOW-261] Add bcc and cc fields to EmailOperator
+- [AIRFLOW-348] Fix code style warnings
+- [AIRFLOW-349] Add metric for number of zombies killed
+- [AIRFLOW-340] Remove unused dependency on Babel
+- [AIRFLOW-339]: Ability to pass a flower conf file
+- [AIRFLOW-341][operators] Add resource requirement attributes to operators
+- [AIRFLOW-335] Fix simple style errors/warnings
+- [AIRFLOW-337] Add __repr__ to VariableAccessor and VariableJsonAccessor
+- [AIRFLOW-334] Fix using undefined variable
+- [AIRFLOW-315] Fix blank lines code style warnings
+- [AIRFLOW-306] Add Spark-sql Hook and Operator
+- [AIRFLOW-327] Add rename method to the FTPHook
+- [AIRFLOW-321] Fix a wrong code example about tests/dags
+- [AIRFLOW-316] Always check DB state for Backfill Job execution
+- [AIRFLOW-264] Adding workload management for Hive
+- [AIRFLOW-297] support exponential backoff option for retry delay
+- [AIRFLOW-31][AIRFLOW-200] Add note to updating.md
+- [AIRFLOW-307] There is no __neq__ python magic method.
+- [AIRFLOW-309] Add requirements of develop dependencies to docs
+- [AIRFLOW-307] Rename __neq__ to __ne__ python magic method.
+- [AIRFLOW-313] Fix code style for sqoop_hook.py
+- [AIRFLOW-311] Fix wrong path in CONTRIBUTING.md
+- [AIRFLOW-24] DataFlow Java Operator
+- [AIRFLOW-308] Add link to refresh DAG within DAG view header
+- [AIRFLOW-314] Fix BigQuery cursor run_table_upsert method
+- [AIRFLOW-298] fix incubator diclaimer in docs
+- [AIRFLOW-284] HiveServer2Hook fix for cursor scope for get_results
+- [AIRFLOW-260] More graceful exit when issues can't be closed
+- [AIRFLOW-260] Handle case when no version is found
+- [AIRFLOW-228] Handle empty version list in PR tool
+- [AIRFLOW-302] Improve default squash commit message
+- [AIRFLOW-187] Improve prompt styling
+- [AIRFLOW-187] Fix typo in argument name
+- [AIRFLOW-187] Move "Close XXX" message to end of squash commit
+- [AIRFLOW-247] Add EMR hook, operators and sensors. Add AWS base hook
+- [AIRFLOW-301] Fix broken unit test
+- [AIRFLOW-100] Add execution_date_fn to ExternalTaskSensor
+- [AIRFLOW-282] Remove PR Tool logic that depends on version formatting
+- [AIRFLOW-291] Add index for state in TI table
+- [AIRFLOW-269] Add some unit tests for PostgreSQL
+- [AIRFLOW-296] template_ext is being treated as a string rather than a tuple in qubole operator
+- [AIRFLOW-286] Improve FTPHook to implement context manager interface
+- [AIRFLOW-243] Create NamedHivePartitionSensor
+- [AIRFLOW-246] Improve dag_stats endpoint query
+- [AIRFLOW-189] Highlighting of Parent/Child nodes in Graphs
+- [ARFLOW-255] Check dagrun timeout when comparing active runs
+- [AIRFLOW-281] Add port to mssql_hook
+- [AIRFLOW-285] Use Airflow 2.0 style imports for all remaining hooks/operators
+- [AIRFLOW-40] Add LDAP group filtering feature.
+- [AIRFLOW-277] Multiple deletions does not work in Task Instances view if using SQLite backend
+- [AIRFLOW-200] Make hook/operator imports lazy, and print proper exceptions
+- [AIRFLOW-283] Make store_to_xcom_key a templated field in GoogleCloudStorageDownloadOperator
+- [AIRFLOW-278] Support utf-8 ecoding for SQL
+- [AIRFLOW-280] clean up tmp druid table no matter if an ingestion job succeeds or not
+- [AIRFLOW-274] Add XCom functionality to GoogleCloudStorageDownloadOperator
+- [AIRFLOW-273] Create an svg version of the airflow logo.
+- [AIRFLOW-275] Update contributing guidelines
+- [AIRFLOW-244] Modify hive operator to inject analysis data
+- [AIRFLOW-162] Allow variable to be accessible into templates
+- [AIRFLOW-248] Add Apache license header to all files
+- [AIRFLOW-263] Remove temp backtick file
+- [AIRFLOW-252] Raise Sqlite exceptions when deleting tasks instance in WebUI
+- [AIRFLOW-180] Fix timeout behavior for sensors
+- [AIRFLOW-262] Simplify commands in MANIFEST.in
+- [AIRFLOW-31] Add zope dependency
+- [AIRFLOW-6] Remove dependency on Highcharts
+- [AIRFLOW-234] make task that aren't `running` self-terminate
+- [AIRFLOW-256] Fix test_scheduler_reschedule heartrate
+- Add Python 3 compatibility fix
+- [AIRFLOW-31] Use standard imports for hooks/operators
+- [AIRFLOW-173] Initial implementation of FileSensor
+- [AIRFLOW-224] Collect orphaned tasks and reschedule them
+- [AIRFLOW-239] Fix tests indentation
+- [AIRFLOW-225] Better units for task duration graph
+- [AIRFLOW-241] Add testing done section to PR template
+- [AIRFLOW-222] Show duration of task instances in ui
+- [AIRFLOW-231] Do not eval user input in PrestoHook
+- [AIRFLOW-216] Add Sqoop Hook and Operator
+- [AIRFLOW-171] Add upgrade notes on email and S3 to 1.7.1.2
+- [AIRFLOW-238] Make compatible with flask-admin 1.4.1
+- [AIRFLOW-230] [HiveServer2Hook] adding multi statements support
+- [AIRFLOW-142] setup_env.sh doesn't download hive tarball if hdp is specified as distro
+- [AIRFLOW-223] Make parametrable the IP on which Flower binds to
+- [AIRFLOW-218] Added option to enable webserver gunicorn access/err logs
+- [AIRFLOW-213] Add "Closes #X" phrase to commit messages
+- [AIRFLOW-68] Align start_date with the schedule_interval
+- [AIRFLOW-9] Improving docs to meet Apache's standards
+- [AIRFLOW-131] Make XCom.clear more selective
+- [AIRFLOW-214] Fix occasion of detached taskinstance
+- [AIRFLOW-206] Add commit to close PR
+- [AIRFLOW-206] Always load local log files if they exist
+- [AIRFLOW-211] Fix JIRA "resolve" vs "close" behavior
+- [AIRFLOW-64] Add note about relative DAGS_FOLDER
+- [AIRFLOW-114] Sort plugins dropdown
+- [AIRFLOW-209] Add scheduler tests and improve lineage handling
+- [AIRFLOW-207] Improve JIRA auth workflow
+- [AIRFLOW-187] Improve PR tool UX
+- [AIRFLOW-155] Documentation of Qubole Operator
+- Optimize and refactor process_dag
+- [AIRFLOW-185] Handle empty versions list
+- [AIRFLOW-201] Fix for HiveMetastoreHook + kerberos
+- [AIRFLOW-202]: Fixes stray print line
+- [AIRFLOW-196] Fix bug that exception is not handled in HttpSensor
+- [AIRFLOW-195] : Add toggle support to subdag clearing in the CLI
+- [AIRFLOW-23] Support for Google Cloud DataProc
+- [AIRFLOW-25] Configuration for Celery always required
+- [AIRFLOW-190] Add codecov and remove download count
+- [AIRFLOW-168] Correct evaluation of @once schedule
+- [AIRFLOW-183] Fetch log from remote when worker returns 4xx/5xx response
+- [AIRFLOW-181] Fix failing unpacking of hadoop by redownloading
+- [AIRFLOW-176] remove unused formatting key
+- [AIRFLOW-167]: Add dag_state option in cli
+- [AIRFLOW-178] Fix bug so that zip file is detected in DAG folder
+- [AIRFLOW-176] Improve PR Tool JIRA workflow
+- AIRFLOW-45: Support Hidden Airflow Variables
+- [AIRFLOW-175] Run git-reset before checkout in PR tool
+- [AIRFLOW-157] Make PR tool Py3-compat; add JIRA command
+- [AIRFLOW-170] Add missing @apply_defaults
 
-AIRFLOW 1.7.1, 2016-05-19
+Airflow 1.7.1, 2016-05-19
 -------------------------
 
 - Fix : Don't treat premature tasks as could_not_run tasks
-- AIRFLOW-92 Avoid unneeded upstream_failed session closes apache/incubator-airflow#1485
+- AIRFLOW-92 Avoid unneeded upstream_failed session closes apache/airflow#1485
 - Add logic to lock DB and avoid race condition
 - Handle queued tasks from multiple jobs/executors
 - AIRFLOW-52 Warn about overwriting tasks in a DAG
@@ -978,7 +2496,7 @@ AIRFLOW 1.7.1, 2016-05-19
 - Show only Airflow's deprecation warnings
 - Set DAG_FOLDER for unit tests
 - Missing comma in setup.py
-- Deprecate *args and **kwargs in BaseOperator
+- Deprecate args and kwargs in BaseOperator
 - Raise deep scheduler exceptions to force a process restart.
 - Change inconsistent example DAG owners
 - Fix module path of send_email_smtp in configuration
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e45060bfe75c0..56258d5d75142 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,9 +1,27 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Contributing
 
 Contributions are welcome and are greatly appreciated! Every
 little bit helps, and credit will always be given.
 
-
 # Table of Contents
   * [TOC](#table-of-contents)
   * [Types of Contributions](#types-of-contributions)
@@ -15,11 +33,10 @@ little bit helps, and credit will always be given.
   * [Documentation](#documentation)
   * [Development and Testing](#development-and-testing)
       - [Setting up a development environment](#setting-up-a-development-environment)
-      - [Pull requests guidelines](#pull-request-guidelines)
-      - [Testing Locally](#testing-locally)
+      - [Running unit tests](#running-unit-tests)
+  * [Pull requests guidelines](#pull-request-guidelines)
   * [Changing the Metadata Database](#changing-the-metadata-database)
 
-
 ## Types of Contributions
 
 ### Report Bugs
@@ -64,11 +81,13 @@ If you are proposing a feature:
 ## Documentation
 
 The latest API documentation is usually available
-[here](https://airflow.incubator.apache.org/). To generate a local version,
+[here](https://airflow.apache.org/). To generate a local version,
 you need to have set up an Airflow development environemnt (see below). Also
 install the `doc` extra.
 
-    pip install -e .[doc]
+```
+pip install -e '.[doc]'
+```
 
 Generate the documentation by running:
 
@@ -79,65 +98,170 @@ extras to build the full API reference.
 
 ## Development and Testing
 
-### Set up a development env using Docker
+### Setting up a development environment
 
-Go to your Airflow directory and start a new docker container. You can choose between Python 2 or 3, whatever you prefer.
+There are three ways to setup an Apache Airflow development environment.
 
-```
-# Start docker in your Airflow directory
-docker run -t -i -v `pwd`:/airflow/ python:2 bash
+1. Using tools and libraries installed directly on your system.
+
+  Install Python (2.7.x or 3.4.x), MySQL, and libxml by using system-level package
+  managers like yum, apt-get for Linux, or Homebrew for Mac OS at first. Refer to the [base CI Dockerfile](https://github.com/apache/incubator-airflow-ci/blob/master/Dockerfile) for
+  a comprehensive list of required packages.
+
+  Then install python development requirements. It is usually best to work in a virtualenv:
+
+  ```bash
+  cd $AIRFLOW_HOME
+  virtualenv env
+  source env/bin/activate
+  pip install -e '.[devel]'
+  ```
+
+2. Using a Docker container
+
+  Go to your Airflow directory and start a new docker container. You can choose between Python 2 or 3, whatever you prefer.
+
+  ```
+  # Start docker in your Airflow directory
+  docker run -t -i -v `pwd`:/airflow/ -w /airflow/ python:3 bash
+
+
+  # Install Airflow with all the required dependencies,
+  # including the devel which will provide the development tools
+  pip install -e '.[hdfs,hive,druid,devel]'
+
+  # Init the database
+  airflow initdb
+
+  nosetests -v tests/hooks/test_druid_hook.py
+
+    test_get_first_record (tests.hooks.test_druid_hook.TestDruidDbApiHook) ... ok
+    test_get_records (tests.hooks.test_druid_hook.TestDruidDbApiHook) ... ok
+    test_get_uri (tests.hooks.test_druid_hook.TestDruidDbApiHook) ... ok
+    test_get_conn_url (tests.hooks.test_druid_hook.TestDruidHook) ... ok
+    test_submit_gone_wrong (tests.hooks.test_druid_hook.TestDruidHook) ... ok
+    test_submit_ok (tests.hooks.test_druid_hook.TestDruidHook) ... ok
+    test_submit_timeout (tests.hooks.test_druid_hook.TestDruidHook) ... ok
+    test_submit_unknown_response (tests.hooks.test_druid_hook.TestDruidHook) ... ok
+
+    ----------------------------------------------------------------------
+    Ran 8 tests in 3.036s
+
+    OK
+  ```
+
+  The Airflow code is mounted inside of the Docker container, so if you change something using your favorite IDE, you can directly test is in the container.
+
+3. Using [Docker Compose](https://docs.docker.com/compose/) and Airflow's CI scripts.
+
+  Start a docker container through Compose for development to avoid installing the packages directly on your system. The following will give you a shell inside a container, run all required service containers (MySQL, PostgresSQL, krb5 and so on) and install all the dependencies:
+
+  ```bash
+  docker-compose -f scripts/ci/docker-compose.yml run airflow-testing bash
+  # From the container
+  export TOX_ENV=py27-backend_mysql-env_docker
+  /app/scripts/ci/run-ci.sh
+  ```
 
-# Go to the Airflow directory
-cd /airflow/
+  If you wish to run individual tests inside of Docker environment you can do as follows:
 
-# Install Airflow with all the required dependencies,
-# including the devel which will provide the development tools
-pip install -e ".[hdfs,hive,druid,devel]"
+  ```bash
+  # From the container (with your desired environment) with druid hook
+  export TOX_ENV=py27-backend_mysql-env_docker
+  /app/scripts/ci/run-ci.sh -- tests/hooks/test_druid_hook.py
+  ```
 
-# Init the database
-airflow initdb
 
-nosetests -v tests/hooks/test_druid_hook.py
+### Running unit tests
 
-  test_get_first_record (tests.hooks.test_druid_hook.TestDruidDbApiHook) ... ok
-  test_get_records (tests.hooks.test_druid_hook.TestDruidDbApiHook) ... ok
-  test_get_uri (tests.hooks.test_druid_hook.TestDruidDbApiHook) ... ok
-  test_get_conn_url (tests.hooks.test_druid_hook.TestDruidHook) ... ok
-  test_submit_gone_wrong (tests.hooks.test_druid_hook.TestDruidHook) ... ok
-  test_submit_ok (tests.hooks.test_druid_hook.TestDruidHook) ... ok
-  test_submit_timeout (tests.hooks.test_druid_hook.TestDruidHook) ... ok
-  test_submit_unknown_response (tests.hooks.test_druid_hook.TestDruidHook) ... ok
+To run tests locally, once your unit test environment is setup (directly on your
+system or through our Docker setup) you should be able to simply run
+``./run_unit_tests.sh`` at will.
+
+For example, in order to just execute the "core" unit tests, run the following:
+
+```
+./run_unit_tests.sh tests.core:CoreTest -s --logging-level=DEBUG
+```
 
-  ----------------------------------------------------------------------
-  Ran 8 tests in 3.036s
+or a single test method:
 
-  OK
+```
+./run_unit_tests.sh tests.core:CoreTest.test_check_operators -s --logging-level=DEBUG
 ```
 
-The Airflow code is mounted inside of the Docker container, so if you change something using your favorite IDE, you can directly test is in the container.
+To run the whole test suite with Docker Compose, do:
 
-### Set up a development env using Virtualenv
+```
+# Install Docker Compose first, then this will run the tests
+docker-compose -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh
+```
 
-Please install python(2.7.x or 3.4.x), mysql, and libxml by using system-level package
-managers like yum, apt-get for Linux, or homebrew for Mac OS at first.
-It is usually best to work in a virtualenv and tox. Install development requirements:
+Alternatively can also set up [Travis CI](https://travis-ci.org/) on your repo to automate this.
+It is free for open source projects.
 
-    cd $AIRFLOW_HOME
-    virtualenv env
-    source env/bin/activate
-    pip install -e .[devel]
-    tox
+Another great way of automating linting and testing is to use [Git Hooks](https://git-scm.com/book/uz/v2/Customizing-Git-Git-Hooks). For example you could create a `pre-commit` file based on the Travis CI Pipeline so that before each commit a local pipeline will be triggered and if this pipeline fails (returns an exit code other than `0`) the commit does not come through.
+This "in theory" has the advantage that you can not commit any code that fails that again reduces the errors in the Travis CI Pipelines.
+
+Since there are a lot of tests the script would last very long so you propably only should test your new feature locally.
+
+The following example of a `pre-commit` file allows you..
+- to lint your code via flake8
+- to test your code via nosetests in a docker container based on python 2
+- to test your code via nosetests in a docker container based on python 3
+
+```
+#!/bin/sh
+
+GREEN='\033[0;32m'
+NO_COLOR='\033[0m'
+
+setup_python_env() {
+    local venv_path=${1}
+
+    echo -e "${GREEN}Activating python virtual environment ${venv_path}..${NO_COLOR}"
+    source ${venv_path}
+}
+run_linting() {
+    local project_dir=$(git rev-parse --show-toplevel)
+
+    echo -e "${GREEN}Running flake8 over directory ${project_dir}..${NO_COLOR}"
+    flake8 ${project_dir}
+}
+run_testing_in_docker() {
+    local feature_path=${1}
+    local airflow_py2_container=${2}
+    local airflow_py3_container=${3}
+
+    echo -e "${GREEN}Running tests in ${feature_path} in airflow python 2 docker container..${NO_COLOR}"
+    docker exec -i -w /airflow/ ${airflow_py2_container} nosetests -v ${feature_path}
+    echo -e "${GREEN}Running tests in ${feature_path} in airflow python 3 docker container..${NO_COLOR}"
+    docker exec -i -w /airflow/ ${airflow_py3_container} nosetests -v ${feature_path}
+}
+
+set -e
+# NOTE: Before running this make sure you have set the function arguments correctly.
+setup_python_env /Users/feluelle/venv/bin/activate
+run_linting
+run_testing_in_docker tests/contrib/hooks/test_imap_hook.py dazzling_chatterjee quirky_stallman
+
+```
+
+For more information on how to run a subset of the tests, take a look at the
+nosetests docs.
+
+See also the list of test classes and methods in `tests/core.py`.
 
 Feel free to customize based on the extras available in [setup.py](./setup.py)
 
-### Pull Request Guidelines
+## Pull Request Guidelines
 
 Before you submit a pull request from your forked repo, check that it
 meets these guidelines:
 
 1. The pull request should include tests, either as doctests, unit tests, or
-both. The airflow repo uses [Travis CI](https://travis-ci.org/apache/incubator-airflow)
-to run the tests and [codecov](https://codecov.io/gh/apache/incubator-airflow)
+both. The airflow repo uses [Travis CI](https://travis-ci.org/apache/airflow)
+to run the tests and [codecov](https://codecov.io/gh/apache/airflow)
 to track coverage. You can set up both for free on your fork. It will
 help you making sure you do not break the build with your PR and that you help
 increase coverage.
@@ -149,7 +273,7 @@ The JIRA link should also be contained in the PR description.
 4. Preface your commit's subject & PR's title with **[AIRFLOW-XXX]**
 where *XXX* is the JIRA number. We compose release notes (i.e. for Airflow releases) from all commit titles in a release.
 By placing the JIRA number in the commit title and hence in the release notes,
-Airflow users can look into JIRA and Github PRs for more details about a particular change.
+Airflow users can look into JIRA and GitHub PRs for more details about a particular change.
 5. Add an [Apache License](http://www.apache.org/legal/src-headers.html)
  header to all new files
 6. If the pull request adds functionality, the docs should be updated as part
@@ -161,7 +285,7 @@ writing code that works in both Python 2 and 3, see the documentation at the
 Airflow requirement and should be used where possible).
 8. As Airflow grows as a project, we try to enforce a more consistent
 style and try to follow the Python community guidelines. We track this
-using [landscape.io](https://landscape.io/github/apache/incubator-airflow/),
+using [landscape.io](https://landscape.io/github/apache/airflow/),
 which you can setup on your fork as well to check before you submit your
 PR. We currently enforce most [PEP8](https://www.python.org/dev/peps/pep-0008/)
 and a few other linting rules. It is usually a good idea to lint locally
@@ -171,74 +295,93 @@ using `flake8 airflow tests`. `git diff upstream/master -u -- "*.py" | flake8 --
 commit messages and adhere to them. It makes the lives of those who
 come after you a lot easier.
 
-### Testing locally
-
-#### TL;DR
-Tests can then be run with (see also the [Running unit tests](#running-unit-tests) section below):
-
-    ./run_unit_tests.sh
+### Changing the Metadata Database
 
-Individual test files can be run with:
+When developing features the need may arise to persist information to the the
+metadata database. Airflow has [Alembic](https://bitbucket.org/zzzeek/alembic)
+built-in to handle all schema changes. Alembic must be installed on your
+development machine before continuing.
 
-    nosetests [path to file]
+```
+# starting at the root of the project
+$ pwd
+~/airflow
+# change to the airflow directory
+$ cd airflow
+$ alembic revision -m "add new field to db"
+  Generating
+~/airflow/airflow/migrations/versions/12341123_add_new_field_to_db.py
+```
 
-#### Running unit tests
+## Setting up the node / npm javascript environment (ONLY FOR www_rbac)
 
-We *highly* recommend setting up [Travis CI](https://travis-ci.org/) on
-your repo to automate this. It is free for open source projects. If for
-some reason you cannot, you can use the steps below to run tests.
+`airflow/www_rbac/` contains all npm-managed, front end assets.
+Flask-Appbuilder itself comes bundled with jQuery and bootstrap.
+While these may be phased out over time, these packages are currently not
+managed with npm.
 
-Here are loose guidelines on how to get your environment to run the unit tests.
-We do understand that no one out there can run the full test suite since
-Airflow is meant to connect to virtually any external system and that you most
-likely have only a subset of these in your environment. You should run the
-CoreTests and tests related to things you touched in your PR.
+### Node/npm versions
+Make sure you are using recent versions of node and npm. No problems have been found with node>=8.11.3 and npm>=6.1.3
 
-To set up a unit test environment, first take a look at `run_unit_tests.sh` and
-understand that your ``AIRFLOW_CONFIG`` points to an alternate config file
-while running the tests. You shouldn't have to alter this config file but
-you may if need be.
+### Using npm to generate bundled files
 
-From that point, you can actually export these same environment variables in
-your shell, start an Airflow webserver ``airflow webserver -d`` and go and
-configure your connection. Default connections that are used in the tests
-should already have been created, you just need to point them to the systems
-where you want your tests to run.
+#### npm
+First, npm must be available in your environment. If it is not you can run the following commands
+(taken from [this source](https://gist.github.com/DanHerbert/9520689))
+```
+brew install node --without-npm
+echo prefix=~/.npm-packages >> ~/.npmrc
+curl -L https://www.npmjs.com/install.sh | sh
+```
 
-Once your unit test environment is setup, you should be able to simply run
-``./run_unit_tests.sh`` at will.
+The final step is to add `~/.npm-packages/bin` to your `PATH` so commands you install globally are usable.
+Add something like this to your `.bashrc` file, then `source ~/.bashrc` to reflect the change.
+```
+export PATH="$HOME/.npm-packages/bin:$PATH"
+```
 
-For example, in order to just execute the "core" unit tests, run the following:
+#### npm packages
+To install third party libraries defined in `package.json`, run the
+following within the `airflow/www_rbac/` directory which will install them in a
+new `node_modules/` folder within `www_rbac/`.
 
-```
-./run_unit_tests.sh tests.core:CoreTest -s --logging-level=DEBUG
+```bash
+# from the root of the repository, move to where our JS package.json lives
+cd airflow/www_rbac/
+# run npm install to fetch all the dependencies
+npm install
 ```
 
-or a single test method:
+To parse and generate bundled files for airflow, run either of the
+following commands. The `dev` flag will keep the npm script running and
+re-run it upon any changes within the assets directory.
 
 ```
-./run_unit_tests.sh tests.core:CoreTest.test_check_operators -s --logging-level=DEBUG
+# Compiles the production / optimized js & css
+npm run prod
+
+# Start a web server that manages and updates your assets as you modify them
+npm run dev
 ```
 
-For more information on how to run a subset of the tests, take a look at the
-nosetests docs.
+#### Upgrading npm packages
 
-See also the list of test classes and methods in `tests/core.py`.
+Should you add or upgrade a npm package, which involves changing `package.json`, you'll need to re-run `npm install` 
+and push the newly generated `package-lock.json` file so we get the reproducible build.
 
-### Changing the Metadata Database
+#### Javascript Style Guide
 
-When developing features the need may arise to persist information to the the
-metadata database. Airflow has [Alembic](https://bitbucket.org/zzzeek/alembic)
-built-in to handle all schema changes. Alembic must be installed on your
-development machine before continuing.
+We try to enforce a more consistent style and try to follow the JS community guidelines. 
+Once you add or modify any javascript code in the project, please make sure it follows the guidelines 
+defined in [Airbnb JavaScript Style Guide](https://github.com/airbnb/javascript).
+Apache Airflow uses [ESLint](https://eslint.org/) as a tool for identifying and reporting on patterns in JavaScript,
+which can be used by running any of the following commands.
 
+```bash
+# Check JS code in .js and .html files, and report any errors/warnings
+npm run lint
+
+# Check JS code in .js and .html files, report any errors/warnings and fix them if possible 
+npm run lint:fix
 ```
-# starting at the root of the project
-$ pwd
-~/airflow
-# change to the airflow directory
-$ cd airflow
-$ alembic revision -m "add new field to db"
-  Generating
-~/airflow/airflow/migrations/versions/12341123_add_new_field_to_db.py
-```
+ 
diff --git a/DISCLAIMER b/DISCLAIMER
deleted file mode 100644
index 8fe69887c3638..0000000000000
--- a/DISCLAIMER
+++ /dev/null
@@ -1 +0,0 @@
-Apache Airflow is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
diff --git a/INSTALL b/INSTALL
index 5c8f03eb663e6..c3d15a23eaa08 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,13 +1,22 @@
-# INSTALL / BUILD instruction for Apache Airflow (incubating)
-# fetch the tarball and untar the source
+# INSTALL / BUILD instructions for Apache Airflow
+
+# [required] fetch the tarball and untar the source
+# change into the directory that was untarred.
 
 # [optional] run Apache RAT (release audit tool) to validate license headers
-# RAT docs here: https://creadur.apache.org/rat/
+# RAT docs here: https://creadur.apache.org/rat/. Requires Java and Apache Rat
 java -jar apache-rat.jar -E ./.rat-excludes -d .
 
-# [optional] by default one of Apache Airflow's dependencies pulls in a GPL
-# library. If this is a concern issue (also every upgrade):
-# export SLUGIFY_USES_TEXT_UNIDECODE=yes
+# [optional] Airflow pulls in quite a lot of dependencies in order
+# to connect to other services. You might want to test or run Airflow
+# from a virtual env to make sure those dependencies are separated
+# from your system wide versions
+python -m my_env
+source my_env/bin/activate
+
+# [required] building and installing
+# by pip (preferred)
+pip install .
 
-# install the release
+# or directly
 python setup.py install
diff --git a/LICENSE b/LICENSE
index 405540c64fb87..e3335acb8014a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -209,6 +209,19 @@ limitations under the License.
    licenses.
 
 
+========================================================================
+Third party Apache 2.0 licenses
+========================================================================
+
+The following components are provided under the Apache 2.0 License.
+See project link for details. The text of each license is also included
+at licenses/LICENSE-[project].txt.
+
+    (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
+    (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
+    (ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
+    (ALv2 License) airflow.contrib.auth.backends.github_enterprise_auth
+
 ========================================================================
 MIT licenses
 ========================================================================
@@ -216,16 +229,19 @@ MIT licenses
 The following components are provided under the MIT License. See project link for details.
 The text of each license is also included at licenses/LICENSE-[project].txt.
 
-    (MIT License) jquery (https://jquery.org/license/)
-    (MIT License) dagre-d3 (https://github.com/cpettitt/dagre-d3)
-    (MIT License) bootstrap (https://github.com/twbs/bootstrap/)
-    (MIT License) d3-tip (https://github.com/Caged/d3-tip)
-    (MIT License) dataTables (https://datatables.net)
-    (MIT License) WebGL-2D (https://github.com/gameclosure/webgl-2d)
-    (MIT License) Underscorejs (http://underscorejs.org)
-    (MIT License) Bootstrap Toggle (http://www.bootstraptoggle.com)
-    (MIT License) normalize.css (http://necolas.github.io/normalize.css/)
-    (MIT License) ElasticMock (https://github.com/vrcmarcos/elasticmock)
+    (MIT License) jquery v2.1.4 (https://jquery.org/license/)
+    (MIT License) dagre-d3 v0.6.1 (https://github.com/cpettitt/dagre-d3)
+    (MIT License) bootstrap v3.2 (https://github.com/twbs/bootstrap/)
+    (MIT License) d3-tip v0.6.3 (https://github.com/Caged/d3-tip)
+    (MIT License) dataTables v1.10.10 (https://datatables.net)
+    (MIT License) WebGL-2D (git-commit 9a7ec26) (https://github.com/gameclosure/webgl-2d)
+    (MIT License) Underscorejs v1.5.0 (http://underscorejs.org)
+    (MIT License) Bootstrap Toggle v2.2.0 (http://www.bootstraptoggle.com)
+    (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
+    (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
+    (MIT License) MomentJS v2.22.2 (http://momentjs.com/)
+    (MIT License) python-slugify v2.0.1 (https://github.com/un33k/python-slugify)
+    (MIT License) python-nvd3 v0.15.0 (https://github.com/areski/python-nvd3)
 
 ========================================================================
 BSD 2-Clause licenses
@@ -234,15 +250,16 @@ The following components are provided under the BSD 2-Clause license.
 See file headers and project links for details.
 The text of each license is also included at licenses/LICENSE-[project].txt.
 
-    (BSD 2 License) flask-kerberos (https://github.com/mkomitee/flask-kerberos)
+    (BSD 2 License) flask-kerberos v1.0.4 (https://github.com/mkomitee/flask-kerberos)
 
 ========================================================================
 BSD 3-Clause licenses
 ========================================================================
-The following components are provided under the BSD 2-Clause license.
-See file headers and project links for details.
+The following components are provided under the BSD 3-Clause license. See project links for details.
 The text of each license is also included at licenses/LICENSE-[project].txt.
 
-    (BSD 3 License) Ace (https://github.com/ajaxorg/ace)
-    (BSD 3 License) d3js (https://d3js.org)
-    (BSD 3 License) parallel-coordinates (http://syntagmatic.github.com/parallel-coordinates/)
+    (BSD 3 License) Ace v1.1.8 (https://github.com/ajaxorg/ace)
+    (BSD 3 License) d3js v3.5.17 (https://d3js.org)
+    (BSD 3 License) parallel-coordinates v0.7.0 (http://syntagmatic.github.com/parallel-coordinates/)
+    (BSD 3 License) scikit-learn v0.19.1 (https://github.com/scikit-learn/scikit-learn)
+
diff --git a/MANIFEST.in b/MANIFEST.in
index c04b80617d18d..5f0921dabf979 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,15 +18,19 @@
 
 include NOTICE
 include LICENSE
-include DISCLAIMER
 include CHANGELOG.txt
 include README.md
+graft licenses/
 graft airflow/www/templates
 graft airflow/www/static
+graft airflow/www_rbac
 graft airflow/www_rbac/static
 graft airflow/www_rbac/templates
 graft airflow/www_rbac/translations
+graft airflow/_vendor/
 include airflow/alembic.ini
 graft scripts/systemd
 graft scripts/upstart
 graft airflow/config_templates
+recursive-exclude airflow/www_rbac/node_modules *
+global-exclude __pycache__  *.pyc
diff --git a/NOTICE b/NOTICE
index b1e78ad9c3380..2e6202e30e809 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,6 +1,38 @@
 Apache Airflow
-Copyright 2016 and onwards The Apache Software Foundation
-
+Copyright 2016-2019 The Apache Software Foundation
 
 This product includes software developed at The Apache Software
 Foundation (http://www.apache.org/).
+
+=======================================================================
+
+Apache Airflow contains subcomponents with separate copyright notices and
+license terms. Your use of the source code for the these subcomponents
+is subject to the terms and conditions of their respective licenses.
+
+See the LICENSE file for a list of subcomponents and dependencies and
+their respective licenses.
+
+airflow.contrib.auth.backends.github_enterprise_auth:
+-----------------------------------------------------
+
+* Copyright 2015 Matthew Pelland (matt@pelland.io)
+
+hue:
+-----
+This product contains a modified portion of 'Hue' developed by Cloudera, Inc.
+(https://github.com/cloudera/hue/).
+
+* Copyright 2009-2017 Cloudera Inc.
+
+python-slugify:
+---------------
+
+* Copyright (c) Val Neekman @ Neekware Inc. http://neekware.com
+
+python-nvd3:
+------------
+
+* Copyright (c) 2013 Arezqui Belaid <areski@gmail.com> and other contributors
+
+
diff --git a/README.md b/README.md
index e68d26cc8cfa2..fceb2a7d7bffb 100644
--- a/README.md
+++ b/README.md
@@ -1,29 +1,43 @@
-# Airflow
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache Airflow
 
 [![PyPI version](https://badge.fury.io/py/apache-airflow.svg)](https://badge.fury.io/py/apache-airflow)
-[![Build Status](https://travis-ci.org/apache/incubator-airflow.svg?branch=master)](https://travis-ci.org/apache/incubator-airflow)
-[![Coverage Status](https://img.shields.io/codecov/c/github/apache/incubator-airflow/master.svg)](https://codecov.io/github/apache/incubator-airflow?branch=master)
+[![Build Status](https://travis-ci.org/apache/airflow.svg?branch=master)](https://travis-ci.org/apache/airflow)
+[![Coverage Status](https://img.shields.io/codecov/c/github/apache/airflow/master.svg)](https://codecov.io/github/apache/airflow?branch=master)
 [![Documentation Status](https://readthedocs.org/projects/airflow/badge/?version=latest)](https://airflow.readthedocs.io/en/latest/?badge=latest)
 [![License](http://img.shields.io/:license-Apache%202-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.txt)
-[![Join the chat at https://gitter.im/apache/incubator-airflow](https://badges.gitter.im/apache/incubator-airflow.svg)](https://gitter.im/apache/incubator-airflow?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/apache-airflow.svg)](https://pypi.org/project/apache-airflow/)
+[![Twitter Follow](https://img.shields.io/twitter/follow/ApacheAirflow.svg?style=social&label=Follow)](https://twitter.com/ApacheAirflow)
 
-_NOTE: The transition from 1.8.0 (or before) to 1.8.1 (or after) requires uninstalling Airflow before installing the new version. The package name was changed from `airflow` to `apache-airflow` as of version 1.8.1._
+_NOTE: The transition from 1.8.0 (or before) to 1.8.1 (or after) requires uninstalling Apache Airflow before installing the new version. The package name was changed from `airflow` to `apache-airflow` as of version 1.8.1._
 
-Airflow is a platform to programmatically author, schedule, and monitor
-workflows.
+Apache Airflow (or simply Airflow) is a platform to programmatically author, schedule, and monitor workflows.
 
 When workflows are defined as code, they become more maintainable,
 versionable, testable, and collaborative.
 
-Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks.
-The Airflow scheduler executes your tasks on an array of workers while
-following the specified dependencies. Rich command line utilities make
-performing complex surgeries on DAGs a snap. The rich user interface
-makes it easy to visualize pipelines running in production,
-monitor progress, and troubleshoot issues when needed.
+Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks. The Airflow scheduler executes your tasks on an array of workers while following the specified dependencies. Rich command line utilities make performing complex surgeries on DAGs a snap. The rich user interface makes it easy to visualize pipelines running in production, monitor progress, and troubleshoot issues when needed.
 
 ## Getting started
-Please visit the Airflow Platform documentation (latest **stable** release) for help with [installing Airflow](https://airflow.incubator.apache.org/installation.html), getting a [quick start](https://airflow.incubator.apache.org/start.html), or a more complete [tutorial](https://airflow.incubator.apache.org/tutorial.html).
+Please visit the Airflow Platform documentation (latest **stable** release) for help with [installing Airflow](https://airflow.apache.org/installation.html), getting a [quick start](https://airflow.apache.org/start.html), or a more complete [tutorial](https://airflow.apache.org/tutorial.html).
 
 Documentation of GitHub master (latest development branch): [ReadTheDocs Documentation](https://airflow.readthedocs.io/en/latest/)
 
@@ -49,7 +63,7 @@ unit of work and continuity.
 - **Dynamic**:  Airflow pipelines are configuration as code (Python), allowing for dynamic pipeline generation. This allows for writing code that instantiates pipelines dynamically.
 - **Extensible**:  Easily define your own operators, executors and extend the library so that it fits the level of abstraction that suits your environment.
 - **Elegant**:  Airflow pipelines are lean and explicit. Parameterizing your scripts is built into the core of Airflow using the powerful **Jinja** templating engine.
-- **Scalable**:  Airflow has a modular architecture and uses a message queue to orchestrate an arbitrary number of workers. Airflow is ready to scale to infinity.
+- **Scalable**:  Airflow has a modular architecture and uses a message queue to orchestrate an arbitrary number of workers.
 
 ## User Interface
 
@@ -71,9 +85,14 @@ unit of work and continuity.
 - **Code View**:  Quick way to view source code of a DAG.
 ![](/docs/img/code.png)
 
+## Contributing
+
+Want to help build Apache Airflow? Check out our [contributing documentation](https://github.com/apache/airflow/blob/master/CONTRIBUTING.md).
+
+
 ## Who uses Airflow?
 
-As the Airflow community grows, we'd like to keep track of who is using
+As the Apache Airflow community grows, we'd like to keep track of who is using
 the platform. Please send a PR with your company name and @githubhandle
 if you may.
 
@@ -233,6 +252,7 @@ Currently **officially** using Airflow:
 1. [Tile](https://tile.com/) [[@ranjanmanish](https://github.com/ranjanmanish)]
 1. [Tokopedia](https://www.tokopedia.com/) [@topedmaria](https://github.com/topedmaria)
 1. [Twine Labs](https://www.twinelabs.com/) [[@ivorpeles](https://github.com/ivorpeles)]
+1. [Twitter](https://www.twitter.com/) [[@aoen](https://github.com/aoen)]
 1. [T2 Systems](http://t2systems.com) [[@unclaimedpants](https://github.com/unclaimedpants)]
 1. [Ubisoft](https://www.ubisoft.com/) [[@Walkoss](https://github.com/Walkoss)]
 1. [United Airlines](https://www.united.com/) [[@ilopezfr](https://github.com/ilopezfr)]
@@ -256,10 +276,17 @@ Currently **officially** using Airflow:
 1. [Zymergen](https://www.zymergen.com/)
 1. [99](https://99taxis.com) [[@fbenevides](https://github.com/fbenevides), [@gustavoamigo](https://github.com/gustavoamigo) & [@mmmaia](https://github.com/mmmaia)]
 
+## Who Maintains Apache Airflow?
+
+Airflow is the work of the [community](https://github.com/apache/airflow/graphs/contributors),
+but the [core committers/maintainers](https://people.apache.org/committers-by-project.html#airflow)
+are responsible for reviewing and merging PRs as well as steering conversation around new feature requests.
+If you would like to become a maintainer, please review the Apache Airflow
+[committer requirements](https://cwiki.apache.org/confluence/display/AIRFLOW/Committers).
+
 ## Links
 
 
-* [Documentation](https://airflow.incubator.apache.org/)
-* [Chat](https://gitter.im/apache/incubator-airflow)
-* [Apache Airflow Incubation Status](http://incubator.apache.org/projects/airflow.html)
-* [More](https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Links)
+- [Documentation](https://airflow.apache.org/)
+- [Chat](https://apache-airflow-slack.herokuapp.com/)
+- [More](https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Links)
diff --git a/TODO.md b/TODO.md
index cf19035e1b995..1e4e6ed249e1a 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,3 +1,22 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 #### Roadmap items
 * UI page answering "Why isn't this task instance running?"
 * Attempt removing DagBag caching for the web server
diff --git a/UPDATING.md b/UPDATING.md
index 3a66e735c3ac4..bf48ee1a708f3 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -1,11 +1,334 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Updating Airflow
 
 This file documents any backwards-incompatible changes in Airflow and
 assists users migrating to a new version.
 
-## Airflow Master
+## Airflow 1.10.3
+
+### RedisPy dependency updated to v3 series
+If you are using the Redis Sensor or Hook you may have to update your code. See
+[redis-py porting instructions] to check if your code might be affected (MSET,
+MSETNX, ZADD, and ZINCRBY all were, but read the full doc).
+
+[redis-py porting instructions]: https://github.com/andymccurdy/redis-py/tree/3.2.0#upgrading-from-redis-py-2x-to-30
+
+### SLUGIFY_USES_TEXT_UNIDECODE or AIRFLOW_GPL_UNIDECODE no longer required
+
+It is no longer required to set one of the environment variables to avoid
+a GPL dependency. Airflow will now always use text-unidecode if unidecode
+was not installed before.
+
+### new `sync_parallelism` config option in celery section
+
+The new `sync_parallelism` config option will control how many processes CeleryExecutor will use to
+fetch celery task state in parallel. Default value is max(1, number of cores - 1)
+
+### Rename of BashTaskRunner to StandardTaskRunner
+
+BashTaskRunner has been renamed to StandardTaskRunner. It is the default task runner
+so you might need to update your config.
+
+`task_runner = StandardTaskRunner`
+
+### Modification to config file discovery
+
+If the `AIRFLOW_CONFIG` environment variable was not set and the
+`~/airflow/airflow.cfg` file existed, airflow previously used
+`~/airflow/airflow.cfg` instead of `$AIRFLOW_HOME/airflow.cfg`. Now airflow
+will discover its config file using the `$AIRFLOW_CONFIG` and `$AIRFLOW_HOME`
+environment variables rather than checking for the presence of a file.
+
+### New `dag_discovery_safe_mode` config option
+
+If `dag_discovery_safe_mode` is enabled, only check files for DAGs if
+they contain the strings "airflow" and "DAG". For backwards
+compatibility, this option is enabled by default.
+
+### Changes in Google Cloud Platform related operators
+
+Most GCP-related operators have now optional `PROJECT_ID` parameter. In case you do not specify it,
+the project id configured in
+[GCP Connection](https://airflow.apache.org/howto/manage-connections.html#connection-type-gcp) is used.
+There will be an `AirflowException` thrown in case `PROJECT_ID` parameter is not specified and the
+connection used has no project id defined. This change should be  backwards compatible as earlier version
+of the operators had `PROJECT_ID` mandatory.
+
+Operators involved:
+
+  * GCP Compute Operators
+    * GceInstanceStartOperator
+    * GceInstanceStopOperator
+    * GceSetMachineTypeOperator
+  * GCP Function Operators
+    * GcfFunctionDeployOperator
+  * GCP Cloud SQL Operators
+    * CloudSqlInstanceCreateOperator
+    * CloudSqlInstancePatchOperator
+    * CloudSqlInstanceDeleteOperator
+    * CloudSqlInstanceDatabaseCreateOperator
+    * CloudSqlInstanceDatabasePatchOperator
+    * CloudSqlInstanceDatabaseDeleteOperator
+
+Other GCP operators are unaffected.
+
+### Changes in Google Cloud Platform related hooks
+
+The change in GCP operators implies that GCP Hooks for those operators require now keyword parameters rather
+than positional ones in all methods where `project_id` is used. The methods throw an explanatory exception
+in case they are called using positional parameters.
+
+Hooks involved:
+
+  * GceHook
+  * GcfHook
+  * CloudSqlHook
+
+Other GCP hooks are unaffected.
+
+### Changed behaviour of using default value when accessing variables
+It's now possible to use `None` as a default value with the `default_var` parameter when getting a variable, e.g.
+
+```python
+foo = Variable.get("foo", default_var=None)
+if foo is None:
+    handle_missing_foo()
+```
+
+(Note: there is already `Variable.setdefault()` which me be helpful in some cases.)
+
+This changes the behaviour if you previously explicitly provided `None` as a default value. If your code expects a `KeyError` to be thrown, then don't pass the `default_var` argument.
+
+### Removal of `airflow_home` config setting
+
+There were previously two ways of specifying the Airflow "home" directory
+(`~/airflow` by default): the `AIRFLOW_HOME` environment variable, and the
+`airflow_home` config setting in the `[core]` section.
+
+If they had two different values different parts of the code base would end up
+with different values. The config setting has been deprecated, and you should
+remove the value from the config file and set `AIRFLOW_HOME` environment
+variable if you need to use a non default value for this.
+
+(Since this setting is used to calculate what config file to load, it is not
+possible to keep just the config option)
+
+### Change of two methods signatures in `GCPTransferServiceHook`
+
+The signature of the `create_transfer_job` method in `GCPTransferServiceHook`
+class has changed. The change does not change the behavior of the method.
+
+Old signature:
+```python
+def create_transfer_job(self, description, schedule, transfer_spec, project_id=None):
+```
+New signature:
+```python
+def create_transfer_job(self, body):
+```
+
+It is necessary to rewrite calls to method. The new call looks like this:
+```python
+body = {
+  'status': 'ENABLED',
+  'projectId': project_id,
+  'description': description,
+  'transferSpec': transfer_spec,
+  'schedule': schedule,
+}
+gct_hook.create_transfer_job(body)
+```
+The change results from the unification of all hooks and adjust to
+[the official recommendations](https://lists.apache.org/thread.html/e8534d82be611ae7bcb21ba371546a4278aad117d5e50361fd8f14fe@%3Cdev.airflow.apache.org%3E)
+for the Google Cloud Platform.
+
+The signature of `wait_for_transfer_job` method in `GCPTransferServiceHook` has changed.
+
+Old signature:
+```python
+def wait_for_transfer_job(self, job):
+```
+New signature:
+```python
+def wait_for_transfer_job(self, job, expected_statuses=(GcpTransferOperationStatus.SUCCESS, )):
+```
+
+The behavior of `wait_for_transfer_job` has changed:
+
+Old behavior:
+
+`wait_for_transfer_job` would wait for the SUCCESS status in specified jobs operations.
+
+New behavior:
+
+You can now specify an array of expected statuses. `wait_for_transfer_job` now waits for any of them.
+
+The default value of `expected_statuses` is SUCCESS so that change is backwards compatible.
+
+### Moved two classes to different modules
+
+The class `GoogleCloudStorageToGoogleCloudStorageTransferOperator` has been moved from
+`airflow.contrib.operators.gcs_to_gcs_transfer_operator` to `airflow.contrib.operators.gcp_transfer_operator`
+
+the class `S3ToGoogleCloudStorageTransferOperator` has been moved from
+`airflow.contrib.operators.s3_to_gcs_transfer_operator` to `airflow.contrib.operators.gcp_transfer_operator`
+
+The change was made to keep all the operators related to GCS Transfer Services in one file.
+
+The previous imports will continue to work until Airflow 2.0
+
+### Fixed typo in --driver-class-path in SparkSubmitHook
+
+The `driver_classapth` argument  to SparkSubmit Hook and Operator was
+generating `--driver-classpath` on the spark command line, but this isn't a
+valid option to spark.
+
+The argument has been renamed to `driver_class_path`  and  the option it
+generates has been fixed.
+
+## Airflow 1.10.2
+
+### DAG level Access Control for new RBAC UI
+
+Extend and enhance new Airflow RBAC UI to support DAG level ACL. Each dag now has two permissions(one for write, one for read) associated('can_dag_edit', 'can_dag_read').
+The admin will create new role, associate the dag permission with the target dag and assign that role to users. That user can only access / view the certain dags on the UI
+that he has permissions on. If a new role wants to access all the dags, the admin could associate dag permissions on an artificial view(``all_dags``) with that role.
+
+We also provide a new cli command(``sync_perm``) to allow admin to auto sync permissions.
+
+### Modification to `ts_nodash` macro
+`ts_nodash` previously contained TimeZone information along with execution date. For Example: `20150101T000000+0000`. This is not user-friendly for file or folder names which was a popular use case for `ts_nodash`. Hence this behavior has been changed and using `ts_nodash` will no longer contain TimeZone information, restoring the pre-1.10 behavior of this macro. And a new macro `ts_nodash_with_tz` has been added which can be used to get a string with execution date and timezone info without dashes.
+
+Examples:
+  * `ts_nodash`: `20150101T000000`
+  * `ts_nodash_with_tz`: `20150101T000000+0000`
+
+### Semantics of next_ds/prev_ds changed for manually triggered runs
+
+next_ds/prev_ds now map to execution_date instead of the next/previous schedule-aligned execution date for DAGs triggered in the UI.
+
+### User model changes
+This patch changes the `User.superuser` field from a hardcoded boolean to a `Boolean()` database column. `User.superuser` will default to `False`, which means that this privilege will have to be granted manually to any users that may require it.
+
+For example, open a Python shell and
+```python
+from airflow import models, settings
+
+session = settings.Session()
+users = session.query(models.User).all()  # [admin, regular_user]
+
+users[1].superuser  # False
+
+admin = users[0]
+admin.superuser = True
+session.add(admin)
+session.commit()
+```
+
+### Custom auth backends interface change
+
+We have updated the version of flask-login we depend upon, and as a result any
+custom auth backends might need a small change: `is_active`,
+`is_authenticated`, and `is_anonymous` should now be properties. What this means is if
+previously you had this in your user class
+
+    def is_active(self):
+      return self.active
+
+then you need to change it like this
+
+    @property
+    def is_active(self):
+      return self.active
+
+## Airflow 1.10.1
+
+### New `dag_processor_manager_log_location` config option
+
+The DAG parsing manager log now by default will be log into a file, where its location is
+controlled by the new `dag_processor_manager_log_location` config option in core section.
+
+### StatsD Metrics
+
+The `scheduler_heartbeat` metric has been changed from a gauge to a counter. Each loop of the scheduler will increment the counter by 1. This provides a higher degree of visibility and allows for better integration with Prometheus using the [StatsD Exporter](https://github.com/prometheus/statsd_exporter). Scheduler upness can be determined by graphing and alerting using a rate. If the scheduler goes down, the rate will drop to 0.
+
+### EMRHook now passes all of connection's extra to CreateJobFlow API
+
+EMRHook.create_job_flow has been changed to pass all keys to the create_job_flow API, rather than
+just specific known keys for greater flexibility.
+
+However prior to this release the "emr_default" sample connection that was created had invalid
+configuration, so creating EMR clusters might fail until your connection is updated. (Ec2KeyName,
+Ec2SubnetId, TerminationProtection and KeepJobFlowAliveWhenNoSteps were all top-level keys when they
+should be inside the "Instances" dict)
+
+### LDAP Auth Backend now requires TLS
+
+Connecting to an LDAP serever over plain text is not supported anymore. The
+certificate presented by the LDAP server must be signed by a trusted
+certificiate, or you must provide the `cacert` option under `[ldap]` in the
+config file.
+
+If you want to use LDAP auth backend without TLS then you will habe to create a
+custom-auth backend based on
+https://github.com/apache/airflow/blob/1.10.0/airflow/contrib/auth/backends/ldap_auth.py
+
+### Custom auth backends interface change
+
+We have updated the version of flask-login we depend upon, and as a result any
+custom auth backends might need a small change: `is_active`,
+`is_authenticated`, and `is_anonymous` should now be properties. What this means is if
+previously you had this in your user class
+
+    def is_active(self):
+      return self.active
+
+then you need to change it like this
+
+    @property
+    def is_active(self):
+      return self.active
+
+## Airflow 1.10
+
+Installation and upgrading requires setting `SLUGIFY_USES_TEXT_UNIDECODE=yes` in your environment or
+`AIRFLOW_GPL_UNIDECODE=yes`. In case of the latter a GPL runtime dependency will be installed due to a
+dependency (python-nvd3 -> python-slugify -> unidecode).
+
+### Replace DataProcHook.await calls to DataProcHook.wait
+
+The method name was changed to be compatible with the Python 3.7 async/await keywords
+
+### DAG level Access Control for new RBAC UI
+
+Extend and enhance new Airflow RBAC UI to support DAG level ACL. Each dag now has two permissions(one for write, one for read) associated('can_dag_edit', 'can_dag_read').
+The admin will create new role, associate the dag permission with the target dag and assign that role to users. That user can only access / view the certain dags on the UI
+that he has permissions on. If a new role wants to access all the dags, the admin could associate dag permissions on an artificial view(``all_dags``) with that role.
+
+We also provide a new cli command(``sync_perm``) to allow admin to auto sync permissions.
+
+### Setting UTF-8 as default mime_charset in email utils
 
 ### Add a configuration variable(default_dag_run_display_number) to control numbers of dag run for display
+
 Add a configuration variable(default_dag_run_display_number) under webserver section to control num of dag run to show in UI.
 
 ### Default executor for SubDagOperator is changed to SequentialExecutor
@@ -35,11 +358,13 @@ Run `airflow webserver` to start the new UI. This will bring up a log in page, e
 There are five roles created for Airflow by default: Admin, User, Op, Viewer, and Public. To configure roles/permissions, go to the `Security` tab and click `List Roles` in the new UI.
 
 #### Breaking changes
+
 - AWS Batch Operator renamed property queue to job_queue to prevent conflict with the internal queue from CeleryExecutor - AIRFLOW-2542
 - Users created and stored in the old users table will not be migrated automatically. FAB's built-in authentication support must be reconfigured.
 - Airflow dag home page is now `/home` (instead of `/admin`).
 - All ModelViews in Flask-AppBuilder follow a different pattern from Flask-Admin. The `/admin` part of the url path will no longer exist. For example: `/admin/connection` becomes `/connection/list`, `/admin/connection/new` becomes `/connection/add`, `/admin/connection/edit` becomes `/connection/edit`, etc.
 - Due to security concerns, the new webserver will no longer support the features in the `Data Profiling` menu of old UI, including `Ad Hoc Query`, `Charts`, and `Known Events`.
+- HiveServer2Hook.get_results() always returns a list of tuples, even when a single column is queried, as per Python API 2.
 
 ### airflow.contrib.sensors.hdfs_sensors renamed to airflow.contrib.sensors.hdfs_sensor
 
@@ -53,33 +378,43 @@ to have specified `explicit_defaults_for_timestamp=1` in your my.cnf under `[mys
 ### Celery config
 
 To make the config of Airflow compatible with Celery, some properties have been renamed:
+
 ```
 celeryd_concurrency -> worker_concurrency
 celery_result_backend -> result_backend
+celery_ssl_active -> ssl_active
+celery_ssl_cert -> ssl_cert
+celery_ssl_key -> ssl_key
 ```
+
 Resulting in the same config parameters as Celery 4, with more transparency.
 
 ### GCP Dataflow Operators
+
 Dataflow job labeling is now supported in Dataflow{Java,Python}Operator with a default
 "airflow-version" label, please upgrade your google-cloud-dataflow or apache-beam version
 to 2.2.0 or greater.
 
 ### BigQuery Hooks and Operator
+
 The `bql` parameter passed to `BigQueryOperator` and `BigQueryBaseCursor.run_query` has been deprecated and renamed to `sql` for consistency purposes. Using `bql` will still work (and raise a `DeprecationWarning`), but is no longer
 supported and will be removed entirely in Airflow 2.0
 
 ### Redshift to S3 Operator
+
 With Airflow 1.9 or lower, Unload operation always included header row. In order to include header row,
 we need to turn off parallel unload. It is preferred to perform unload operation using all nodes so that it is
 faster for larger tables. So, parameter called `include_header` is added and default is set to False.
-Header row will be added only if this parameter is set True and also in that case parallel will be automatically turned off (`PARALLEL OFF`)  
+Header row will be added only if this parameter is set True and also in that case parallel will be automatically turned off (`PARALLEL OFF`)
 
 ### Google cloud connection string
 
 With Airflow 1.9 or lower, there were two connection strings for the Google Cloud operators, both `google_cloud_storage_default` and `google_cloud_default`. This can be confusing and therefore the `google_cloud_storage_default` connection id has been replaced with `google_cloud_default` to make the connection id consistent across Airflow.
 
 ### Logging Configuration
+
 With Airflow 1.9 or lower, `FILENAME_TEMPLATE`, `PROCESSOR_FILENAME_TEMPLATE`, `LOG_ID_TEMPLATE`, `END_OF_LOG_MARK` were configured in `airflow_local_settings.py`. These have been moved into the configuration file, and hence if you were using a custom configuration file the following defaults need to be added.
+
 ```
 [core]
 fab_logging_level = WARN
@@ -91,23 +426,36 @@ elasticsearch_log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_
 elasticsearch_end_of_log_mark = end_of_log
 ```
 
+The previous setting of `log_task_reader` is not needed in many cases now when using the default logging config with remote storages. (Previously it needed to be set to `s3.task` or similar. This is not needed with the default config anymore)
+
+#### Change of per-task log path
+
+With the change to Airflow core to be timezone aware the default log path for task instances will now include timezone information. This will by default mean all previous task logs won't be found. You can get the old behaviour back by setting the following config options:
+
+```
+[core]
+log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ execution_date.strftime("%%Y-%%m-%%dT%%H:%%M:%%S") }}/{{ try_number }}.log
+```
+
 ## Airflow 1.9
 
 ### SSH Hook updates, along with new SSH Operator & SFTP Operator
 
 SSH Hook now uses the Paramiko library to create an ssh client connection, instead of the sub-process based ssh command execution previously (<1.9.0), so this is backward incompatible.
-  - update SSHHook constructor
-  - use SSHOperator class in place of SSHExecuteOperator which is removed now. Refer to test_ssh_operator.py for usage info.
-  - SFTPOperator is added to perform secure file transfer from serverA to serverB. Refer to test_sftp_operator.py.py for usage info.
-  - No updates are required if you are using ftpHook, it will continue to work as is.
+
+- update SSHHook constructor
+- use SSHOperator class in place of SSHExecuteOperator which is removed now. Refer to test_ssh_operator.py for usage info.
+- SFTPOperator is added to perform secure file transfer from serverA to serverB. Refer to test_sftp_operator.py.py for usage info.
+- No updates are required if you are using ftpHook, it will continue to work as is.
 
 ### S3Hook switched to use Boto3
 
 The airflow.hooks.S3_hook.S3Hook has been switched to use boto3 instead of the older boto (a.k.a. boto2). This results in a few backwards incompatible changes to the following classes: S3Hook:
-  - the constructors no longer accepts `s3_conn_id`. It is now called `aws_conn_id`.
-  - the default connection is now "aws_default" instead of "s3_default"
-  - the return type of objects returned by `get_bucket` is now boto3.s3.Bucket
-  - the return type of `get_key`, and `get_wildcard_key` is now an boto3.S3.Object.
+
+- the constructors no longer accepts `s3_conn_id`. It is now called `aws_conn_id`.
+- the default connection is now "aws_default" instead of "s3_default"
+- the return type of objects returned by `get_bucket` is now boto3.s3.Bucket
+- the return type of `get_key`, and `get_wildcard_key` is now an boto3.S3.Object.
 
 If you are using any of these in your DAGs and specify a connection ID you will need to update the parameter name for the connection to "aws_conn_id": S3ToHiveTransfer, S3PrefixSensor, S3KeySensor, RedshiftToS3Transfer.
 
@@ -145,17 +493,22 @@ The config can be taken from `airflow/config_templates/airflow_local_settings.py
 ```
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 import os
 
@@ -280,13 +633,14 @@ The `file_task_handler` logger has been made more flexible. The default format c
 
 #### I'm using S3Log or GCSLogs, what do I do!?
 
-If you are logging to Google cloud storage, please see the [Google cloud platform documentation](https://airflow.incubator.apache.org/integration.html#gcp-google-cloud-platform) for logging instructions.
+If you are logging to Google cloud storage, please see the [Google cloud platform documentation](https://airflow.apache.org/integration.html#gcp-google-cloud-platform) for logging instructions.
 
 If you are using S3, the instructions should be largely the same as the Google cloud platform instructions above. You will need a custom logging config. The `REMOTE_BASE_LOG_FOLDER` configuration key in your airflow config has been removed, therefore you will need to take the following steps:
- - Copy the logging configuration from [`airflow/config_templates/airflow_logging_settings.py`](https://github.com/apache/incubator-airflow/blob/master/airflow/config_templates/airflow_local_settings.py).
- - Place it in a directory inside the Python import path `PYTHONPATH`. If you are using Python 2.7, ensuring that any `__init__.py` files exist so that it is importable.
- - Update the config by setting the path of `REMOTE_BASE_LOG_FOLDER` explicitly in the config. The `REMOTE_BASE_LOG_FOLDER` key is not used anymore.
- - Set the `logging_config_class` to the filename and dict. For example, if you place `custom_logging_config.py` on the base of your pythonpath, you will need to set `logging_config_class = custom_logging_config.LOGGING_CONFIG` in your config as Airflow 1.8.
+
+- Copy the logging configuration from [`airflow/config_templates/airflow_logging_settings.py`](https://github.com/apache/airflow/blob/master/airflow/config_templates/airflow_local_settings.py).
+- Place it in a directory inside the Python import path `PYTHONPATH`. If you are using Python 2.7, ensuring that any `__init__.py` files exist so that it is importable.
+- Update the config by setting the path of `REMOTE_BASE_LOG_FOLDER` explicitly in the config. The `REMOTE_BASE_LOG_FOLDER` key is not used anymore.
+- Set the `logging_config_class` to the filename and dict. For example, if you place `custom_logging_config.py` on the base of your pythonpath, you will need to set `logging_config_class = custom_logging_config.LOGGING_CONFIG` in your config as Airflow 1.8.
 
 ### New Features
 
@@ -295,8 +649,10 @@ If you are using S3, the instructions should be largely the same as the Google c
 A new DaskExecutor allows Airflow tasks to be run in Dask Distributed clusters.
 
 ### Deprecated Features
+
 These features are marked for deprecation. They may still work (and raise a `DeprecationWarning`), but are no longer
 supported and will be removed entirely in Airflow 2.0
+
 - If you're using the `google_cloud_conn_id` or `dataproc_cluster` argument names explicitly in `contrib.operators.Dataproc{*}Operator`(s), be sure to rename them to `gcp_conn_id` or `cluster_name`, respectively. We've renamed these arguments for consistency. (AIRFLOW-1323)
 
 - `post_execute()` hooks now take two arguments, `context` and `result`
@@ -320,30 +676,36 @@ a previously installed version of Airflow before installing 1.8.1.
 ## Airflow 1.8
 
 ### Database
+
 The database schema needs to be upgraded. Make sure to shutdown Airflow and make a backup of your database. To
 upgrade the schema issue `airflow upgradedb`.
 
 ### Upgrade systemd unit files
+
 Systemd unit files have been updated. If you use systemd please make sure to update these.
 
 > Please note that the webserver does not detach properly, this will be fixed in a future version.
 
 ### Tasks not starting although dependencies are met due to stricter pool checking
+
 Airflow 1.7.1 has issues with being able to over subscribe to a pool, ie. more slots could be used than were
 available. This is fixed in Airflow 1.8.0, but due to past issue jobs may fail to start although their
 dependencies are met after an upgrade. To workaround either temporarily increase the amount of slots above
 the amount of queued tasks or use a new pool.
 
 ### Less forgiving scheduler on dynamic start_date
+
 Using a dynamic start_date (e.g. `start_date = datetime.now()`) is not considered a best practice. The 1.8.0 scheduler
 is less forgiving in this area. If you encounter DAGs not being scheduled you can try using a fixed start_date and
 renaming your DAG. The last step is required to make sure you start with a clean slate, otherwise the old schedule can
 interfere.
 
 ### New and updated scheduler options
+
 Please read through the new scheduler options, defaults have changed since 1.7.1.
 
 #### child_process_log_directory
+
 In order to increase the robustness of the scheduler, DAGS are now processed in their own process. Therefore each
 DAG has its own log file for the scheduler. These log files are placed in `child_process_log_directory` which defaults to
 `<AIRFLOW_HOME>/scheduler/latest`. You will need to make sure these log files are removed.
@@ -351,24 +713,30 @@ DAG has its own log file for the scheduler. These log files are placed in `child
 > DAG logs or processor logs ignore and command line settings for log file locations.
 
 #### run_duration
+
 Previously the command line option `num_runs` was used to let the scheduler terminate after a certain amount of
 loops. This is now time bound and defaults to `-1`, which means run continuously. See also num_runs.
 
 #### num_runs
+
 Previously `num_runs` was used to let the scheduler terminate after a certain amount of loops. Now num_runs specifies
 the number of times to try to schedule each DAG file within `run_duration` time. Defaults to `-1`, which means try
 indefinitely. This is only available on the command line.
 
 #### min_file_process_interval
+
 After how much time should an updated DAG be picked up from the filesystem.
 
 #### min_file_parsing_loop_time
+CURRENTLY DISABLED DUE TO A BUG
 How many seconds to wait between file-parsing loops to prevent the logs from being spammed.
 
 #### dag_dir_list_interval
+
 The frequency with which the scheduler should relist the contents of the DAG directory. If while developing +dags, they are not being picked up, have a look at this number and decrease it when necessary.
 
 #### catchup_by_default
+
 By default the scheduler will fill any missing interval DAG Runs between the last execution date and the current date.
 This setting changes that behavior to only execute the latest interval. This can also be specified per DAG as
 `catchup = False / True`. Command line backfills will still work.
@@ -399,6 +767,7 @@ required to whitelist these variables by adding the following to your configurat
      <value>airflow\.ctx\..*</value>
 </property>
 ```
+
 ### Google Cloud Operator and Hook alignment
 
 All Google Cloud Operators and Hooks are aligned and use the same client library. Now you have a single connection
@@ -410,6 +779,7 @@ Also the old P12 key file type is not supported anymore and only the new JSON ke
 account.
 
 ### Deprecated Features
+
 These features are marked for deprecation. They may still work (and raise a `DeprecationWarning`), but are no longer
 supported and will be removed entirely in Airflow 2.0
 
@@ -421,11 +791,12 @@ supported and will be removed entirely in Airflow 2.0
 - Operators no longer accept arbitrary arguments
 
   Previously, `Operator.__init__()` accepted any arguments (either positional `*args` or keyword `**kwargs`) without
-  complaint. Now, invalid arguments will be rejected. (https://github.com/apache/incubator-airflow/pull/1285)
+  complaint. Now, invalid arguments will be rejected. (https://github.com/apache/airflow/pull/1285)
 
 - The config value secure_mode will default to True which will disable some insecure endpoints/features
 
 ### Known Issues
+
 There is a report that the default of "-1" for num_runs creates an issue where errors are reported while parsing tasks.
 It was not confirmed, but a workaround was found by changing the default back to `None`.
 
@@ -452,7 +823,9 @@ To continue using the default smtp email backend, change the email_backend line
 [email]
 email_backend = airflow.utils.send_email_smtp
 ```
+
 to:
+
 ```
 [email]
 email_backend = airflow.utils.email.send_email_smtp
@@ -465,7 +838,9 @@ To continue using S3 logging, update your config file so:
 ```
 s3_log_folder = s3://my-airflow-log-bucket/logs
 ```
+
 becomes:
+
 ```
 remote_base_log_folder = s3://my-airflow-log-bucket/logs
 remote_log_conn_id = <your desired s3 connection>
diff --git a/airflow/__init__.py b/airflow/__init__.py
index f40b08aab5e77..4434dc4413e1f 100644
--- a/airflow/__init__.py
+++ b/airflow/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,6 +25,8 @@
 `airflow.www.login`
 """
 from builtins import object
+from typing import Any
+
 from airflow import version
 from airflow.utils.log.logging_mixin import LoggingMixin
 
@@ -32,17 +34,16 @@
 
 import sys
 
-from airflow import configuration as conf
-from airflow import settings
+# flake8: noqa: F401
+from airflow import settings, configuration as conf
 from airflow.models import DAG
 from flask_admin import BaseView
 from importlib import import_module
 from airflow.exceptions import AirflowException
 
-if settings.DAGS_FOLDER not in sys.path:
-    sys.path.append(settings.DAGS_FOLDER)
+settings.initialize()
 
-login = None
+login = None  # type: Any
 
 
 def load_login():
@@ -80,11 +81,12 @@ class AirflowMacroPlugin(object):
     def __init__(self, namespace):
         self.namespace = namespace
 
-from airflow import operators
+
+from airflow import operators  # noqa: E402
 from airflow import sensors  # noqa: E402
-from airflow import hooks
-from airflow import executors
-from airflow import macros
+from airflow import hooks  # noqa: E402
+from airflow import executors  # noqa: E402
+from airflow import macros  # noqa: E402
 
 operators._integrate_plugins()
 sensors._integrate_plugins()  # noqa: E402
diff --git a/airflow/_vendor/README b/airflow/_vendor/README
new file mode 100644
index 0000000000000..a79ea89eae536
--- /dev/null
+++ b/airflow/_vendor/README
@@ -0,0 +1,13 @@
+Original files in this directory were created with the following commands::
+
+    mkdir -p slugify/
+    curl -fsSL -O https://files.pythonhosted.org/packages/1f/9c/8b07d625e9c9df567986d887f0375075abb1923e49d074a7803cd1527dae/python-slugify-2.0.1.tar.gz
+    tar -xzf python-slugify-*.tar.gz --strip-components=2 -C slugify/ '*/slugify/*'
+    tar -xzf python-slugify-*.tar.gz --strip-components=1 -C slugify/ '*/LICENSE'
+    rm *.tar.gz
+
+    mkdir -p nvd3/
+    curl -fsSL -O https://files.pythonhosted.org/packages/0b/aa/97165daa6e319409c5c2582e62736a7353bda3c90d90fdcb0b11e116dd2d/python-nvd3-0.15.0.tar.gz
+    tar -xzf python-nvd3-*.tar.gz --strip-components=2 -C nvd3/ '*/nvd3/*'
+    tar -xzf python-nvd3-*.tar.gz --strip-components=1 -C nvd3/ '*/LICENSE'
+    rm *.tar.gz
diff --git a/.codecov.yml b/airflow/_vendor/__init__.py
similarity index 96%
rename from .codecov.yml
rename to airflow/_vendor/__init__.py
index 4094f35dccf0d..114d189da14ab 100644
--- a/.codecov.yml
+++ b/airflow/_vendor/__init__.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -6,14 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-# keep default
diff --git a/airflow/_vendor/nvd3/LICENSE b/airflow/_vendor/nvd3/LICENSE
new file mode 100644
index 0000000000000..1add6249e57b4
--- /dev/null
+++ b/airflow/_vendor/nvd3/LICENSE
@@ -0,0 +1,24 @@
+The MIT License (MIT)
+
+Python-nvd3
+
+Copyright (c) 2013 Arezqui Belaid <areski@gmail.com> and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/airflow/_vendor/nvd3/NVD3Chart.py b/airflow/_vendor/nvd3/NVD3Chart.py
new file mode 100644
index 0000000000000..faefe5d3a0fcf
--- /dev/null
+++ b/airflow/_vendor/nvd3/NVD3Chart.py
@@ -0,0 +1,506 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from __future__ import unicode_literals
+from optparse import OptionParser
+from jinja2 import Environment, PackageLoader
+from airflow._vendor.slugify import slugify
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+CONTENT_FILENAME = "./content.html"
+PAGE_FILENAME = "./page.html"
+
+
+pl = PackageLoader('airflow._vendor.nvd3', 'templates')
+jinja2_env = Environment(lstrip_blocks=True, trim_blocks=True, loader=pl)
+
+template_content = jinja2_env.get_template(CONTENT_FILENAME)
+template_page = jinja2_env.get_template(PAGE_FILENAME)
+
+
+def stab(tab=1):
+    """
+    create space tabulation
+    """
+    return ' ' * 4 * tab
+
+
+class NVD3Chart(object):
+    """
+    NVD3Chart Base class.
+    """
+    #: chart count
+    count = 0
+    #:  directory holding the assets (bower_components)
+    assets_directory = './bower_components/'
+
+    # this attribute is overriden by children of this
+    # class
+    CHART_FILENAME = None
+    template_environment = Environment(lstrip_blocks=True, trim_blocks=True,
+                                       loader=pl)
+
+    def __init__(self, **kwargs):
+        """
+        This is the base class for all the charts. The following keywords are
+        accepted:
+
+        :keyword: **display_container** - default: ``True``
+        :keyword: **jquery_on_ready** - default: ``False``
+        :keyword: **charttooltip_dateformat** - default: ``'%d %b %Y'``
+        :keyword: **name** - default: the class name
+                    ``model`` - set the model (e.g. ``pieChart``, `
+                    ``LineWithFocusChart``, ``MultiBarChart``).
+        :keyword: **color_category** - default - ``None``
+        :keyword: **color_list** - default - ``None``
+                  used by pieChart (e.g. ``['red', 'blue', 'orange']``)
+        :keyword: **margin_bottom** - default - ``20``
+        :keyword: **margin_left** - default - ``60``
+        :keyword: **margin_right** - default - ``60``
+        :keyword: **margin_top** - default - ``30``
+        :keyword: **height** - default - ``''``
+        :keyword: **width** - default - ``''``
+        :keyword: **stacked** - default - ``False``
+        :keyword: **focus_enable** - default - ``False``
+        :keyword: **resize** - define - ``False``
+        :keyword: **show_legend** - default - ``True``
+        :keyword: **show_labels** - default - ``True``
+        :keyword: **tag_script_js** - default - ``True``
+        :keyword: **use_interactive_guideline** - default - ``False``
+        :keyword: **chart_attr** - default - ``None``
+        :keyword: **extras** - default - ``None``
+
+            Extra chart modifiers. Use this to modify different attributes of
+            the chart.
+        :keyword: **x_axis_date** - default - False
+            Signal that x axis is a date axis
+        :keyword: **date_format** - default - ``%x``
+                  see https://github.com/mbostock/d3/wiki/Time-Formatting
+        :keyword: **x_axis_format** - default - ``''``.
+        :keyword: **y_axis_format** - default - ``''``.
+        :keyword: **style** - default - ``''``
+            Style modifiers for the DIV container.
+        :keyword: **color_category** - default - ``category10``
+
+            Acceptable values are nvd3 categories such as
+            ``category10``, ``category20``, ``category20c``.
+        """
+        # set the model
+        self.model = self.__class__.__name__  #: The chart model,
+
+        #: an Instance of Jinja2 template
+        self.template_page_nvd3 = template_page
+        self.template_content_nvd3 = template_content
+        self.series = []
+        self.axislist = {}
+        # accepted keywords
+        self.display_container = kwargs.get('display_container', True)
+        self.charttooltip_dateformat = kwargs.get('charttooltip_dateformat',
+                                                  '%d %b %Y')
+        self._slugify_name(kwargs.get('name', self.model))
+        self.jquery_on_ready = kwargs.get('jquery_on_ready', False)
+        self.color_category = kwargs.get('color_category', None)
+        self.color_list = kwargs.get('color_list', None)
+        self.margin_bottom = kwargs.get('margin_bottom', 20)
+        self.margin_left = kwargs.get('margin_left', 60)
+        self.margin_right = kwargs.get('margin_right', 60)
+        self.margin_top = kwargs.get('margin_top', 30)
+        self.height = kwargs.get('height', '')
+        self.width = kwargs.get('width', '')
+        self.stacked = kwargs.get('stacked', False)
+        self.focus_enable = kwargs.get('focus_enable', False)
+        self.resize = kwargs.get('resize', False)
+        self.show_legend = kwargs.get('show_legend', True)
+        self.show_labels = kwargs.get('show_labels', True)
+        self.tag_script_js = kwargs.get('tag_script_js', True)
+        self.use_interactive_guideline = kwargs.get("use_interactive_guideline",
+                                                    False)
+        self.chart_attr = kwargs.get("chart_attr", {})
+        self.extras = kwargs.get('extras', None)
+        self.style = kwargs.get('style', '')
+        self.date_format = kwargs.get('date_format', '%x')
+        self.x_axis_date = kwargs.get('x_axis_date', False)
+        #: x-axis contain date format or not
+        # possible duplicate of x_axis_date
+        self.date_flag = kwargs.get('date_flag', False)
+        self.x_axis_format = kwargs.get('x_axis_format', '')
+        # Load remote JS assets or use the local bower assets?
+        self.remote_js_assets = kwargs.get('remote_js_assets', True)
+
+        # None keywords attribute that should be modified by methods
+        # We should change all these to _attr
+
+        self.htmlcontent = ''  #: written by buildhtml
+        self.htmlheader = ''
+        #: Place holder for the graph (the HTML div)
+        #: Written by ``buildcontainer``
+        self.container = u''
+        #: Header for javascript code
+        self.containerheader = u''
+        # CDN http://cdnjs.com/libraries/nvd3/ needs to make sure it's up to
+        # date
+        self.header_css = [
+            '<link href="%s" rel="stylesheet" />' % h for h in
+            (
+                'https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css' if self.remote_js_assets else self.assets_directory + 'nvd3/src/nv.d3.css',
+            )
+        ]
+
+        self.header_js = [
+            '<script src="%s"></script>' % h for h in
+            (
+                'https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js' if self.remote_js_assets else self.assets_directory + 'd3/d3.min.js',
+                'https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js' if self.remote_js_assets else self.assets_directory + 'nvd3/nv.d3.min.js'
+            )
+        ]
+
+        #: Javascript code as string
+        self.jschart = None
+        self.custom_tooltip_flag = False
+        self.tooltip_condition_string = ''
+        self.charttooltip = ''
+        self.serie_no = 1
+
+    def _slugify_name(self, name):
+        """Slufigy name with underscore"""
+        self.name = slugify(name, separator='_')
+
+    def add_serie(self, y, x, name=None, extra=None, **kwargs):
+        """
+        add serie - Series are list of data that will be plotted
+        y {1, 2, 3, 4, 5} / x {1, 2, 3, 4, 5}
+
+        **Attributes**:
+
+            * ``name`` - set Serie name
+            * ``x`` - x-axis data
+            * ``y`` - y-axis data
+
+            kwargs:
+
+            * ``shape`` - for scatterChart, you can set different shapes
+                          (circle, triangle etc...)
+            * ``size`` - for scatterChart, you can set size of different shapes
+            * ``type`` - for multiChart, type should be bar
+            * ``bar`` - to display bars in Chart
+            * ``color_list`` - define list of colors which will be
+                               used by pieChart
+            * ``color`` - set axis color
+            * ``disabled`` -
+
+            extra:
+
+            * ``tooltip`` - set tooltip flag
+            * ``date_format`` - set date_format for tooltip if x-axis is in
+              date format
+
+        """
+        if not name:
+            name = "Serie %d" % (self.serie_no)
+
+        # For scatterChart shape & size fields are added in serie
+        if 'shape' in kwargs or 'size' in kwargs:
+            csize = kwargs.get('size', 1)
+            cshape = kwargs.get('shape', 'circle')
+
+            serie = [{
+                'x': x[i],
+                'y': j,
+                'shape': cshape,
+                'size': csize[i] if isinstance(csize, list) else csize
+            } for i, j in enumerate(y)]
+        else:
+            if self.model == 'pieChart':
+                serie = [{'label': x[i], 'value': y} for i, y in enumerate(y)]
+            else:
+                serie = [{'x': x[i], 'y': y} for i, y in enumerate(y)]
+
+        data_keyvalue = {'values': serie, 'key': name}
+
+        # multiChart
+        # Histogram type='bar' for the series
+        if 'type' in kwargs and kwargs['type']:
+            data_keyvalue['type'] = kwargs['type']
+
+        # Define on which Y axis the serie is related
+        # a chart can have 2 Y axis, left and right, by default only one Y Axis is used
+        if 'yaxis' in kwargs and kwargs['yaxis']:
+            data_keyvalue['yAxis'] = kwargs['yaxis']
+        else:
+            if self.model != 'pieChart':
+                data_keyvalue['yAxis'] = '1'
+
+        if 'bar' in kwargs and kwargs['bar']:
+            data_keyvalue['bar'] = 'true'
+
+        if 'disabled' in kwargs and kwargs['disabled']:
+            data_keyvalue['disabled'] = 'true'
+
+        if 'color' in kwargs and kwargs['color']:
+            data_keyvalue['color'] = kwargs['color']
+
+        if extra:
+            if self.model == 'pieChart':
+                if 'color_list' in extra and extra['color_list']:
+                    self.color_list = extra['color_list']
+
+            if extra.get('date_format'):
+                self.charttooltip_dateformat = extra['date_format']
+
+            if extra.get('tooltip'):
+                self.custom_tooltip_flag = True
+
+                if self.model != 'pieChart':
+                    _start = extra['tooltip']['y_start']
+                    _end = extra['tooltip']['y_end']
+                    _start = ("'" + str(_start) + "' + ") if _start else ''
+                    _end = (" + '" + str(_end) + "'") if _end else ''
+
+                    if self.model == 'linePlusBarChart':
+                        if self.tooltip_condition_string:
+                            self.tooltip_condition_string += stab(5)
+                        self.tooltip_condition_string += stab(0) + "if(key.indexOf('" + name + "') > -1 ){\n" +\
+                            stab(6) + "var y = " + _start + " String(graph.point.y) " + _end + ";\n" +\
+                            stab(5) + "}\n"
+                    elif self.model == 'cumulativeLineChart':
+                        self.tooltip_condition_string += stab(0) + "if(key == '" + name + "'){\n" +\
+                            stab(6) + "var y = " + _start + " String(e) " + _end + ";\n" +\
+                            stab(5) + "}\n"
+                    else:
+                        self.tooltip_condition_string += stab(5) + "if(key == '" + name + "'){\n" +\
+                            stab(6) + "var y = " + _start + " String(graph.point.y) " + _end + ";\n" +\
+                            stab(5) + "}\n"
+
+                if self.model == 'pieChart':
+                    _start = extra['tooltip']['y_start']
+                    _end = extra['tooltip']['y_end']
+                    _start = ("'" + str(_start) + "' + ") if _start else ''
+                    _end = (" + '" + str(_end) + "'") if _end else ''
+                    self.tooltip_condition_string += "var y = " + _start + " String(y) " + _end + ";\n"
+
+        # Increment series counter & append
+        self.serie_no += 1
+        self.series.append(data_keyvalue)
+
+    def add_chart_extras(self, extras):
+        """
+        Use this method to add extra d3 properties to your chart.
+        For example, you want to change the text color of the graph::
+
+            chart = pieChart(name='pieChart', color_category='category20c', height=400, width=400)
+
+            xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawberry", "Pineapple"]
+            ydata = [3, 4, 0, 1, 5, 7, 3]
+
+            extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
+            chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
+
+        The above code will create graph with a black text, the following will change it::
+
+            text_white="d3.selectAll('#pieChart text').style('fill', 'white');"
+            chart.add_chart_extras(text_white)
+
+        The above extras will be appended to the java script generated.
+
+        Alternatively, you can use the following initialization::
+
+            chart = pieChart(name='pieChart',
+                             color_category='category20c',
+                             height=400, width=400,
+                             extras=text_white)
+        """
+        self.extras = extras
+
+    def set_graph_height(self, height):
+        """Set Graph height"""
+        self.height = str(height)
+
+    def set_graph_width(self, width):
+        """Set Graph width"""
+        self.width = str(width)
+
+    def set_containerheader(self, containerheader):
+        """Set containerheader"""
+        self.containerheader = containerheader
+
+    def set_date_flag(self, date_flag=False):
+        """Set date flag"""
+        self.date_flag = date_flag
+
+    def set_custom_tooltip_flag(self, custom_tooltip_flag):
+        """Set custom_tooltip_flag & date_flag"""
+        self.custom_tooltip_flag = custom_tooltip_flag
+
+    def __str__(self):
+        """return htmlcontent"""
+        self.buildhtml()
+        return self.htmlcontent
+
+    def buildcontent(self):
+        """Build HTML content only, no header or body tags. To be useful this
+        will usually require the attribute `juqery_on_ready` to be set which
+        will wrap the js in $(function(){<regular_js>};)
+        """
+        self.buildcontainer()
+        # if the subclass has a method buildjs this method will be
+        # called instead of the method defined here
+        # when this subclass method is entered it does call
+        # the method buildjschart defined here
+        self.buildjschart()
+        self.htmlcontent = self.template_content_nvd3.render(chart=self)
+
+    def buildhtml(self):
+        """Build the HTML page
+        Create the htmlheader with css / js
+        Create html page
+        Add Js code for nvd3
+        """
+        self.buildcontent()
+        self.content = self.htmlcontent
+        self.htmlcontent = self.template_page_nvd3.render(chart=self)
+
+    # this is used by django-nvd3
+    def buildhtmlheader(self):
+        """generate HTML header content"""
+        self.htmlheader = ''
+        # If the JavaScript assets have already been injected, don't bother re-sourcing them.
+        global _js_initialized
+        if '_js_initialized' not in globals() or not _js_initialized:
+            for css in self.header_css:
+                self.htmlheader += css
+            for js in self.header_js:
+                self.htmlheader += js
+
+    def buildcontainer(self):
+        """generate HTML div"""
+        if self.container:
+            return
+
+        # Create SVG div with style
+        if self.width:
+            if self.width[-1] != '%':
+                self.style += 'width:%spx;' % self.width
+            else:
+                self.style += 'width:%s;' % self.width
+        if self.height:
+            if self.height[-1] != '%':
+                self.style += 'height:%spx;' % self.height
+            else:
+                self.style += 'height:%s;' % self.height
+        if self.style:
+            self.style = 'style="%s"' % self.style
+
+        self.container = self.containerheader + \
+            '<div id="%s"><svg %s></svg></div>\n' % (self.name, self.style)
+
+    def buildjschart(self):
+        """generate javascript code for the chart"""
+        self.jschart = ''
+
+        # add custom tooltip string in jschart
+        # default condition (if build_custom_tooltip is not called explicitly with date_flag=True)
+        if self.tooltip_condition_string == '':
+            self.tooltip_condition_string = 'var y = String(graph.point.y);\n'
+
+        # Include data
+        self.series_js = json.dumps(self.series)
+
+    def create_x_axis(self, name, label=None, format=None, date=False, custom_format=False):
+        """Create X-axis"""
+        axis = {}
+        if custom_format and format:
+            axis['tickFormat'] = format
+        elif format:
+            if format == 'AM_PM':
+                axis['tickFormat'] = "function(d) { return get_am_pm(parseInt(d)); }"
+            else:
+                axis['tickFormat'] = "d3.format(',%s')" % format
+
+        if label:
+            axis['axisLabel'] = "'" + label + "'"
+
+        # date format : see https://github.com/mbostock/d3/wiki/Time-Formatting
+        if date:
+            self.dateformat = format
+            axis['tickFormat'] = ("function(d) { return d3.time.format('%s')"
+                                  "(new Date(parseInt(d))) }\n"
+                                  "" % self.dateformat)
+            # flag is the x Axis is a date
+            if name[0] == 'x':
+                self.x_axis_date = True
+
+        # Add new axis to list of axis
+        self.axislist[name] = axis
+
+        # Create x2Axis if focus_enable
+        if name == "xAxis" and self.focus_enable:
+            self.axislist['x2Axis'] = axis
+
+    def create_y_axis(self, name, label=None, format=None, custom_format=False):
+        """
+        Create Y-axis
+        """
+        axis = {}
+
+        if custom_format and format:
+            axis['tickFormat'] = format
+        elif format:
+            axis['tickFormat'] = "d3.format(',%s')" % format
+
+        if label:
+            axis['axisLabel'] = "'" + label + "'"
+
+        # Add new axis to list of axis
+        self.axislist[name] = axis
+
+
+class TemplateMixin(object):
+    """
+    A mixin that override buildcontent. Instead of building the complex
+    content template we exploit Jinja2 inheritance. Thus each chart class
+    renders it's own chart template which inherits from content.html
+    """
+    def buildcontent(self):
+        """Build HTML content only, no header or body tags. To be useful this
+        will usually require the attribute `juqery_on_ready` to be set which
+        will wrap the js in $(function(){<regular_js>};)
+        """
+        self.buildcontainer()
+        # if the subclass has a method buildjs this method will be
+        # called instead of the method defined here
+        # when this subclass method is entered it does call
+        # the method buildjschart defined here
+        self.buildjschart()
+        self.htmlcontent = self.template_chart_nvd3.render(chart=self)
+
+
+def _main():
+    """
+    Parse options and process commands
+    """
+    # Parse arguments
+    usage = "usage: nvd3.py [options]"
+    parser = OptionParser(usage=usage,
+                          version=("python-nvd3 - Charts generator with "
+                                   "nvd3.js and d3.js"))
+    parser.add_option("-q", "--quiet",
+                      action="store_false", dest="verbose", default=True,
+                      help="don't print messages to stdout")
+
+    (options, args) = parser.parse_args()
+
+
+if __name__ == '__main__':
+    _main()
diff --git a/airflow/_vendor/nvd3/__init__.py b/airflow/_vendor/nvd3/__init__.py
new file mode 100755
index 0000000000000..5b737b45361ad
--- /dev/null
+++ b/airflow/_vendor/nvd3/__init__.py
@@ -0,0 +1,29 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+__version__ = '0.15.0'
+__all__ = ['lineChart', 'pieChart', 'lineWithFocusChart',
+           'stackedAreaChart', 'multiBarHorizontalChart',
+           'linePlusBarChart', 'cumulativeLineChart',
+           'scatterChart', 'discreteBarChart', 'multiBarChart']
+
+
+from .lineChart import lineChart
+from .pieChart import pieChart
+from .lineWithFocusChart import lineWithFocusChart
+from .stackedAreaChart import stackedAreaChart
+from .multiBarHorizontalChart import multiBarHorizontalChart
+from .linePlusBarChart import linePlusBarChart
+from .cumulativeLineChart import cumulativeLineChart
+from .scatterChart import scatterChart
+from .discreteBarChart import discreteBarChart
+from .multiBarChart import multiBarChart
+from . import ipynb
diff --git a/airflow/_vendor/nvd3/cumulativeLineChart.py b/airflow/_vendor/nvd3/cumulativeLineChart.py
new file mode 100644
index 0000000000000..d98d0867e4d99
--- /dev/null
+++ b/airflow/_vendor/nvd3/cumulativeLineChart.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class cumulativeLineChart(TemplateMixin, NVD3Chart):
+    """
+    A cumulative line chart is used when you have one important grouping representing
+    an ordered set of data and one value to show, summed over time.
+
+    Python example::
+
+        from nvd3 import cumulativeLineChart
+        chart = cumulativeLineChart(name='cumulativeLineChart', x_is_date=True)
+        xdata = [1365026400000000, 1365026500000000, 1365026600000000]
+        ydata = [6, 5, 1]
+        y2data = [36, 55, 11]
+
+        extra_serie = {"tooltip": {"y_start": "There are ", "y_end": " calls"}}
+        chart.add_serie(name="Serie 1", y=ydata, x=xdata, extra=extra_serie)
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " mins"}}
+        chart.add_serie(name="Serie 2", y=y2data, x=xdata, extra=extra_serie)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="cumulativeLineChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+            data_cumulativeLineChart=[{"values": [{"y": 6, "x": 1365026400000000},
+            {"y": 5, "x": 1365026500000000},
+            {"y": 1, "x": 1365026600000000}],
+            "key": "Serie 1", "yAxis": "1"},
+            {"values": [{"y": 36, "x": 1365026400000000},
+            {"y": 55, "x": 1365026500000000},
+            {"y": 11, "x": 1365026600000000}], "key": "Serie 2", "yAxis": "1"}];
+            nv.addGraph(function() {
+                var chart = nv.models.cumulativeLineChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_cumulativeLineChart;
+
+                        chart.xAxis
+                            .tickFormat(function(d) { return d3.time.format('%d %b %Y')(new Date(parseInt(d))) });
+                        chart.yAxis
+                            .tickFormat(d3.format(',.1%'));
+
+                    chart.tooltipContent(function(key, y, e, graph) {
+                        var x = d3.time.format("%d %b %Y")(new Date(parseInt(graph.point.x)));
+                        var y = String(graph.point.y);
+                        if(key == 'Serie 1'){
+                                var y = 'There are ' +  String(e)  + ' calls';
+                            }if(key == 'Serie 2'){
+                                var y =  String(e)  + ' mins';
+                            }
+                        tooltip_str = '<center><b>'+key+'</b></center>' + y + ' on ' + x;
+                        return tooltip_str;
+                    });
+                    chart.showLegend(true);
+
+                d3.select('#cumulativeLineChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('height', 450)
+                    .call(chart); });
+        </script>
+
+    """
+
+    CHART_FILENAME = "./cumulativelinechart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(cumulativeLineChart, self).__init__(**kwargs)
+        self.model = 'cumulativeLineChart'
+
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis',
+                               format=kwargs.get('x_axis_format', '%d %b %Y'),
+                               date=True)
+            self.set_custom_tooltip_flag(True)
+        else:
+            self.create_x_axis('xAxis', format=kwargs.get(
+                'x_axis_format', '.2f'))
+
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', '.1%'))
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/discreteBarChart.py b/airflow/_vendor/nvd3/discreteBarChart.py
new file mode 100644
index 0000000000000..cf6c8a4a8ff4b
--- /dev/null
+++ b/airflow/_vendor/nvd3/discreteBarChart.py
@@ -0,0 +1,91 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class discreteBarChart(TemplateMixin, NVD3Chart):
+    """
+    A discrete bar chart or bar graph is a chart with rectangular bars with
+    lengths proportional to the values that they represent.
+
+    Python example::
+
+        from nvd3 import discreteBarChart
+        chart = discreteBarChart(name='discreteBarChart', height=400, width=400)
+
+        xdata = ["A", "B", "C", "D", "E", "F"]
+        ydata = [3, 4, 0, -3, 5, 7]
+
+        chart.add_serie(y=ydata, x=xdata)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="discreteBarChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+            data_discreteBarChart=[{"values": [{"y": 3, "x": "A"}, {"y": 4, "x": "B"}, {"y": 0, "x": "C"}, {"y": -3, "x": "D"}, {"y": 5, "x": "E"}, {"y": 7, "x": "F"}], "key": "Serie 1", "yAxis": "1"}];
+
+            nv.addGraph(function() {
+                var chart = nv.models.discreteBarChart();
+
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+
+                var datum = data_discreteBarChart;
+                        chart.yAxis
+                            .tickFormat(d3.format(',.0f'));
+                        chart.tooltipContent(function(key, y, e, graph) {
+                            var x = String(graph.point.x);
+                            var y = String(graph.point.y);
+                            var y = String(graph.point.y);
+
+                            tooltip_str = '<center><b>'+key+'</b></center>' + y + ' at ' + x;
+                            return tooltip_str;
+                        });
+
+                d3.select('#discreteBarChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('width', 400)
+                    .attr('height', 400)
+                    .call(chart);
+            });
+        </script>
+
+
+    """
+    CHART_FILENAME = "./discretebarchart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(discreteBarChart, self).__init__(**kwargs)
+        self.model = 'discreteBarChart'
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis',
+                               format=kwargs.get('x_axis_format',
+                                                 "%d %b %Y %H %S"),
+                               date=True)
+        else:
+            self.create_x_axis('xAxis', format=None)
+
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', ".0f"))
+
+        self.set_custom_tooltip_flag(True)
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/ipynb.py b/airflow/_vendor/nvd3/ipynb.py
new file mode 100644
index 0000000000000..f421afc0a8a50
--- /dev/null
+++ b/airflow/_vendor/nvd3/ipynb.py
@@ -0,0 +1,91 @@
+'''
+ipython compatability module for nvd3-python
+This adds simple ipython compatibility to the nvd3-python package, without making any
+major modifications to how the main package is structured.  It utilizes the IPython
+display-formatter functionality, as described at:
+http://nbviewer.ipython.org/github/ipython/ipython/blob/master/examples/notebooks/Custom%20Display%20Logic.ipynb
+For additional examples, see:
+https://github.com/sympy/sympy/blob/master/sympy/interactive/printing.py
+'''
+
+try:
+    _ip = get_ipython()
+except:
+    _ip = None
+if _ip and _ip.__module__.lower().startswith('ipy'):
+    global _js_initialized
+    _js_initialized = False
+
+    def _print_html(chart):
+        '''Function to return the HTML code for the div container plus the javascript
+        to generate the chart.  This function is bound to the ipython formatter so that
+        charts are displayed inline.'''
+        global _js_initialized
+        if not _js_initialized:
+            print('js not initialized - pausing to allow time for it to load...')
+            initialize_javascript()
+            import time
+            time.sleep(5)
+        chart.buildhtml()
+        return chart.htmlcontent
+
+    def _setup_ipython_formatter(ip):
+        ''' Set up the ipython formatter to display HTML formatted output inline'''
+        from IPython import __version__ as IPython_version
+        from nvd3 import __all__ as nvd3_all
+
+        if IPython_version >= '0.11':
+            html_formatter = ip.display_formatter.formatters['text/html']
+            for chart_type in nvd3_all:
+                html_formatter.for_type_by_name('nvd3.' + chart_type, chart_type, _print_html)
+
+    def initialize_javascript(d3_js_url='https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js',
+                              nvd3_js_url='https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js',
+                              nvd3_css_url='https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css',
+                              use_remote=False):
+        '''Initialize the ipython notebook to be able to display nvd3 results.
+        by instructing IPython to load the nvd3 JS and css files, and the d3 JS file.
+
+        by default, it looks for the files in your IPython Notebook working directory.
+
+        Takes the following options:
+
+        use_remote: use remote hosts for d3.js, nvd3.js, and nv.d3.css (default False)
+        * Note:  the following options are ignored if use_remote is False:
+        nvd3_css_url: location of nvd3 css file (default https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css)
+        nvd3_js_url: location of nvd3 javascript file (default  https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css)
+        d3_js_url: location of d3 javascript file (default https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js)
+        '''
+        from IPython.display import display, Javascript, HTML
+
+        if not use_remote:
+            # these file locations are for IPython 1.x, and will probably change when 2.x is released
+            d3_js_url = 'files/d3.v3.js'
+            nvd3_js_url = 'files/nv.d3.js'
+            nvd3_css_url = 'files/nv.d3.css'
+
+        # load the required javascript files
+
+        #display(Javascript('''$.getScript("%s")''' %(d3_js_url)))
+        display(HTML('''<link media="all" href="%s" type="text/css"
+                        rel="stylesheet"/>''' % (nvd3_css_url)))
+        # The following two methods for loading the script file are redundant.
+        # This is intentional.
+        # Ipython's loading of javscript in version 1.x is a bit squirrely, especially
+        # when creating demos to view in nbviewer.
+        # by trying twice, in two different ways (one using jquery and one using plain old
+        # HTML), we maximize our chances of successfully loading the script.
+        display(Javascript('''$.getScript("%s")''' % (nvd3_js_url)))
+        display(Javascript('''$.getScript("%s", function() {
+                              $.getScript("%s", function() {})});''' % (d3_js_url, nvd3_js_url)))
+        display(HTML('<script src="%s"></script>' % (d3_js_url)))
+        display(HTML('<script src="%s"></script>' % (nvd3_js_url)))
+
+        global _js_initialized
+        _js_initialized = True
+
+    print('loaded nvd3 IPython extension\n'
+          'run nvd3.ipynb.initialize_javascript() to set up the notebook\n'
+          'help(nvd3.ipynb.initialize_javascript) for options')
+
+    _setup_ipython_formatter(_ip)
diff --git a/airflow/_vendor/nvd3/lineChart.py b/airflow/_vendor/nvd3/lineChart.py
new file mode 100644
index 0000000000000..c237d069802ad
--- /dev/null
+++ b/airflow/_vendor/nvd3/lineChart.py
@@ -0,0 +1,120 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class lineChart(TemplateMixin, NVD3Chart):
+
+    """
+    A line chart or line graph is a type of chart which displays information
+    as a series of data points connected by straight line segments.
+
+    Python example::
+
+        from nvd3 import lineChart
+        chart = lineChart(name="lineChart", x_is_date=False, x_axis_format="AM_PM")
+
+        xdata = range(24)
+        ydata = [0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 4, 3, 3, 5, 7, 5, 3, 16, 6, 9, 15, 4, 12]
+        ydata2 = [9, 8, 11, 8, 3, 7, 10, 8, 6, 6, 9, 6, 5, 4, 3, 10, 0, 6, 3, 1, 0, 0, 0, 1]
+
+        extra_serie = {"tooltip": {"y_start": "There are ", "y_end": " calls"}}
+        chart.add_serie(y=ydata, x=xdata, name='sine', extra=extra_serie, **kwargs1)
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " min"}}
+        chart.add_serie(y=ydata2, x=xdata, name='cose', extra=extra_serie, **kwargs2)
+        chart.buildhtml()
+
+    Javascript renderd to:
+
+    .. raw:: html
+
+        <div id="lineChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+
+            data_lineChart=[{"values": [{"y": 0, "x": 0}, {"y": 0, "x": 1}, {"y": 1, "x": 2}, {"y": 1, "x": 3}, {"y": 0, "x": 4}, {"y": 0, "x": 5}, {"y": 0, "x": 6}, {"y": 0, "x": 7}, {"y": 1, "x": 8}, {"y": 0, "x": 9}, {"y": 0, "x": 10}, {"y": 4, "x": 11}, {"y": 3, "x": 12}, {"y": 3, "x": 13}, {"y": 5, "x": 14}, {"y": 7, "x": 15}, {"y": 5, "x": 16}, {"y": 3, "x": 17}, {"y": 16, "x": 18}, {"y": 6, "x": 19}, {"y": 9, "x": 20}, {"y": 15, "x": 21}, {"y": 4, "x": 22}, {"y": 12, "x": 23}], "key": "sine", "yAxis": "1"}, {"values": [{"y": 9, "x": 0}, {"y": 8, "x": 1}, {"y": 11, "x": 2}, {"y": 8, "x": 3}, {"y": 3, "x": 4}, {"y": 7, "x": 5}, {"y": 10, "x": 6}, {"y": 8, "x": 7}, {"y": 6, "x": 8}, {"y": 6, "x": 9}, {"y": 9, "x": 10}, {"y": 6, "x": 11}, {"y": 5, "x": 12}, {"y": 4, "x": 13}, {"y": 3, "x": 14}, {"y": 10, "x": 15}, {"y": 0, "x": 16}, {"y": 6, "x": 17}, {"y": 3, "x": 18}, {"y": 1, "x": 19}, {"y": 0, "x": 20}, {"y": 0, "x": 21}, {"y": 0, "x": 22}, {"y": 1, "x": 23}], "key": "cose", "yAxis": "1"}];
+
+            nv.addGraph(function() {
+                var chart = nv.models.lineChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_lineChart;
+                        chart.xAxis
+                            .tickFormat(function(d) { return get_am_pm(parseInt(d)); });
+                        chart.yAxis
+                            .tickFormat(d3.format(',.02f'));
+
+                        chart.tooltipContent(function(key, y, e, graph) {
+                            var x = String(graph.point.x);
+                            var y = String(graph.point.y);
+                                                if(key == 'sine'){
+                                var y = 'There is ' +  String(graph.point.y)  + ' calls';
+                            }
+                            if(key == 'cose'){
+                                var y =  String(graph.point.y)  + ' min';
+                            }
+
+                            tooltip_str = '<center><b>'+key+'</b></center>' + y + ' at ' + x;
+                            return tooltip_str;
+                        });
+                    chart.showLegend(true);
+                        function get_am_pm(d){
+                    if (d > 12) {
+                        d = d - 12; return (String(d) + 'PM');
+                    }
+                    else {
+                        return (String(d) + 'AM');
+                    }
+                };
+
+                d3.select('#lineChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('height', 200)
+                    .call(chart);
+            return chart;
+        });
+        </script>
+
+    See the source code of this page, to see the underlying javascript.
+    """
+    CHART_FILENAME = "./linechart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(lineChart, self).__init__(**kwargs)
+        self.model = 'lineChart'
+
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis',
+                               format=kwargs.get('x_axis_format', '%d %b %Y'),
+                               date=True)
+            self.set_custom_tooltip_flag(True)
+        else:
+            if kwargs.get('x_axis_format') == 'AM_PM':
+                self.x_axis_format = format = 'AM_PM'
+            else:
+                format = kwargs.get('x_axis_format', 'r')
+            self.create_x_axis('xAxis', format=format,
+                               custom_format=kwargs.get('x_custom_format',
+                                                        False))
+        self.create_y_axis(
+            'yAxis',
+            format=kwargs.get('y_axis_format', '.02f'),
+            custom_format=kwargs.get('y_custom_format', False))
+
+        # must have a specified height, otherwise it superimposes both chars
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/linePlusBarChart.py b/airflow/_vendor/nvd3/linePlusBarChart.py
new file mode 100644
index 0000000000000..4eaa5fc6ffdbf
--- /dev/null
+++ b/airflow/_vendor/nvd3/linePlusBarChart.py
@@ -0,0 +1,131 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class linePlusBarChart(TemplateMixin, NVD3Chart):
+
+    """
+    A linePlusBarChart Chart is a type of chart which displays information
+    as a series of data points connected by straight line segments
+    and with some series with rectangular bars with lengths proportional
+    to the values that they represent.
+
+    Python example::
+
+        from nvd3 import linePlusBarChart
+        chart = linePlusBarChart(name="linePlusBarChart",
+                             width=500, height=400, x_axis_format="%d %b %Y",
+                             x_is_date=True, focus_enable=True,
+                             yaxis2_format="function(d) { return d3.format(',0.3f')(d) }")
+
+        xdata = [1338501600000, 1345501600000, 1353501600000]
+        ydata = [6, 5, 1]
+        y2data = [0.002, 0.003, 0.004]
+
+        extra_serie = {"tooltip": {"y_start": "There are ", "y_end": " calls"},
+                       "date_format": "%d %b %Y %H:%S" }
+        chart.add_serie(name="Serie 1", y=ydata, x=xdata, extra=extra_serie,
+                        bar=True)
+
+        extra_serie = {"tooltip": {"y_start": "There are ", "y_end": " min"}}
+        chart.add_serie(name="Serie 2", y=y2data, x=xdata, extra=extra_serie)
+        chart.buildcontent()
+
+    Note that in case you have two data serie with extreme different numbers,
+    that you would like to format in different ways,
+    you can pass a keyword *yaxis1_format* or *yaxis2_format* when
+    creating the graph.
+
+    In the example above the graph created presents the values of the second
+    data series with three digits right  of the decimal point.
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="linePlusBarChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+            data_linePlusBarChart=[{"bar": "true", "values": [{"y": 6, "x": 1338501600000}, {"y": 5, "x": 1345501600000}, {"y": 1, "x": 1353501600000}], "key": "Serie 1", "yAxis": "1"}, {"values": [{"y": 0.002, "x": 1338501600000}, {"y": 0.003, "x": 1345501600000}, {"y": 0.004, "x": 1353501600000}], "key": "Serie 2", "yAxis": "1"}];
+            nv.addGraph(function() {
+                var chart = nv.models.linePlusBarChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_linePlusBarChart;
+
+                    chart.y2Axis
+                        .tickFormat(function(d) { return d3.format(',0.3f')(d) });
+                    chart.xAxis
+                        .tickFormat(function(d) { return d3.time.format('%d %b %Y')(new Date(parseInt(d))) });
+                    chart.y1Axis
+                        .tickFormat(function(d) { return d3.format(',f')(d) });
+
+                    chart.tooltipContent(function(key, y, e, graph) {
+                        var x = d3.time.format("%d %b %Y %H:%S")(new Date(parseInt(graph.point.x)));
+                        var y = String(graph.point.y);
+                        if(key.indexOf('Serie 1') > -1 ){
+                                var y = 'There are ' +  String(graph.point.y)  + ' calls';
+                            }
+                            if(key.indexOf('Serie 2') > -1 ){
+                                var y = 'There are ' +  String(graph.point.y)  + ' min';
+                            }
+                        tooltip_str = '<center><b>'+key+'</b></center>' + y + ' on ' + x;
+                        return tooltip_str;
+                    });
+                    chart.showLegend(true);
+                d3.select('#linePlusBarChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('width', 500)
+                    .attr('height', 400)
+                    .call(chart);
+            });
+        </script>
+
+    """
+    CHART_FILENAME = "./lineplusbarchart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(linePlusBarChart, self).__init__(**kwargs)
+        self.model = 'linePlusBarChart'
+
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+        self.yaxis1_format = kwargs.get('yaxis1_format',
+                                        "function(d) { return d3.format(',f')(d) }")
+        self.yaxis2_format = kwargs.get('yaxis2_format',
+                                        "function(d) { return d3.format(',f')(d) }")
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis',
+                               format=kwargs.get('x_axis_format',
+                                                 '%d %b %Y %H %S'),
+                               date=True)
+            self.create_x_axis('x2Axis', format=kwargs.get('x_axis_format',
+                                                           '%d %b %Y %H %S'),
+                               date=True)
+            self.set_custom_tooltip_flag(True)
+        else:
+            self.create_x_axis('xAxis', format=kwargs.get('x_axis_format',
+                                                          '.2f'))
+            self.create_x_axis('x2Axis', format=kwargs.get('x_axis_format',
+                                                           '.2f'))
+
+        self.create_y_axis('y1Axis', format=self.yaxis1_format,
+                           custom_format=True)
+        self.create_y_axis('y2Axis', format=self.yaxis2_format,
+                           custom_format=True)
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/lineWithFocusChart.py b/airflow/_vendor/nvd3/lineWithFocusChart.py
new file mode 100644
index 0000000000000..cd26cd4716652
--- /dev/null
+++ b/airflow/_vendor/nvd3/lineWithFocusChart.py
@@ -0,0 +1,105 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class lineWithFocusChart(TemplateMixin, NVD3Chart):
+    """
+    A lineWithFocusChart or line graph is a type of chart which displays information
+    as a series of data points connected by straight line segments.
+    The lineWithFocusChart provide a smaller chart that act as a selector,
+    this is very useful if you want to zoom on a specific time period.
+
+    Python example::
+
+        from nvd3 import lineWithFocusChart
+        chart = lineWithFocusChart(name='lineWithFocusChart', x_is_date=True, x_axis_format="%d %b %Y")
+        xdata = [1365026400000000, 1365026500000000, 1365026600000000, 1365026700000000, 1365026800000000, 1365026900000000, 1365027000000000]
+        ydata = [-6, 5, -1, 2, 4, 8, 10]
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " ext"},
+                       "date_format": "%d %b %Y"}
+        chart.add_serie(name="Serie 1", y=ydata, x=xdata, extra=extra_serie)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="lineWithFocusChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+            data_lineWithFocusChart=[{"values": [{"y": -6, "x": 1365026400000000}, {"y": 5, "x": 1365026500000000}, {"y": -1, "x": 1365026600000000}], "key": "Serie 1", "yAxis": "1"}];
+            nv.addGraph(function() {
+                var chart = nv.models.lineWithFocusChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_lineWithFocusChart;
+                        chart.yAxis
+                            .tickFormat(d3.format(',.2f'));
+                        chart.y2Axis
+                            .tickFormat(d3.format(',.2f'));
+                        chart.xAxis
+                            .tickFormat(function(d) { return d3.time.format('%d %b %Y')(new Date(parseInt(d))) });
+                        chart.x2Axis
+                            .tickFormat(function(d) { return d3.time.format('%d %b %Y')(new Date(parseInt(d))) });
+
+                    chart.tooltipContent(function(key, y, e, graph) {
+                        var x = d3.time.format("%d %b %Y")(new Date(parseInt(graph.point.x)));
+                        var y = String(graph.point.y);
+                                            if(key == 'Serie 1'){
+                                var y =  String(graph.point.y)  + ' ext';
+                            }
+
+                        tooltip_str = '<center><b>'+key+'</b></center>' + y + ' on ' + x;
+                        return tooltip_str; });
+
+                    chart.showLegend(true);
+
+                d3.select('#lineWithFocusChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('height', 450)
+                    .call(chart); });
+        </script>
+
+    """
+
+    CHART_FILENAME = "./linewfocuschart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(lineWithFocusChart, self).__init__(**kwargs)
+        self.model = 'lineWithFocusChart'
+
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis', format=kwargs.get('x_axis_format',
+                                                          '%d %b %Y %H %S'),
+                               date=True)
+            self.create_x_axis('x2Axis', format=kwargs.get('x_axis_format',
+                                                           '%d %b %Y %H %S'),
+                               date=True)
+            self.set_custom_tooltip_flag(True)
+        else:
+            self.create_x_axis('xAxis', format=kwargs.get('x_axis_format',
+                                                          '.2f'))
+            self.create_x_axis('x2Axis', format=kwargs.get('x_axis_format',
+                                                           '.2f'))
+
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', '.2f'))
+        self.create_y_axis('y2Axis', format=kwargs.get('y_axis_format', '.2f'))
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/multiBarChart.py b/airflow/_vendor/nvd3/multiBarChart.py
new file mode 100644
index 0000000000000..cf335919a84c7
--- /dev/null
+++ b/airflow/_vendor/nvd3/multiBarChart.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class multiBarChart(TemplateMixin, NVD3Chart):
+    """
+    A multiple bar graph contains comparisons of two or more categories or bars.
+    One axis represents a quantity and the other axis identifies a specific feature
+    about the categories. Reading a multiple bar graph includes looking at extremes
+    (tallest/longest vs. shortest) in each grouping.
+
+    Python example::
+
+        from nvd3 import multiBarChart
+        chart = multiBarChart(width=500, height=400, x_axis_format=None)
+        xdata = ['one', 'two', 'three', 'four']
+        ydata1 = [6, 12, 9, 16]
+        ydata2 = [8, 14, 7, 11]
+
+        chart.add_serie(name="Serie 1", y=ydata1, x=xdata)
+        chart.add_serie(name="Serie 2", y=ydata2, x=xdata)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="multiBarChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+
+            data_multiBarChart=[{"values":
+                                [{"y": 6, "x": "one"},
+                                {"y": 12, "x": "two"},
+                                {"y": 9, "x": "three"},
+                                {"y": 16, "x": "four"}],
+                                "key": "Serie 1", "yAxis": "1"},
+                                {"values":
+                                    [{"y": 8, "x": "one"},
+                                    {"y": 14, "x": "two"},
+                                    {"y": 7, "x": "three"},
+                                    {"y": 11, "x": "four"}],
+                                "key": "Serie 2", "yAxis": "1"}];
+
+            nv.addGraph(function() {
+                var chart = nv.models.multiBarChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_multiBarChart;
+                        chart.yAxis
+                            .tickFormat(d3.format(',.2f'));
+                    chart.showLegend(true);
+                d3.select('#multiBarChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('width', 500)
+                    .attr('height', 400)
+                    .call(chart);
+            });
+
+
+        </script>
+
+    """
+
+    CHART_FILENAME = "./multibarchart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(multiBarChart, self).__init__(**kwargs)
+
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis',
+                               format=kwargs.get('x_axis_format', '%d %b %Y'),
+                               date=True)
+            self.set_custom_tooltip_flag(True)
+        else:
+            self.create_x_axis('xAxis', format=kwargs.get('x_axis_format', '.2f'))
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', '.2f'))
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/multiBarHorizontalChart.py b/airflow/_vendor/nvd3/multiBarHorizontalChart.py
new file mode 100644
index 0000000000000..ac969c31b548c
--- /dev/null
+++ b/airflow/_vendor/nvd3/multiBarHorizontalChart.py
@@ -0,0 +1,100 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class multiBarHorizontalChart(TemplateMixin, NVD3Chart):
+    """
+    A multiple horizontal bar graph contains comparisons of two or more categories or bars.
+
+    Python example::
+
+        from nvd3 import multiBarHorizontalChart
+        chart = multiBarHorizontalChart(name='multiBarHorizontalChart', height=400, width=400)
+        xdata = [-14, -7, 7, 14]
+        ydata = [-6, 5, -1, 9]
+        y2data = [-23, -6, -32, 9]
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " balls"}}
+        chart.add_serie(name="Serie 1", y=ydata, x=xdata, extra=extra_serie)
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " calls"}}
+        chart.add_serie(name="Serie 2", y=y2data, x=xdata, extra=extra_serie)
+        chart.buildcontent()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="multiBarHorizontalChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+
+            data_multiBarHorizontalChart=[{"values":
+                [{"y": -6, "x": -14}, {"y": 5, "x": -7}, {"y": -1, "x": 7}, {"y": 9, "x": 14}],
+                "key": "Serie 1", "yAxis": "1"},
+                {"values":
+                    [{"y": -23, "x": -14}, {"y": -6, "x": -7}, {"y": -32, "x": 7}, {"y": 9, "x": 14}],
+                "key": "Serie 2", "yAxis": "1"}];
+
+            nv.addGraph(function() {
+                var chart = nv.models.multiBarHorizontalChart();
+
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+
+                var datum = data_multiBarHorizontalChart;
+
+                        chart.xAxis
+                            .tickFormat(d3.format(',.2f'));
+                        chart.yAxis
+                            .tickFormat(d3.format(',.2f'));
+
+                        chart.tooltipContent(function(key, y, e, graph) {
+                            var x = String(graph.point.x);
+                            var y = String(graph.point.y);
+                                                if(key == 'Serie 1'){
+                                var y =  String(graph.point.y)  + ' balls';
+                            }
+                            if(key == 'Serie 2'){
+                                var y =  String(graph.point.y)  + ' calls';
+                            }
+
+                            tooltip_str = '<center><b>'+key+'</b></center>' + y + ' at ' + x;
+                            return tooltip_str;
+                        });
+
+                    chart.showLegend(true);
+
+                d3.select('#multiBarHorizontalChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('width', 400)
+                    .attr('height', 400)
+                    .call(chart);
+            });
+        </script>
+
+    """
+
+    CHART_FILENAME = "./multibarcharthorizontal.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(multiBarHorizontalChart, self).__init__(**kwargs)
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+
+        self.create_x_axis('xAxis', format=kwargs.get('x_axis_format', '.2f'))
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', '.2f'))
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/pieChart.py b/airflow/_vendor/nvd3/pieChart.py
new file mode 100644
index 0000000000000..1db76bdb3424c
--- /dev/null
+++ b/airflow/_vendor/nvd3/pieChart.py
@@ -0,0 +1,101 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class pieChart(TemplateMixin, NVD3Chart):
+
+    """
+    A pie chart (or a circle graph) is a circular chart divided into sectors,
+    illustrating numerical proportion. In chart, the arc length of each sector
+    is proportional to the quantity it represents.
+
+    Python example::
+
+        from nvd3 import pieChart
+        chart = pieChart(name='pieChart', color_category='category20c',
+                         height=400, width=400)
+
+        xdata = ["Orange", "Banana", "Pear", "Kiwi", "Apple", "Strawbery",
+                 "Pineapple"]
+        ydata = [3, 4, 0, 1, 5, 7, 3]
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
+        chart.add_serie(y=ydata, x=xdata, extra=extra_serie)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="pieChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+
+
+            data_pieChart=[{"values": [{"value": 3, "label": "Orange"},
+                           {"value": 4, "label": "Banana"},
+                           {"value": 0, "label": "Pear"},
+                           {"value": 1, "label": "Kiwi"},
+                           {"value": 5, "label": "Apple"},
+                           {"value": 7, "label": "Strawberry"},
+                           {"value": 3, "label": "Pineapple"}],
+                           "key": "Serie 1"}];
+
+            nv.addGraph(function() {
+                var chart = nv.models.pieChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_pieChart[0].values;
+                        chart.tooltipContent(function(key, y, e, graph) {
+                            var x = String(key);
+                            var y =  String(y)  + ' cal';
+
+                            tooltip_str = '<center><b>'+x+'</b></center>' + y;
+                            return tooltip_str;
+                        });
+                    chart.showLegend(true);
+                    chart.showLabels(true);
+                    chart.donut(false);
+                chart
+                    .x(function(d) { return d.label })
+                    .y(function(d) { return d.value });
+                chart.width(400);
+                chart.height(400);
+
+                d3.select('#pieChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('width', 400)
+                    .attr('height', 400)
+                    .call(chart);  });
+        </script>
+
+    """
+    CHART_FILENAME = "./piechart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(pieChart, self).__init__(**kwargs)
+
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+        self.donut = kwargs.get('donut', False)
+        self.donutRatio = kwargs.get('donutRatio', 0.35)
+        self.color_list = []
+        self.create_x_axis('xAxis', format=None)
+        self.create_y_axis('yAxis', format=None)
+        # must have a specified height, otherwise it superimposes both chars
+        if height:
+            self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
+        self.donut = kwargs.get('donut', False)
+        self.donutRatio = kwargs.get('donutRatio', 0.35)
diff --git a/airflow/_vendor/nvd3/scatterChart.py b/airflow/_vendor/nvd3/scatterChart.py
new file mode 100644
index 0000000000000..c3a87d2908bde
--- /dev/null
+++ b/airflow/_vendor/nvd3/scatterChart.py
@@ -0,0 +1,121 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class scatterChart(TemplateMixin, NVD3Chart):
+
+    """
+    A scatter plot or scattergraph is a type of mathematical diagram using Cartesian
+    coordinates to display values for two variables for a set of data.
+    The data is displayed as a collection of points, each having the value of one variable
+    determining the position on the horizontal axis and the value of the other variable
+    determining the position on the vertical axis.
+
+    Python example::
+
+        from nvd3 import scatterChart
+        chart = scatterChart(name='scatterChart', height=400, width=400)
+        xdata = [3, 4, 0, -3, 5, 7]
+        ydata = [-1, 2, 3, 3, 15, 2]
+        ydata2 = [1, -2, 4, 7, -5, 3]
+
+        kwargs1 = {'shape': 'circle', 'size': '1'}
+        kwargs2 = {'shape': 'cross', 'size': '10'}
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " call"}}
+        chart.add_serie(name="series 1", y=ydata, x=xdata, extra=extra_serie, **kwargs1)
+
+        extra_serie = {"tooltip": {"y_start": "", "y_end": " min"}}
+        chart.add_serie(name="series 2", y=ydata2, x=xdata, extra=extra_serie, **kwargs2)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="scatterChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+
+        data_scatterChart=[{"values": [{"y": -1, "x": 3, "shape": "circle",
+            "size": "1"}, {"y": 2, "x": 4, "shape": "circle", "size": "1"},
+            {"y": 3, "x": 0, "shape": "circle", "size": "1"},
+            {"y": 3, "x": -3, "shape": "circle", "size": "1"},
+            {"y": 15, "x": 5, "shape": "circle", "size": "1"},
+            {"y": 2, "x": 7, "shape": "circle", "size": "1"}],
+            "key": "series 1", "yAxis": "1"},
+            {"values": [{"y": 1, "x": 3, "shape": "cross", "size": "10"},
+            {"y": -2, "x": 4, "shape": "cross", "size": "10"},
+            {"y": 4, "x": 0, "shape": "cross", "size": "10"},
+            {"y": 7, "x": -3, "shape": "cross", "size": "10"},
+            {"y": -5, "x": 5, "shape": "cross", "size": "10"},
+            {"y": 3, "x": 7, "shape": "cross", "size": "10"}],
+            "key": "series 2", "yAxis": "1"}];
+        nv.addGraph(function() {
+        var chart = nv.models.scatterChart();
+
+        chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+
+        var datum = data_scatterChart;
+
+                chart.xAxis
+                    .tickFormat(d3.format(',.02f'));
+                chart.yAxis
+                    .tickFormat(d3.format(',.02f'));
+
+                chart.tooltipContent(function(key, y, e, graph) {
+                    var x = String(graph.point.x);
+                    var y = String(graph.point.y);
+                                        if(key == 'series 1'){
+                        var y =  String(graph.point.y)  + ' call';
+                    }
+                    if(key == 'series 2'){
+                        var y =  String(graph.point.y)  + ' min';
+                    }
+
+                    tooltip_str = '<center><b>'+key+'</b></center>' + y + ' at ' + x;
+                    return tooltip_str;
+                });
+
+        chart.showLegend(true);
+
+        chart
+        .showDistX(true)
+        .showDistY(true)
+        .color(d3.scale.category10().range());
+
+            d3.select('#scatterChart svg')
+                .datum(datum)
+                .transition().duration(500)
+                .attr('width', 400)
+                .attr('height', 400)
+                .call(chart);
+        });
+        </script>
+
+    """
+
+    CHART_FILENAME = "./scatterchart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(scatterChart, self).__init__(**kwargs)
+        self.model = 'scatterChart'
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+        self.create_x_axis('xAxis', format=kwargs.get('x_axis_format', '.02f'),
+                           label=kwargs.get('x_axis_label', None))
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', '.02f'),
+                           label=kwargs.get('y_axis_label', None))
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/stackedAreaChart.py b/airflow/_vendor/nvd3/stackedAreaChart.py
new file mode 100644
index 0000000000000..8346cd2c53879
--- /dev/null
+++ b/airflow/_vendor/nvd3/stackedAreaChart.py
@@ -0,0 +1,99 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Python-nvd3 is a Python wrapper for NVD3 graph library.
+NVD3 is an attempt to build re-usable charts and chart components
+for d3.js without taking away the power that d3.js gives you.
+
+Project location : https://github.com/areski/python-nvd3
+"""
+
+from .NVD3Chart import NVD3Chart, TemplateMixin
+
+
+class stackedAreaChart(TemplateMixin, NVD3Chart):
+    """
+    The stacked area chart is identical to the area chart, except the areas are stacked
+    on top of each other, rather than overlapping. This can make the chart much easier to read.
+
+    Python example::
+
+        from nvd3 import stackedAreaChart
+        chart = stackedAreaChart(name='stackedAreaChart', height=400, width=400)
+
+        xdata = [100, 101, 102, 103, 104, 105, 106,]
+        ydata = [6, 11, 12, 7, 11, 10, 11]
+        ydata2 = [8, 20, 16, 12, 20, 28, 28]
+
+        extra_serie = {"tooltip": {"y_start": "There is ", "y_end": " min"}}
+        chart.add_serie(name="Serie 1", y=ydata, x=xdata, extra=extra_serie)
+        chart.add_serie(name="Serie 2", y=ydata2, x=xdata, extra=extra_serie)
+        chart.buildhtml()
+
+    Javascript generated:
+
+    .. raw:: html
+
+        <div id="stackedAreaChart"><svg style="height:450px; width:100%"></svg></div>
+        <script>
+
+
+            data_stackedAreaChart=[{"values": [{"y": 6, "x": 100}, {"y": 11, "x": 101}, {"y": 12, "x": 102}, {"y": 7, "x": 103}, {"y": 11, "x": 104}, {"y": 10, "x": 105}, {"y": 11, "x": 106}], "key": "Serie 1", "yAxis": "1"}, {"values": [{"y": 8, "x": 100}, {"y": 20, "x": 101}, {"y": 16, "x": 102}, {"y": 12, "x": 103}, {"y": 20, "x": 104}, {"y": 28, "x": 105}, {"y": 28, "x": 106}], "key": "Serie 2", "yAxis": "1"}];
+            nv.addGraph(function() {
+                var chart = nv.models.stackedAreaChart();
+                chart.margin({top: 30, right: 60, bottom: 20, left: 60});
+                var datum = data_stackedAreaChart;
+                        chart.xAxis
+                            .tickFormat(d3.format(',.2f'));
+                        chart.yAxis
+                            .tickFormat(d3.format(',.2f'));
+
+                        chart.tooltipContent(function(key, y, e, graph) {
+                            var x = String(graph.point.x);
+                            var y = String(graph.point.y);
+                                                if(key == 'Serie 1'){
+                                var y = 'There is ' +  String(graph.point.y)  + ' min';
+                            }
+                            if(key == 'Serie 2'){
+                                var y = 'There is ' +  String(graph.point.y)  + ' min';
+                            }
+
+                            tooltip_str = '<center><b>'+key+'</b></center>' + y + ' at ' + x;
+                            return tooltip_str;
+                        });
+                    chart.showLegend(true);
+                d3.select('#stackedAreaChart svg')
+                    .datum(datum)
+                    .transition().duration(500)
+                    .attr('width', 400)
+                    .attr('height', 400)
+                    .call(chart);
+            });
+        </script>
+
+    """
+
+    CHART_FILENAME = "./stackedareachart.html"
+    template_chart_nvd3 = NVD3Chart.template_environment.get_template(CHART_FILENAME)
+
+    def __init__(self, **kwargs):
+        super(stackedAreaChart, self).__init__(**kwargs)
+        height = kwargs.get('height', 450)
+        width = kwargs.get('width', None)
+        self.model = 'stackedAreaChart'
+
+        if kwargs.get('x_is_date', False):
+            self.set_date_flag(True)
+            self.create_x_axis('xAxis',
+                               format=kwargs.get('x_axis_format', '%d %b %Y'),
+                               date=True)
+            self.set_custom_tooltip_flag(True)
+        else:
+            self.create_x_axis('xAxis', format=kwargs.get('x_axis_format',
+                                                          '.2f'))
+        self.create_y_axis('yAxis', format=kwargs.get('y_axis_format', '.2f'))
+
+        self.set_graph_height(height)
+        if width:
+            self.set_graph_width(width)
diff --git a/airflow/_vendor/nvd3/templates/base.html b/airflow/_vendor/nvd3/templates/base.html
new file mode 100644
index 0000000000000..e2d39dd7642cb
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/base.html
@@ -0,0 +1,35 @@
+{% block container %}
+{% endblock %}
+
+{% block start_script %}
+    {% if chart.tag_script_js %}
+    <script>
+    {% endif %}
+
+    {% if chart.jquery_on_ready %}
+    $(function(){
+    {% endif %}
+{% endblock start_script %}
+
+{% block body %}
+    
+    {% block data %}
+    {% endblock data %}
+
+    {% block inject %}
+    {% endblock inject %}
+    
+    {% block foot %}
+    {% endblock foot %}
+     <!-- end of body from base-->
+    
+     {% endblock body%}
+
+{% block endscript %}
+    {% if chart.jquery_on_ready %}
+        });
+    {% endif %}
+    {% if chart.tag_script_js %}
+    </script>
+    {% endif %}
+{% endblock endscript %}
diff --git a/airflow/_vendor/nvd3/templates/content.html b/airflow/_vendor/nvd3/templates/content.html
new file mode 100644
index 0000000000000..787f39b555a4a
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/content.html
@@ -0,0 +1,123 @@
+{% extends "base.html" %}
+{% block container %}
+{% if chart.display_container %}
+    {{ chart.container }}
+{% endif %}
+{%  endblock  container %}
+
+{% block body %}
+    {% block data %}
+        data_{{ chart.name }}={{ chart.series_js }};
+    {% endblock data %}
+
+    {% block init %}
+    nv.addGraph(function() {
+        var chart = nv.models.{{ chart.model }}(){% if chart.use_interactive_guideline %}.useInteractiveGuideline(true){% endif %};
+
+        chart.margin({top: {{ chart.margin_top }}, right: {{ chart.margin_right }}, bottom: {{ chart.margin_bottom }}, left: {{ chart.margin_left }}});
+
+        var datum = data_{{ chart.name }};
+
+    {% if not chart.color_list and chart.color_category %}
+        chart.color(d3.scale.{{ chart.color_category }}().range());
+    {% endif %}
+    {% endblock init %}
+
+    {% if chart.stacked %}
+        chart.stacked(true);
+    {% endif %}
+
+    {% block focus %}
+    {% endblock focus %}
+
+
+    {% block axes %}
+        {% for axis, a in chart.axislist.items() %}
+            {% if a.items() %}
+            chart.{{ axis }}
+            {% for attr, value in a.items() %}
+                .{{ attr}}({{ value}}){% if loop.last %};
+                {% endif %}
+            {% endfor %}
+            {% endif %}
+        {% endfor %}
+    {% endblock axes %}
+
+    {# generate custom tooltip for the chart #}
+    {% block tooltip %}
+        {% if chart.custom_tooltip_flag %}
+            {% if not chart.date_flag %}
+                {% if chart.model == 'pieChart' %}
+                    {% block pietooltip %}
+                    {% endblock pietooltip %}
+                {% else %}
+                    chart.tooltipContent(function(key, y, e, graph) {
+                        var x = String(graph.point.x);
+                        var y = String(graph.point.y);
+                        {{ chart.tooltip_condition_string }}
+                        tooltip_str = '<center><b>'+key+'</b></center>' + y + ' at ' + x;
+                        return tooltip_str;
+                    });
+                {% endif %}
+            {% else %}
+                chart.tooltipContent(function(key, y, e, graph) {
+                    var x = d3.time.format("{{ chart.charttooltip_dateformat }}")(new Date(parseInt(graph.point.x)));
+                    var y = String(graph.point.y);
+                    {{ chart.tooltip_condition_string }}
+                    tooltip_str = '<center><b>'+key+'</b></center>' + y + ' on ' + x;
+                    return tooltip_str;
+                });
+            {% endif %}
+        {% endif %}
+    {% endblock tooltip %}
+
+    {# the shape attribute in kwargs is not applied when #}
+    {# not allowing other shapes to be rendered #}
+    {% block legend %}
+      chart.showLegend({{chart.show_legend|lower}});
+    {% endblock legend %}
+
+    {% block custoattr %}
+        {# add custom chart attributes #}
+        {% for attr, value in chart.chart_attr.items() %}
+            {% if value is string and value.startswith(".") %}:
+                chart.{{ attr }}{{ value }};
+            {% else %}
+                chart.{{ attr }}({{ value }});
+            {% endif %}
+        {% endfor %}
+
+        {% if chart.resize %}
+            nv.utils.windowResize(chart.update);
+        {% endif %}
+
+        {# include specific subchart #}
+        {{ chart.jschart }}
+
+    {% endblock custoattr %}
+
+    {% block inject %}
+        {# Inject data to D3 #}
+        d3.select('#{{ chart.name }} svg')
+            .datum(datum)
+            .transition().duration(500)
+            {% if chart.width %}
+            .attr('width', {{ chart.width}})
+            {% endif %}
+            {% if chart.height %}
+            .attr('height', {{ chart.height}})
+            {% endif %}
+            .call(chart);
+    {% endblock inject %}
+
+    {# extra chart attributes #}
+    {% if chart.extras %}
+        {{ chart.extras }}
+    {% endif %}
+
+    {# closing nv.addGraph #}
+    {% block close %}
+    });
+    {% endblock close %}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/cumulativelinechart.html b/airflow/_vendor/nvd3/templates/cumulativelinechart.html
new file mode 100644
index 0000000000000..546a3e8e55171
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/cumulativelinechart.html
@@ -0,0 +1,11 @@
+{# This template adds attributes unique
+   to cumulativeLineChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+{# calling super guarantees everying in content is also found here ...#}
+{{super()}}
+
+{% endblock body %}
+
diff --git a/airflow/_vendor/nvd3/templates/discretebarchart.html b/airflow/_vendor/nvd3/templates/discretebarchart.html
new file mode 100644
index 0000000000000..2e31ae4874be0
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/discretebarchart.html
@@ -0,0 +1,31 @@
+{# This is a dummy template, we can use that template to add attributes unique
+   to discreteBarChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+    {% block data %}
+        {{super()}}
+    {% endblock data %}
+
+    {% block init %}
+        {{super()}}
+    {% endblock init %}
+
+    {% block axes %}
+        {{super()}}
+    {% endblock axes %}
+    
+    {% block custoattr %}
+    {{super()}}
+    {% endblock custoattr %}
+
+    {% block inject %}
+    {{ super() }}
+    {% endblock inject %}
+    
+    {% block close %}
+    {{ super() }}
+    {% endblock close %}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/linebarwfocuschart.html b/airflow/_vendor/nvd3/templates/linebarwfocuschart.html
new file mode 100644
index 0000000000000..ad4866c8153f9
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/linebarwfocuschart.html
@@ -0,0 +1,60 @@
+{# This template adds attributes unique
+   to lineChart #}
+
+{% extends "content.html" %}
+{% block body %}
+    {% block data %}
+        data_{{ chart.name }}={{ chart.series_js }};
+    {% endblock data %}
+
+
+    {% block init %}
+        {{super()}}
+    {% endblock init %}
+    {% block axes %}
+        {{super()}}
+    {% endblock axes %}
+    {% block tooltip %}
+        {{super()}}
+    {% endblock tooltip %}
+
+    chart.showLegend({{chart.show_legend|lower}});
+
+    {# add custom chart attributes #}
+    {% for attr, value in chart.chart_attr.items() %}
+        {% if value is string and value.startswith(".") %}:
+            chart.{{ attr }}{{ value }};
+        {% else %}
+            chart.{{ attr }}({{ value }});
+        {% endif %}
+    {% endfor %}
+
+    {% if chart.x_axis_format == 'AM_PM' %}
+        function get_am_pm(d){
+            if (d > 12) {
+                d = d - 12; return (String(d) + 'PM');
+            }
+            else {
+                return (String(d) + 'AM');
+            }
+        };
+    {% else %}
+        chart.x(function(d,i) { return i });
+    {% endif %}
+
+    {% if chart.resize %}
+        nv.utils.windowResize(chart.update);
+    {% endif %}
+    {% block inject %}
+    {{super()}}
+    {% endblock inject %}
+
+		{% if chart.extras %}
+			{{ chart.extras }}
+		{% endif %}
+
+    {% block close %}
+    });
+    {% endblock close %}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/linechart.html b/airflow/_vendor/nvd3/templates/linechart.html
new file mode 100644
index 0000000000000..cf15d33041558
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/linechart.html
@@ -0,0 +1,47 @@
+{# This template adds attributes unique
+   to lineChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+    {% block data %}
+    {{super()}}
+    {% endblock data %}
+
+    {% block init %}
+    {{super()}}
+    {% endblock init %}
+
+    {% block axes %}
+    {{super()}}
+    {% endblock axes %}
+
+    {% if chart.x_axis_format == 'AM_PM' %}
+        function get_am_pm(d){
+            if (d > 12) {
+                d = d - 12; return (String(d) + 'PM');
+            }
+            else {
+                return (String(d) + 'AM');
+            }
+        };
+    {% endif %}
+
+    {% block legend %}
+    {{super()}}
+    {% endblock legend %}
+
+    {% block custoattr %}
+    {{super()}}
+    {% endblock custoattr %}
+
+    {% block inject %}
+    {{ super() }}
+    {% endblock inject %}
+
+    {% block close %}
+    {{ super() }}
+    {% endblock close %}
+
+{% endblock body %}
+
diff --git a/airflow/_vendor/nvd3/templates/lineplusbarchart.html b/airflow/_vendor/nvd3/templates/lineplusbarchart.html
new file mode 100644
index 0000000000000..73aeceacd2419
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/lineplusbarchart.html
@@ -0,0 +1,44 @@
+{# This template adds attributes unique
+   to linePlusBarChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+    {% block data %}
+    {{super()}}
+    {% endblock data %}
+
+    {% block init %}
+    {{super()}}
+    {% endblock init %}
+
+    {% block focus %}
+    {% if chart.focus_enable %}
+        chart.focusEnable(true);
+    {% else %}
+        chart.focusEnable(false);
+    {% endif %}
+    {% endblock focus %}
+
+    {% block axes %}
+    {{super()}}
+    {% endblock axes %}
+
+    {% block legend %}
+    {{super()}}
+    {% endblock legend %}
+
+    {% block custoattr %}
+    {{super()}}
+    {% endblock custoattr %}
+
+    {% block inject %}
+    {{ super() }}
+    {% endblock inject %}
+
+    {% block close %}
+    {{ super() }}
+    {% endblock close %}
+
+{% endblock body %}
+
diff --git a/airflow/_vendor/nvd3/templates/linewfocuschart.html b/airflow/_vendor/nvd3/templates/linewfocuschart.html
new file mode 100644
index 0000000000000..5abe983d7efc3
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/linewfocuschart.html
@@ -0,0 +1,10 @@
+{# This template adds attributes unique
+   to lineWithFocusChart #}
+
+{% extends "content.html" %}
+{% block body %}
+    
+{# calling super guarantees everying in content is also found here ...#}
+{{super()}}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/multibarchart.html b/airflow/_vendor/nvd3/templates/multibarchart.html
new file mode 100644
index 0000000000000..17eae7a634fef
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/multibarchart.html
@@ -0,0 +1,10 @@
+{# This template adds attributes unique
+   to multiBarChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+{# calling super guarantees everying in content is also found here ...#}
+{{super()}}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/multibarcharthorizontal.html b/airflow/_vendor/nvd3/templates/multibarcharthorizontal.html
new file mode 100644
index 0000000000000..17eae7a634fef
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/multibarcharthorizontal.html
@@ -0,0 +1,10 @@
+{# This template adds attributes unique
+   to multiBarChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+{# calling super guarantees everying in content is also found here ...#}
+{{super()}}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/page.html b/airflow/_vendor/nvd3/templates/page.html
new file mode 100644
index 0000000000000..2dd0f5d16f829
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/page.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8" />
+        {% for header_element in chart.header_css+chart.header_js %}
+        {{ header_element }}
+        {% endfor %}
+    </head>
+    <body>
+        {{ chart.content }}
+    </body>
+</html>
diff --git a/airflow/_vendor/nvd3/templates/piechart.html b/airflow/_vendor/nvd3/templates/piechart.html
new file mode 100644
index 0000000000000..a200e6d4a21bb
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/piechart.html
@@ -0,0 +1,80 @@
+{# This template adds attributes unique
+   to pieChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+    data_{{ chart.name }}={{ chart.series_js }};
+
+    nv.addGraph(function() {
+        var chart = nv.models.{{ chart.model }}(){% if chart.use_interactive_guideline %}.useInteractiveGuideline(true){% endif %};
+        chart.margin({top: {{ chart.margin_top }}, right: {{ chart.margin_right }}, bottom: {{ chart.margin_bottom }}, left: {{ chart.margin_left }}});
+        var datum = data_{{ chart.name }}[0].values;
+
+    {% if not chart.color_list and chart.color_category %}
+        chart.color(d3.scale.{{ chart.color_category }}().range());
+    {% endif %}
+
+    chart.tooltipContent(function(key, y, e, graph) {
+          var x = String(key);
+              {{ chart.tooltip_condition_string }}
+              tooltip_str = '<center><b>'+x+'</b></center>' + y;
+              return tooltip_str;
+              });
+    {# showLabels only supported in pieChart #}
+        chart.showLabels({{chart.show_labels|lower}});
+
+        {% if chart.donut %}
+            chart.donut(true);
+            chart.donutRatio({{ chart.donutRatio }});
+        {% else %}
+            chart.donut(false);
+        {% endif %}
+
+    chart.showLegend({{chart.show_legend|lower}});
+
+    {# add custom chart attributes #}
+    {% for attr, value in chart.chart_attr.items() %}
+        {% if value is string and value.startswith(".") %}:
+            chart.{{ attr }}{{ value }};
+        {% else %}
+            chart.{{ attr }}({{ value }});
+        {% endif %}
+    {% endfor %}
+
+    {% if chart.resize %}
+        nv.utils.windowResize(chart.update);
+    {% endif %}
+
+    {% if chart.color_list %}
+        var mycolor = new Array();
+        {% for color in chart.color_list %}
+            mycolor[{{ loop.index - 1}}] = "{{ color }}";
+        {% endfor %}
+    {% endif %}
+
+        chart
+            .x(function(d) { return d.label })
+            .y(function(d) { return d.value });
+
+    {% if chart.width %}
+        chart.width({{ chart.width }});
+    {% endif %}
+
+    {% if chart.height %}
+        chart.height({{ chart.height }});
+    {% endif %}
+
+    {% if chart.color_list %}
+        chart.color(mycolor);
+    {% endif %}
+
+    {% block inject %}
+    {{super()}}
+    {% endblock inject %}
+
+    {% block close %}
+    {{ super() }}
+    {% endblock close %}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/scatterchart.html b/airflow/_vendor/nvd3/templates/scatterchart.html
new file mode 100644
index 0000000000000..8c2adaae34cee
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/scatterchart.html
@@ -0,0 +1,52 @@
+{# This template adds attributes unique
+   to  scatterChart #}
+
+{% extends "content.html" %}
+{% block body %}
+
+    {% block data %}
+    {{super()}}
+    {% endblock data %}
+
+    {% block init %}
+    {{super()}}
+    {% endblock init %}
+
+    {% block axes %}
+    {{super()}}
+    {% endblock axes %}
+
+    {% if chart.x_axis_format == 'AM_PM' %}
+        function get_am_pm(d){
+            if (d > 12) {
+                d = d - 12; return (String(d) + 'PM');
+            }
+            else {
+                return (String(d) + 'AM');
+            }
+        };
+    {% endif %}
+
+    {% block legend %}
+    {{super()}}
+    {% endblock legend %}
+
+    {% block custoattr %}
+    {{super()}}
+    {% endblock custoattr %}
+
+    {% block inject %}
+
+    chart
+        .showDistX(true)
+        .showDistY(true)
+        .color(d3.scale.category10().range());
+
+    {{ super() }}
+    {% endblock inject %}
+
+    {% block close %}
+    {{ super() }}
+    {% endblock close %}
+
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/templates/stackedareachart.html b/airflow/_vendor/nvd3/templates/stackedareachart.html
new file mode 100644
index 0000000000000..b70833d2b385d
--- /dev/null
+++ b/airflow/_vendor/nvd3/templates/stackedareachart.html
@@ -0,0 +1,7 @@
+{# This is a dummy template, we can use that template to add attributes unique
+   to stackedareachart #}
+
+{% extends "content.html" %}
+{% block body %}
+    {{ super() }}
+{% endblock body %}
diff --git a/airflow/_vendor/nvd3/translator.py b/airflow/_vendor/nvd3/translator.py
new file mode 100644
index 0000000000000..ffde2c2a1cec9
--- /dev/null
+++ b/airflow/_vendor/nvd3/translator.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+
+
+class Tag(object):
+    """Tag class"""
+
+    def __init__(self, content=None):
+        self.content = content
+        self.attrs = ' '.join(['%s="%s"' % (attr, value)
+                               for attr, value in self.attrs])
+
+    def __str__(self):
+        return '<%s%s>\n    %s\n</%s>' % (self.name,
+                                          ' ' + self.attrs if self.attrs else '',
+                                          self.content,
+                                          self.name)
+
+
+class ScriptTag(Tag):
+    name = 'script'
+    attrs = (('type', 'text/javascript'),)
+
+
+class AnonymousFunction(object):
+    def __init__(self, arguments, content):
+        self.arguments = arguments
+        self.content = content
+
+    def __str__(self):
+        return 'function(%s) { %s }' % (self.arguments, self.content)
+
+
+class Function(object):
+
+    def __init__(self, name):
+        self.name = name
+        self._calls = []
+
+    def __str__(self):
+        operations = [self.name]
+        operations.extend(str(call) for call in self._calls)
+        return '%s' % ('.'.join(operations),)
+
+    def __getattr__(self, attr):
+        self._calls.append(attr)
+        return self
+
+    def __call__(self, *args):
+        if not args:
+            self._calls[-1] = self._calls[-1] + '()'
+        else:
+            arguments = ','.join([str(arg) for arg in args])
+            self._calls[-1] = self._calls[-1] + '(%s)' % (arguments,)
+        return self
+
+
+class Assignment(object):
+
+    def __init__(self, key, value, scoped=True):
+        self.key = key
+        self.value = value
+        self.scoped = scoped
+
+    def __str__(self):
+        return '%s%s = %s;' % ('var ' if self.scoped else '', self.key, self.value)
+
+
+def indent(func):
+    # TODO: Add indents to function str
+    return str(func)
diff --git a/airflow/_vendor/slugify/LICENSE b/airflow/_vendor/slugify/LICENSE
new file mode 100644
index 0000000000000..82af695f594e8
--- /dev/null
+++ b/airflow/_vendor/slugify/LICENSE
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) Val Neekman @ Neekware Inc. http://neekware.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/airflow/_vendor/slugify/__init__.py b/airflow/_vendor/slugify/__init__.py
new file mode 100644
index 0000000000000..7358b998cd543
--- /dev/null
+++ b/airflow/_vendor/slugify/__init__.py
@@ -0,0 +1,6 @@
+from .slugify import *
+
+
+__author__ = 'Val Neekman @ Neekware Inc. [@vneekman]'
+__description__ = 'A Python slugify application that also handles Unicode'
+__version__ = '2.0.1'
diff --git a/airflow/_vendor/slugify/slugify.py b/airflow/_vendor/slugify/slugify.py
new file mode 100644
index 0000000000000..0e9886d827138
--- /dev/null
+++ b/airflow/_vendor/slugify/slugify.py
@@ -0,0 +1,185 @@
+import re
+import unicodedata
+import types
+import sys
+
+try:
+    from htmlentitydefs import name2codepoint
+    _unicode = unicode
+    _unicode_type = types.UnicodeType
+except ImportError:
+    from html.entities import name2codepoint
+    _unicode = str
+    _unicode_type = str
+    unichr = chr
+
+import text_unidecode as unidecode
+
+__all__ = ['slugify', 'smart_truncate']
+
+
+CHAR_ENTITY_PATTERN = re.compile(r'&(%s);' % '|'.join(name2codepoint))
+DECIMAL_PATTERN = re.compile(r'&#(\d+);')
+HEX_PATTERN = re.compile(r'&#x([\da-fA-F]+);')
+QUOTE_PATTERN = re.compile(r'[\']+')
+ALLOWED_CHARS_PATTERN = re.compile(r'[^-a-z0-9]+')
+ALLOWED_CHARS_PATTERN_WITH_UPPERCASE = re.compile(r'[^-a-zA-Z0-9]+')
+DUPLICATE_DASH_PATTERN = re.compile(r'-{2,}')
+NUMBERS_PATTERN = re.compile(r'(?<=\d),(?=\d)')
+DEFAULT_SEPARATOR = '-'
+
+
+def smart_truncate(string, max_length=0, word_boundary=False, separator=' ', save_order=False):
+    """
+    Truncate a string.
+    :param string (str): string for modification
+    :param max_length (int): output string length
+    :param word_boundary (bool):
+    :param save_order (bool): if True then word order of output string is like input string
+    :param separator (str): separator between words
+    :return:
+    """
+
+    string = string.strip(separator)
+
+    if not max_length:
+        return string
+
+    if len(string) < max_length:
+        return string
+
+    if not word_boundary:
+        return string[:max_length].strip(separator)
+
+    if separator not in string:
+        return string[:max_length]
+
+    truncated = ''
+    for word in string.split(separator):
+        if word:
+            next_len = len(truncated) + len(word)
+            if next_len < max_length:
+                truncated += '{0}{1}'.format(word, separator)
+            elif next_len == max_length:
+                truncated += '{0}'.format(word)
+                break
+            else:
+                if save_order:
+                    break
+    if not truncated: # pragma: no cover
+        truncated = string[:max_length]
+    return truncated.strip(separator)
+
+
+def slugify(text, entities=True, decimal=True, hexadecimal=True, max_length=0, word_boundary=False,
+            separator=DEFAULT_SEPARATOR, save_order=False, stopwords=(), regex_pattern=None, lowercase=True,
+            replacements=()):
+    """
+    Make a slug from the given text.
+    :param text (str): initial text
+    :param entities (bool):
+    :param decimal (bool):
+    :param hexadecimal (bool):
+    :param max_length (int): output string length
+    :param word_boundary (bool):
+    :param save_order (bool): if parameter is True and max_length > 0 return whole words in the initial order
+    :param separator (str): separator between words
+    :param stopwords (iterable): words to discount
+    :param regex_pattern (str): regex pattern for allowed characters
+    :param lowercase (bool): activate case sensitivity by setting it to False
+    :param replacements (iterable): list of replacement rules e.g. [['|', 'or'], ['%', 'percent']]
+    :return (str):
+    """
+
+    # user-specific replacements
+    if replacements:
+        for old, new in replacements:
+            text = text.replace(old, new)
+
+    # ensure text is unicode
+    if not isinstance(text, _unicode_type):
+        text = _unicode(text, 'utf-8', 'ignore')
+
+    # replace quotes with dashes - pre-process
+    text = QUOTE_PATTERN.sub(DEFAULT_SEPARATOR, text)
+
+    # decode unicode
+    text = unidecode.unidecode(text)
+
+    # ensure text is still in unicode
+    if not isinstance(text, _unicode_type):
+        text = _unicode(text, 'utf-8', 'ignore')
+
+    # character entity reference
+    if entities:
+        text = CHAR_ENTITY_PATTERN.sub(lambda m: unichr(name2codepoint[m.group(1)]), text)
+
+    # decimal character reference
+    if decimal:
+        try:
+            text = DECIMAL_PATTERN.sub(lambda m: unichr(int(m.group(1))), text)
+        except Exception:
+            pass
+
+    # hexadecimal character reference
+    if hexadecimal:
+        try:
+            text = HEX_PATTERN.sub(lambda m: unichr(int(m.group(1), 16)), text)
+        except Exception:
+            pass
+
+    # translate
+    text = unicodedata.normalize('NFKD', text)
+    if sys.version_info < (3,):
+        text = text.encode('ascii', 'ignore')
+
+    # make the text lowercase (optional)
+    if lowercase:
+        text = text.lower()
+
+    # remove generated quotes -- post-process
+    text = QUOTE_PATTERN.sub('', text)
+
+    # cleanup numbers
+    text = NUMBERS_PATTERN.sub('', text)
+
+    # replace all other unwanted characters
+    if lowercase:
+        pattern = regex_pattern or ALLOWED_CHARS_PATTERN
+    else:
+        pattern = regex_pattern or ALLOWED_CHARS_PATTERN_WITH_UPPERCASE
+    text = re.sub(pattern, DEFAULT_SEPARATOR, text)
+
+    # remove redundant
+    text = DUPLICATE_DASH_PATTERN.sub(DEFAULT_SEPARATOR, text).strip(DEFAULT_SEPARATOR)
+
+    # remove stopwords
+    if stopwords:
+        if lowercase:
+            stopwords_lower = [s.lower() for s in stopwords]
+            words = [w for w in text.split(DEFAULT_SEPARATOR) if w not in stopwords_lower]
+        else:
+            words = [w for w in text.split(DEFAULT_SEPARATOR) if w not in stopwords]
+        text = DEFAULT_SEPARATOR.join(words)
+
+    # finalize user-specific replacements
+    if replacements:
+        for old, new in replacements:
+            text = text.replace(old, new)
+
+    # smart truncate if requested
+    if max_length > 0:
+        text = smart_truncate(text, max_length, word_boundary, DEFAULT_SEPARATOR, save_order)
+
+    if separator != DEFAULT_SEPARATOR:
+        text = text.replace(DEFAULT_SEPARATOR, separator)
+
+    return text
+
+
+def main(): # pragma: no cover
+    if len(sys.argv) < 2:
+        print("Usage %s TEXT TO SLUGIFY" % sys.argv[0])
+    else:
+        text = ' '.join(sys.argv[1:])
+        print(slugify(text))
diff --git a/airflow/api/__init__.py b/airflow/api/__init__.py
index b4a2f8f5bc36f..67cb51d199e63 100644
--- a/airflow/api/__init__.py
+++ b/airflow/api/__init__.py
@@ -19,13 +19,15 @@
 
 from __future__ import print_function
 
+from typing import Any
+
 from airflow.exceptions import AirflowException
 from airflow import configuration as conf
 from importlib import import_module
 
 from airflow.utils.log.logging_mixin import LoggingMixin
 
-api_auth = None
+api_auth = None  # type: Any
 
 log = LoggingMixin().log
 
diff --git a/airflow/api/auth/backend/kerberos_auth.py b/airflow/api/auth/backend/kerberos_auth.py
index 7e560fb296dad..1b80e2efeb0cd 100644
--- a/airflow/api/auth/backend/kerberos_auth.py
+++ b/airflow/api/auth/backend/kerberos_auth.py
@@ -27,7 +27,7 @@
 from airflow import configuration as conf
 
 from flask import Response
-from flask import _request_ctx_stack as stack
+from flask import _request_ctx_stack as stack  # type: ignore
 from flask import make_response
 from flask import request
 from flask import g
diff --git a/airflow/api/common/experimental/delete_dag.py b/airflow/api/common/experimental/delete_dag.py
index b9ce736b48dfc..365997638b64c 100644
--- a/airflow/api/common/experimental/delete_dag.py
+++ b/airflow/api/common/experimental/delete_dag.py
@@ -19,13 +19,22 @@
 
 from sqlalchemy import or_
 
-from airflow import models, settings
+from airflow import models
+from airflow.models.taskfail import TaskFail
+from airflow.utils.db import provide_session
 from airflow.exceptions import DagNotFound, DagFileExists
 
 
-def delete_dag(dag_id):
-    session = settings.Session()
-
+@provide_session
+def delete_dag(dag_id, keep_records_in_log=True, session=None):
+    """
+    :param dag_id: the dag_id of the DAG to delete
+    :type dag_id: str
+    :param keep_records_in_log: whether keep records of the given dag_id
+        in the Log table in the backend database (for reasons like auditing).
+        The default value is True.
+    :type keep_records_in_log: bool
+    """
     DM = models.DagModel
     dag = session.query(DM).filter(DM.dag_id == dag_id).first()
     if dag is None:
@@ -39,16 +48,16 @@ def delete_dag(dag_id):
     count = 0
 
     # noinspection PyUnresolvedReferences,PyProtectedMember
-    for m in models.Base._decl_class_registry.values():
+    for m in models.base.Base._decl_class_registry.values():
         if hasattr(m, "dag_id"):
+            if keep_records_in_log and m.__name__ == 'Log':
+                continue
             cond = or_(m.dag_id == dag_id, m.dag_id.like(dag_id + ".%"))
             count += session.query(m).filter(cond).delete(synchronize_session='fetch')
 
     if dag.is_subdag:
         p, c = dag_id.rsplit(".", 1)
-        for m in models.DagRun, models.TaskFail, models.TaskInstance:
+        for m in models.DagRun, TaskFail, models.TaskInstance:
             count += session.query(m).filter(m.dag_id == p, m.task_id == c).delete()
 
-    session.commit()
-
     return count
diff --git a/airflow/api/common/experimental/get_code.py b/airflow/api/common/experimental/get_code.py
new file mode 100644
index 0000000000000..f082cb03837da
--- /dev/null
+++ b/airflow/api/common/experimental/get_code.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.exceptions import AirflowException, DagNotFound
+from airflow import models, settings
+from airflow.www import utils as wwwutils
+
+
+def get_code(dag_id):
+    """Return python code of a given dag_id."""
+    session = settings.Session()
+    DM = models.DagModel
+    dag = session.query(DM).filter(DM.dag_id == dag_id).first()
+    session.close()
+    # Check DAG exists.
+    if dag is None:
+        error_message = "Dag id {} not found".format(dag_id)
+        raise DagNotFound(error_message)
+
+    try:
+        with wwwutils.open_maybe_zipped(dag.fileloc, 'r') as f:
+            code = f.read()
+            return code
+    except IOError as e:
+        error_message = "Error {} while reading Dag id {} Code".format(str(e), dag_id)
+        raise AirflowException(error_message)
diff --git a/airflow/api/common/experimental/get_dag_runs.py b/airflow/api/common/experimental/get_dag_runs.py
index 63b1f993d392d..739905d5df903 100644
--- a/airflow/api/common/experimental/get_dag_runs.py
+++ b/airflow/api/common/experimental/get_dag_runs.py
@@ -22,7 +22,7 @@
 from airflow.models import DagBag, DagRun
 
 
-def get_dag_runs(dag_id, state=None):
+def get_dag_runs(dag_id, state=None, run_url_route='Airflow.graph'):
     """
     Returns a list of Dag Runs for a specific DAG ID.
     :param dag_id: String identifier of a DAG
@@ -48,7 +48,7 @@ def get_dag_runs(dag_id, state=None):
             'execution_date': run.execution_date.isoformat(),
             'start_date': ((run.start_date or '') and
                            run.start_date.isoformat()),
-            'dag_run_url': url_for('Airflow.graph', dag_id=run.dag_id,
+            'dag_run_url': url_for(run_url_route, dag_id=run.dag_id,
                                    execution_date=run.execution_date)
         })
 
diff --git a/airflow/api/common/experimental/mark_tasks.py b/airflow/api/common/experimental/mark_tasks.py
index e9e4fec2235a5..b761d8dc5b594 100644
--- a/airflow/api/common/experimental/mark_tasks.py
+++ b/airflow/api/common/experimental/mark_tasks.py
@@ -17,15 +17,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from sqlalchemy import or_
+
 from airflow.jobs import BackfillJob
 from airflow.models import DagRun, TaskInstance
 from airflow.operators.subdag_operator import SubDagOperator
-from airflow.settings import Session
 from airflow.utils import timezone
+from airflow.utils.db import provide_session
 from airflow.utils.state import State
 
-from sqlalchemy import or_
-
 
 def _create_dagruns(dag, execution_dates, state, run_id_template):
     """
@@ -53,8 +53,9 @@ def _create_dagruns(dag, execution_dates, state, run_id_template):
     return drs
 
 
+@provide_session
 def set_state(task, execution_date, upstream=False, downstream=False,
-              future=False, past=False, state=State.SUCCESS, commit=False):
+              future=False, past=False, state=State.SUCCESS, commit=False, session=None):
     """
     Set the state of a task instance and if needed its relatives. Can set state
     for future tasks (calculated from execution_date) and retroactively
@@ -70,14 +71,11 @@ def set_state(task, execution_date, upstream=False, downstream=False,
     :param past: Retroactively mark all tasks starting from start_date of the DAG
     :param state: State to which the tasks need to be set
     :param commit: Commit tasks to be altered to the database
+    :param session: database session
     :return: list of tasks that have been created and updated
     """
     assert timezone.is_localized(execution_date)
 
-    # microseconds are supported by the database, but is not handled
-    # correctly by airflow on e.g. the filesystem and in other places
-    execution_date = execution_date.replace(microsecond=0)
-
     assert task.dag is not None
     dag = task.dag
 
@@ -123,7 +121,6 @@ def set_state(task, execution_date, upstream=False, downstream=False,
     # go through subdagoperators and create dag runs. We will only work
     # within the scope of the subdag. We wont propagate to the parent dag,
     # but we will propagate from parent to subdag.
-    session = Session()
     dags = [dag]
     sub_dag_ids = []
     while len(dags) > 0:
@@ -179,27 +176,80 @@ def set_state(task, execution_date, upstream=False, downstream=False,
             tis_altered += qry_sub_dag.with_for_update().all()
         for ti in tis_altered:
             ti.state = state
-        session.commit()
     else:
         tis_altered = qry_dag.all()
         if len(sub_dag_ids) > 0:
             tis_altered += qry_sub_dag.all()
 
-    session.expunge_all()
-    session.close()
-
     return tis_altered
 
 
-def set_dag_run_state(dag, execution_date, state=State.SUCCESS, commit=False):
+@provide_session
+def _set_dag_run_state(dag_id, execution_date, state, session=None):
     """
-    Set the state of a dag run and all task instances associated with the dag
-    run for a specific execution date.
+    Helper method that set dag run state in the DB.
+    :param dag_id: dag_id of target dag run
+    :param execution_date: the execution date from which to start looking
+    :param state: target state
+    :param session: database session
+    """
+    DR = DagRun
+    dr = session.query(DR).filter(
+        DR.dag_id == dag_id,
+        DR.execution_date == execution_date
+    ).one()
+    dr.state = state
+    if state == State.RUNNING:
+        dr.start_date = timezone.utcnow()
+        dr.end_date = None
+    else:
+        dr.end_date = timezone.utcnow()
+    session.merge(dr)
+
+
+@provide_session
+def set_dag_run_state_to_success(dag, execution_date, commit=False, session=None):
+    """
+    Set the dag run for a specific execution date and its task instances
+    to success.
     :param dag: the DAG of which to alter state
     :param execution_date: the execution date from which to start looking
-    :param state: the state to which the DAG need to be set
     :param commit: commit DAG and tasks to be altered to the database
-    :return: list of tasks that have been created and updated
+    :param session: database session
+    :return: If commit is true, list of tasks that have been updated,
+             otherwise list of tasks that will be updated
+    :raises: AssertionError if dag or execution_date is invalid
+    """
+    res = []
+
+    if not dag or not execution_date:
+        return res
+
+    # Mark the dag run to success.
+    if commit:
+        _set_dag_run_state(dag.dag_id, execution_date, State.SUCCESS, session)
+
+    # Mark all task instances of the dag run to success.
+    for task in dag.tasks:
+        task.dag = dag
+        new_state = set_state(task=task, execution_date=execution_date,
+                              state=State.SUCCESS, commit=commit)
+        res.extend(new_state)
+
+    return res
+
+
+@provide_session
+def set_dag_run_state_to_failed(dag, execution_date, commit=False, session=None):
+    """
+    Set the dag run for a specific execution date and its running task instances
+    to failed.
+    :param dag: the DAG of which to alter state
+    :param execution_date: the execution date from which to start looking
+    :param commit: commit DAG and tasks to be altered to the database
+    :param session: database session
+    :return: If commit is true, list of tasks that have been updated,
+             otherwise list of tasks that will be updated
     :raises: AssertionError if dag or execution_date is invalid
     """
     res = []
@@ -207,18 +257,47 @@ def set_dag_run_state(dag, execution_date, state=State.SUCCESS, commit=False):
     if not dag or not execution_date:
         return res
 
-    # Mark all task instances in the dag run
+    # Mark the dag run to failed.
+    if commit:
+        _set_dag_run_state(dag.dag_id, execution_date, State.FAILED, session)
+
+    # Mark only RUNNING task instances.
+    TI = TaskInstance
+    task_ids = [task.task_id for task in dag.tasks]
+    tis = session.query(TI).filter(
+        TI.dag_id == dag.dag_id,
+        TI.execution_date == execution_date,
+        TI.task_id.in_(task_ids)).filter(TI.state == State.RUNNING)
+    task_ids_of_running_tis = [ti.task_id for ti in tis]
     for task in dag.tasks:
+        if task.task_id not in task_ids_of_running_tis:
+            continue
         task.dag = dag
         new_state = set_state(task=task, execution_date=execution_date,
-                              state=state, commit=commit)
+                              state=State.FAILED, commit=commit)
         res.extend(new_state)
 
-    # Mark the dag run
+    return res
+
+
+@provide_session
+def set_dag_run_state_to_running(dag, execution_date, commit=False, session=None):
+    """
+    Set the dag run for a specific execution date to running.
+    :param dag: the DAG of which to alter state
+    :param execution_date: the execution date from which to start looking
+    :param commit: commit DAG and tasks to be altered to the database
+    :param session: database session
+    :return: If commit is true, list of tasks that have been updated,
+             otherwise list of tasks that will be updated
+    """
+    res = []
+    if not dag or not execution_date:
+        return res
+
+    # Mark the dag run to running.
     if commit:
-        drs = DagRun.find(dag.dag_id, execution_date=execution_date)
-        for dr in drs:
-            dr.dag = dag
-            dr.update_state()
+        _set_dag_run_state(dag.dag_id, execution_date, State.RUNNING, session)
 
+    # To keep the return type consistent with the other similar functions.
     return res
diff --git a/airflow/api/common/experimental/trigger_dag.py b/airflow/api/common/experimental/trigger_dag.py
index 86be6aa544089..3989de4ebd8f0 100644
--- a/airflow/api/common/experimental/trigger_dag.py
+++ b/airflow/api/common/experimental/trigger_dag.py
@@ -59,7 +59,10 @@ def _trigger_dag(
 
     run_conf = None
     if conf:
-        run_conf = json.loads(conf)
+        if type(conf) is dict:
+            run_conf = conf
+        else:
+            run_conf = json.loads(conf)
 
     triggers = list()
     dags_to_trigger = list()
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index b56e3253276bc..422942f14944d 100644
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -19,7 +19,7 @@
 # under the License.
 
 from __future__ import print_function
-from backports.configparser import NoSectionError
+import importlib
 import logging
 
 import os
@@ -33,7 +33,7 @@
 import reprlib
 import argparse
 from builtins import input
-from collections import namedtuple
+
 from airflow.utils.timezone import parse as parsedate
 import json
 from tabulate import tabulate
@@ -48,6 +48,7 @@
 import psutil
 import re
 from urllib.parse import urlunparse
+from typing import Any
 
 import airflow
 from airflow import api
@@ -55,13 +56,11 @@
 from airflow import configuration as conf
 from airflow.exceptions import AirflowException, AirflowWebServerTimeout
 from airflow.executors import GetDefaultExecutor
-from airflow.models import (DagModel, DagBag, TaskInstance,
-                            DagPickle, DagRun, Variable, DagStat,
-                            Connection, DAG)
-
+from airflow.models import DagModel, DagBag, TaskInstance, DagRun, Variable, DAG
+from airflow.models.connection import Connection
+from airflow.models.dagpickle import DagPickle
 from airflow.ti_deps.dep_context import (DepContext, SCHEDULER_DEPS)
-from airflow.utils import cli as cli_utils
-from airflow.utils import db as db_utils
+from airflow.utils import cli as cli_utils, db
 from airflow.utils.net import get_hostname
 from airflow.utils.log.logging_mixin import (LoggingMixin, redirect_stderr,
                                              redirect_stdout)
@@ -70,16 +69,20 @@
 from airflow.www_rbac.app import create_app as create_app_rbac
 from airflow.www_rbac.app import cached_appbuilder
 
-from sqlalchemy import func
 from sqlalchemy.orm import exc
 
 api.load_auth()
-api_module = import_module(conf.get('cli', 'api_client'))
+api_module = import_module(conf.get('cli', 'api_client'))  # type: Any
 api_client = api_module.Client(api_base_url=conf.get('cli', 'endpoint_url'),
                                auth=api.api_auth.client_auth)
 
 log = LoggingMixin().log
 
+DAGS_FOLDER = settings.DAGS_FOLDER
+
+if "BUILDING_AIRFLOW_DOCS" in os.environ:
+    DAGS_FOLDER = '[AIRFLOW_HOME]/dags'
+
 
 def sigint_handler(sig, frame):
     sys.exit(0)
@@ -116,24 +119,20 @@ def setup_logging(filename):
 
 def setup_locations(process, pid=None, stdout=None, stderr=None, log=None):
     if not stderr:
-        stderr = os.path.join(os.path.expanduser(settings.AIRFLOW_HOME),
-                              'airflow-{}.err'.format(process))
+        stderr = os.path.join(settings.AIRFLOW_HOME, 'airflow-{}.err'.format(process))
     if not stdout:
-        stdout = os.path.join(os.path.expanduser(settings.AIRFLOW_HOME),
-                              'airflow-{}.out'.format(process))
+        stdout = os.path.join(settings.AIRFLOW_HOME, 'airflow-{}.out'.format(process))
     if not log:
-        log = os.path.join(os.path.expanduser(settings.AIRFLOW_HOME),
-                           'airflow-{}.log'.format(process))
+        log = os.path.join(settings.AIRFLOW_HOME, 'airflow-{}.log'.format(process))
     if not pid:
-        pid = os.path.join(os.path.expanduser(settings.AIRFLOW_HOME),
-                           'airflow-{}.pid'.format(process))
+        pid = os.path.join(settings.AIRFLOW_HOME, 'airflow-{}.pid'.format(process))
 
     return pid, stdout, stderr, log
 
 
 def process_subdir(subdir):
     if subdir:
-        subdir = subdir.replace('DAGS_FOLDER', settings.DAGS_FOLDER)
+        subdir = subdir.replace('DAGS_FOLDER', DAGS_FOLDER)
         subdir = os.path.abspath(os.path.expanduser(subdir))
         return subdir
 
@@ -215,6 +214,7 @@ def backfill(args, dag=None):
             verbose=args.verbose,
             conf=run_conf,
             rerun_failed_tasks=args.rerun_failed_tasks,
+            run_backwards=args.run_backwards
         )
 
 
@@ -267,6 +267,7 @@ def _tabulate(pools):
                                  tablefmt="fancy_grid")
 
     try:
+        imp = getattr(args, 'import')
         if args.get is not None:
             pools = [api_client.get_pool(name=args.get)]
         elif args.set:
@@ -275,6 +276,14 @@ def _tabulate(pools):
                                             description=args.set[2])]
         elif args.delete:
             pools = [api_client.delete_pool(name=args.delete)]
+        elif imp:
+            if os.path.exists(imp):
+                pools = pool_import_helper(imp)
+            else:
+                print("Missing pools file.")
+                pools = api_client.get_pools()
+        elif args.export:
+            pools = pool_export_helper(args.export)
         else:
             pools = api_client.get_pools()
     except (AirflowException, IOError) as err:
@@ -283,6 +292,43 @@ def _tabulate(pools):
         log.info(_tabulate(pools=pools))
 
 
+def pool_import_helper(filepath):
+    with open(filepath, 'r') as poolfile:
+        pl = poolfile.read()
+    try:
+        d = json.loads(pl)
+    except Exception as e:
+        print("Please check the validity of the json file: " + str(e))
+    else:
+        try:
+            pools = []
+            n = 0
+            for k, v in d.items():
+                if isinstance(v, dict) and len(v) == 2:
+                    pools.append(api_client.create_pool(name=k,
+                                                        slots=v["slots"],
+                                                        description=v["description"]))
+                    n += 1
+                else:
+                    pass
+        except Exception:
+            pass
+        finally:
+            print("{} of {} pool(s) successfully updated.".format(n, len(d)))
+            return pools
+
+
+def pool_export_helper(filepath):
+    pool_dict = {}
+    pools = api_client.get_pools()
+    for pool in pools:
+        pool_dict[pool[0]] = {"slots": pool[1], "description": pool[2]}
+    with open(filepath, 'w') as poolfile:
+        poolfile.write(json.dumps(pool_dict, sort_keys=True, indent=4))
+    print("{} pools successfully exported to {}".format(len(pool_dict), filepath))
+    return pools
+
+
 @cli_utils.action_logging
 def variables(args):
     if args.get:
@@ -294,10 +340,8 @@ def variables(args):
         except ValueError as e:
             print(e)
     if args.delete:
-        session = settings.Session()
-        session.query(Variable).filter_by(key=args.delete).delete()
-        session.commit()
-        session.close()
+        with db.create_session() as session:
+            session.query(Variable).filter_by(key=args.delete).delete()
     if args.set:
         Variable.set(args.set[0], args.set[1])
     # Work around 'import' as a reserved keyword
@@ -311,10 +355,10 @@ def variables(args):
         export_helper(args.export)
     if not (args.set or args.get or imp or args.export or args.delete):
         # list all variables
-        session = settings.Session()
-        vars = session.query(Variable)
-        msg = "\n".join(var.key for var in vars)
-        print(msg)
+        with db.create_session() as session:
+            vars = session.query(Variable)
+            msg = "\n".join(var.key for var in vars)
+            print(msg)
 
 
 def import_helper(filepath):
@@ -341,19 +385,17 @@ def import_helper(filepath):
 
 
 def export_helper(filepath):
-    session = settings.Session()
-    qry = session.query(Variable).all()
-    session.close()
-
     var_dict = {}
-    d = json.JSONDecoder()
-    for var in qry:
-        val = None
-        try:
-            val = d.decode(var.val)
-        except Exception:
-            val = var.val
-        var_dict[var.key] = val
+    with db.create_session() as session:
+        qry = session.query(Variable).all()
+
+        d = json.JSONDecoder()
+        for var in qry:
+            try:
+                val = d.decode(var.val)
+            except Exception:
+                val = var.val
+            var_dict[var.key] = val
 
     with open(filepath, 'w') as varfile:
         varfile.write(json.dumps(var_dict, sort_keys=True, indent=4))
@@ -373,14 +415,12 @@ def unpause(args, dag=None):
 def set_is_paused(is_paused, args, dag=None):
     dag = dag or get_dag(args)
 
-    session = settings.Session()
-    dm = session.query(DagModel).filter(
-        DagModel.dag_id == dag.dag_id).first()
-    dm.is_paused = is_paused
-    session.commit()
+    with db.create_session() as session:
+        dm = session.query(DagModel).filter(DagModel.dag_id == dag.dag_id).first()
+        dm.is_paused = is_paused
+        session.commit()
 
-    msg = "Dag: {}, paused: {}".format(dag, str(dag.is_paused))
-    print(msg)
+    print("Dag: {}, paused: {}".format(dag, str(dag.is_paused)))
 
 
 def _run(args, dag, ti):
@@ -406,14 +446,12 @@ def _run(args, dag, ti):
         if args.ship_dag:
             try:
                 # Running remotely, so pickling the DAG
-                session = settings.Session()
-                pickle = DagPickle(dag)
-                session.add(pickle)
-                session.commit()
-                pickle_id = pickle.id
-                # TODO: This should be written to a log
-                print('Pickled dag {dag} as pickle_id:{pickle_id}'
-                      .format(**locals()))
+                with db.create_session() as session:
+                    pickle = DagPickle(dag)
+                    session.add(pickle)
+                    pickle_id = pickle.id
+                    # TODO: This should be written to a log
+                    print('Pickled dag {dag} as pickle_id:{pickle_id}'.format(**locals()))
             except Exception as e:
                 print('Could not pickle the DAG')
                 print(e)
@@ -450,19 +488,7 @@ def run(args, dag=None):
         if os.path.exists(args.cfg_path):
             os.remove(args.cfg_path)
 
-        # Do not log these properties since some may contain passwords.
-        # This may also set default values for database properties like
-        # core.sql_alchemy_pool_size
-        # core.sql_alchemy_pool_recycle
-        for section, config in conf_dict.items():
-            for option, value in config.items():
-                try:
-                    conf.set(section, option, value)
-                except NoSectionError:
-                    log.error('Section {section} Option {option} '
-                              'does not exist in the config!'.format(section=section,
-                                                                     option=option))
-
+        conf.conf.read_dict(conf_dict, source=args.cfg_path)
         settings.configure_vars()
 
     # IMPORTANT, have to use the NullPool, otherwise, each "run" command may leave
@@ -474,13 +500,12 @@ def run(args, dag=None):
     if not args.pickle and not dag:
         dag = get_dag(args)
     elif not dag:
-        session = settings.Session()
-        log.info('Loading pickle id {args.pickle}'.format(args=args))
-        dag_pickle = session.query(
-            DagPickle).filter(DagPickle.id == args.pickle).first()
-        if not dag_pickle:
-            raise AirflowException("Who hid the pickle!? [missing pickle]")
-        dag = dag_pickle.pickle
+        with db.create_session() as session:
+            log.info('Loading pickle id %s', args.pickle)
+            dag_pickle = session.query(DagPickle).filter(DagPickle.id == args.pickle).first()
+            if not dag_pickle:
+                raise AirflowException("Who hid the pickle!? [missing pickle]")
+            dag = dag_pickle.pickle
 
     task = dag.get_task(task_id=args.task_id)
     ti = TaskInstance(task, args.execution_date)
@@ -551,6 +576,42 @@ def dag_state(args):
     print(dr[0].state if len(dr) > 0 else None)
 
 
+@cli_utils.action_logging
+def next_execution(args):
+    """
+    Returns the next execution datetime of a DAG at the command line.
+    >>> airflow next_execution tutorial
+    2018-08-31 10:38:00
+    """
+    dag = get_dag(args)
+
+    if dag.is_paused:
+        print("[INFO] Please be reminded this DAG is PAUSED now.")
+
+    if dag.latest_execution_date:
+        next_execution_dttm = dag.following_schedule(dag.latest_execution_date)
+
+        if next_execution_dttm is None:
+            print("[WARN] No following schedule can be found. " +
+                  "This DAG may have schedule interval '@once' or `None`.")
+
+        print(next_execution_dttm)
+    else:
+        print("[WARN] Only applicable when there is execution record found for the DAG.")
+        print(None)
+
+
+@cli_utils.action_logging
+def rotate_fernet_key(args):
+    session = settings.Session()
+    for conn in session.query(Connection).filter(
+            Connection.is_encrypted | Connection.is_extra_encrypted):
+        conn.rotate_fernet_key()
+    for var in session.query(Variable).filter(Variable.is_encrypted):
+        var.rotate_fernet_key()
+    session.commit()
+
+
 @cli_utils.action_logging
 def list_dags(args):
     dagbag = DagBag(process_subdir(args.subdir))
@@ -578,6 +639,11 @@ def list_tasks(args, dag=None):
 
 @cli_utils.action_logging
 def test(args, dag=None):
+    # We want log outout from operators etc to show up here. Normally
+    # airflow.task would redirect to a file, but here we want it to propagate
+    # up to the normal airflow handler.
+    logging.getLogger('airflow.task').propagate = True
+
     dag = dag or get_dag(args)
 
     task = dag.get_task(task_id=args.task_id)
@@ -587,10 +653,20 @@ def test(args, dag=None):
         task.params.update(passed_in_params)
     ti = TaskInstance(task, args.execution_date)
 
-    if args.dry_run:
-        ti.dry_run()
-    else:
-        ti.run(ignore_task_deps=True, ignore_ti_state=True, test_mode=True)
+    try:
+        if args.dry_run:
+            ti.dry_run()
+        else:
+            ti.run(ignore_task_deps=True, ignore_ti_state=True, test_mode=True)
+    except Exception:
+        if args.post_mortem:
+            try:
+                debugger = importlib.import_module("ipdb")
+            except ImportError:
+                debugger = importlib.import_module("pdb")
+            debugger.post_mortem()
+        else:
+            raise
 
 
 @cli_utils.action_logging
@@ -629,7 +705,9 @@ def clear(args):
         only_failed=args.only_failed,
         only_running=args.only_running,
         confirm_prompt=not args.no_confirm,
-        include_subdags=not args.exclude_subdags)
+        include_subdags=not args.exclude_subdags,
+        include_parentdag=not args.exclude_parentdag,
+    )
 
 
 def get_num_ready_workers_running(gunicorn_master_proc):
@@ -783,13 +861,19 @@ def webserver(args):
         print(
             "Starting the web server on port {0} and host {1}.".format(
                 args.port, args.hostname))
-        app = create_app_rbac(conf) if settings.RBAC else create_app(conf)
-        app.run(debug=True, port=args.port, host=args.hostname,
+        if settings.RBAC:
+            app, _ = create_app_rbac(None, testing=conf.get('core', 'unit_test_mode'))
+        else:
+            app = create_app(None, testing=conf.get('core', 'unit_test_mode'))
+        app.run(debug=True, use_reloader=False if app.config['TESTING'] else True,
+                port=args.port, host=args.hostname,
                 ssl_context=(ssl_cert, ssl_key) if ssl_cert and ssl_key else None)
     else:
-        app = cached_app_rbac(conf) if settings.RBAC else cached_app(conf)
+        os.environ['SKIP_DAGS_PARSING'] = 'True'
+        app = cached_app_rbac(None) if settings.RBAC else cached_app(None)
         pid, stdout, stderr, log_file = setup_locations(
             "webserver", args.pid, args.stdout, args.stderr, args.log_file)
+        os.environ.pop('SKIP_DAGS_PARSING')
         if args.daemon:
             handle = setup_logging(log_file)
             stdout = open(stdout, 'w+')
@@ -951,17 +1035,27 @@ def worker(args):
     env = os.environ.copy()
     env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME
 
+    if not settings.validate_session():
+        log = LoggingMixin().log
+        log.error("Worker exiting... database connection precheck failed! ")
+        sys.exit(1)
+
     # Celery worker
     from airflow.executors.celery_executor import app as celery_app
     from celery.bin import worker
 
+    autoscale = args.autoscale
+    if autoscale is None and conf.has_option("celery", "worker_autoscale"):
+        autoscale = conf.get("celery", "worker_autoscale")
     worker = worker.worker(app=celery_app)
     options = {
         'optimization': 'fair',
         'O': 'fair',
         'queues': args.queues,
         'concurrency': args.concurrency,
+        'autoscale': autoscale,
         'hostname': args.celery_hostname,
+        'loglevel': conf.get('core', 'LOGGING_LEVEL'),
     }
 
     if args.daemon:
@@ -999,17 +1093,16 @@ def worker(args):
 
 def initdb(args):  # noqa
     print("DB: " + repr(settings.engine.url))
-    db_utils.initdb(settings.RBAC)
+    db.initdb(settings.RBAC)
     print("Done.")
 
 
-@cli_utils.action_logging
 def resetdb(args):
     print("DB: " + repr(settings.engine.url))
     if args.yes or input("This will drop existing tables "
                          "if they exist. Proceed? "
                          "(y/n)").upper() == "Y":
-        db_utils.resetdb(settings.RBAC)
+        db.resetdb(settings.RBAC)
     else:
         print("Bail.")
 
@@ -1017,19 +1110,7 @@ def resetdb(args):
 @cli_utils.action_logging
 def upgradedb(args):  # noqa
     print("DB: " + repr(settings.engine.url))
-    db_utils.upgradedb()
-
-    # Populate DagStats table
-    session = settings.Session()
-    ds_rows = session.query(DagStat).count()
-    if not ds_rows:
-        qry = (
-            session.query(DagRun.dag_id, DagRun.state, func.count('*'))
-                   .group_by(DagRun.dag_id, DagRun.state)
-        )
-        for dag_id, state, count in qry:
-            session.add(DagStat(dag_id=dag_id, state=state, count=count))
-        session.commit()
+    db.upgradedb()
 
 
 @cli_utils.action_logging
@@ -1056,20 +1137,20 @@ def connections(args):
             print(msg)
             return
 
-        session = settings.Session()
-        conns = session.query(Connection.conn_id, Connection.conn_type,
-                              Connection.host, Connection.port,
-                              Connection.is_encrypted,
-                              Connection.is_extra_encrypted,
-                              Connection.extra).all()
-        conns = [map(reprlib.repr, conn) for conn in conns]
-        msg = tabulate(conns, ['Conn Id', 'Conn Type', 'Host', 'Port',
-                               'Is Encrypted', 'Is Extra Encrypted', 'Extra'],
-                       tablefmt="fancy_grid")
-        if sys.version_info[0] < 3:
-            msg = msg.encode('utf-8')
-        print(msg)
-        return
+        with db.create_session() as session:
+            conns = session.query(Connection.conn_id, Connection.conn_type,
+                                  Connection.host, Connection.port,
+                                  Connection.is_encrypted,
+                                  Connection.is_extra_encrypted,
+                                  Connection.extra).all()
+            conns = [map(reprlib.repr, conn) for conn in conns]
+            msg = tabulate(conns, ['Conn Id', 'Conn Type', 'Host', 'Port',
+                                   'Is Encrypted', 'Is Extra Encrypted', 'Extra'],
+                           tablefmt="fancy_grid")
+            if sys.version_info[0] < 3:
+                msg = msg.encode('utf-8')
+            print(msg)
+            return
 
     if args.delete:
         # Check that only the `conn_id` arg was passed to the command
@@ -1089,31 +1170,30 @@ def connections(args):
                   'the --conn_id flag.\n')
             return
 
-        session = settings.Session()
-        try:
-            to_delete = (session
-                         .query(Connection)
-                         .filter(Connection.conn_id == args.conn_id)
-                         .one())
-        except exc.NoResultFound:
-            msg = '\n\tDid not find a connection with `conn_id`={conn_id}\n'
-            msg = msg.format(conn_id=args.conn_id)
-            print(msg)
-            return
-        except exc.MultipleResultsFound:
-            msg = ('\n\tFound more than one connection with ' +
-                   '`conn_id`={conn_id}\n')
-            msg = msg.format(conn_id=args.conn_id)
-            print(msg)
+        with db.create_session() as session:
+            try:
+                to_delete = (session
+                             .query(Connection)
+                             .filter(Connection.conn_id == args.conn_id)
+                             .one())
+            except exc.NoResultFound:
+                msg = '\n\tDid not find a connection with `conn_id`={conn_id}\n'
+                msg = msg.format(conn_id=args.conn_id)
+                print(msg)
+                return
+            except exc.MultipleResultsFound:
+                msg = ('\n\tFound more than one connection with ' +
+                       '`conn_id`={conn_id}\n')
+                msg = msg.format(conn_id=args.conn_id)
+                print(msg)
+                return
+            else:
+                deleted_conn_id = to_delete.conn_id
+                session.delete(to_delete)
+                msg = '\n\tSuccessfully deleted `conn_id`={conn_id}\n'
+                msg = msg.format(conn_id=deleted_conn_id)
+                print(msg)
             return
-        else:
-            deleted_conn_id = to_delete.conn_id
-            session.delete(to_delete)
-            session.commit()
-            msg = '\n\tSuccessfully deleted `conn_id`={conn_id}\n'
-            msg = msg.format(conn_id=deleted_conn_id)
-            print(msg)
-        return
 
     if args.add:
         # Check that the conn_id and conn_uri args were passed to the command:
@@ -1152,26 +1232,25 @@ def connections(args):
         if args.conn_extra is not None:
             new_conn.set_extra(args.conn_extra)
 
-        session = settings.Session()
-        if not (session.query(Connection)
-                       .filter(Connection.conn_id == new_conn.conn_id).first()):
-            session.add(new_conn)
-            session.commit()
-            msg = '\n\tSuccessfully added `conn_id`={conn_id} : {uri}\n'
-            msg = msg.format(conn_id=new_conn.conn_id,
-                             uri=args.conn_uri or
-                             urlunparse((args.conn_type,
-                                        '{login}:{password}@{host}:{port}'
-                                         .format(login=args.conn_login or '',
-                                                 password=args.conn_password or '',
-                                                 host=args.conn_host or '',
-                                                 port=args.conn_port or ''),
-                                         args.conn_schema or '', '', '', '')))
-            print(msg)
-        else:
-            msg = '\n\tA connection with `conn_id`={conn_id} already exists\n'
-            msg = msg.format(conn_id=new_conn.conn_id)
-            print(msg)
+        with db.create_session() as session:
+            if not (session.query(Connection)
+                           .filter(Connection.conn_id == new_conn.conn_id).first()):
+                session.add(new_conn)
+                msg = '\n\tSuccessfully added `conn_id`={conn_id} : {uri}\n'
+                msg = msg.format(conn_id=new_conn.conn_id,
+                                 uri=args.conn_uri or
+                                 urlunparse((args.conn_type,
+                                            '{login}:{password}@{host}:{port}'
+                                             .format(login=args.conn_login or '',
+                                                     password=args.conn_password or '',
+                                                     host=args.conn_host or '',
+                                                     port=args.conn_port or ''),
+                                             args.conn_schema or '', '', '', '')))
+                print(msg)
+            else:
+                msg = '\n\tA connection with `conn_id`={conn_id} already exists\n'
+                msg = msg.format(conn_id=new_conn.conn_id)
+                print(msg)
 
         return
 
@@ -1189,6 +1268,10 @@ def flower(args):
     if args.url_prefix:
         url_prefix = '--url-prefix=' + args.url_prefix
 
+    basic_auth = ''
+    if args.basic_auth:
+        basic_auth = '--basic_auth=' + args.basic_auth
+
     flower_conf = ''
     if args.flower_conf:
         flower_conf = '--conf=' + args.flower_conf
@@ -1210,7 +1293,7 @@ def flower(args):
 
         with ctx:
             os.execvp("flower", ['flower', '-b',
-                                 broka, address, port, api, flower_conf, url_prefix])
+                                 broka, address, port, api, flower_conf, url_prefix, basic_auth])
 
         stdout.close()
         stderr.close()
@@ -1219,7 +1302,7 @@ def flower(args):
         signal.signal(signal.SIGTERM, sigint_handler)
 
         os.execvp("flower", ['flower', '-b',
-                             broka, address, port, api, flower_conf, url_prefix])
+                             broka, address, port, api, flower_conf, url_prefix, basic_auth])
 
 
 @cli_utils.action_logging
@@ -1243,12 +1326,12 @@ def kerberos(args):  # noqa
         )
 
         with ctx:
-            airflow.security.kerberos.run()
+            airflow.security.kerberos.run(principal=args.principal, keytab=args.keytab)
 
         stdout.close()
         stderr.close()
     else:
-        airflow.security.kerberos.run()
+        airflow.security.kerberos.run(principal=args.principal, keytab=args.keytab)
 
 
 @cli_utils.action_logging
@@ -1280,6 +1363,9 @@ def create_user(args):
         if password != password_confirmation:
             raise SystemExit('Passwords did not match!')
 
+    if appbuilder.sm.find_user(args.username):
+        print('{} already exist in the db'.format(args.username))
+        return
     user = appbuilder.sm.add_user(args.username, args.firstname, args.lastname,
                                   args.email, role, password)
     if user:
@@ -1288,9 +1374,110 @@ def create_user(args):
         raise SystemExit('Failed to create user.')
 
 
-Arg = namedtuple(
-    'Arg', ['flags', 'help', 'action', 'default', 'nargs', 'type', 'choices', 'metavar'])
-Arg.__new__.__defaults__ = (None, None, None, None, None, None, None)
+@cli_utils.action_logging
+def delete_user(args):
+    if not args.username:
+        raise SystemExit('Required arguments are missing: username')
+
+    appbuilder = cached_appbuilder()
+
+    try:
+        u = next(u for u in appbuilder.sm.get_all_users() if u.username == args.username)
+    except StopIteration:
+        raise SystemExit('{} is not a valid user.'.format(args.username))
+
+    if appbuilder.sm.del_register_user(u):
+        print('User {} deleted.'.format(args.username))
+    else:
+        raise SystemExit('Failed to delete user.')
+
+
+@cli_utils.action_logging
+def list_users(args):
+    appbuilder = cached_appbuilder()
+    users = appbuilder.sm.get_all_users()
+    fields = ['id', 'username', 'email', 'first_name', 'last_name', 'roles']
+    users = [[user.__getattribute__(field) for field in fields] for user in users]
+    msg = tabulate(users, [field.capitalize().replace('_', ' ') for field in fields],
+                   tablefmt="fancy_grid")
+    if sys.version_info[0] < 3:
+        msg = msg.encode('utf-8')
+    print(msg)
+
+
+@cli_utils.action_logging
+def list_dag_runs(args, dag=None):
+    if dag:
+        args.dag_id = dag.dag_id
+
+    dagbag = DagBag()
+
+    if args.dag_id not in dagbag.dags:
+        error_message = "Dag id {} not found".format(args.dag_id)
+        raise AirflowException(error_message)
+
+    dag_runs = list()
+    state = args.state.lower() if args.state else None
+    for run in DagRun.find(dag_id=args.dag_id,
+                           state=state,
+                           no_backfills=args.no_backfill):
+        dag_runs.append({
+            'id': run.id,
+            'run_id': run.run_id,
+            'state': run.state,
+            'dag_id': run.dag_id,
+            'execution_date': run.execution_date.isoformat(),
+            'start_date': ((run.start_date or '') and
+                           run.start_date.isoformat()),
+        })
+    if not dag_runs:
+        print('No dag runs for {dag_id}'.format(dag_id=args.dag_id))
+
+    s = textwrap.dedent("""\n
+    {line}
+    DAG RUNS
+    {line}
+    {dag_run_header}
+    """)
+
+    dag_runs.sort(key=lambda x: x['execution_date'], reverse=True)
+    dag_run_header = '%-3s | %-20s | %-10s | %-20s | %-20s |' % ('id',
+                                                                 'run_id',
+                                                                 'state',
+                                                                 'execution_date',
+                                                                 'state_date')
+    print(s.format(dag_run_header=dag_run_header,
+                   line='-' * 120))
+    for dag_run in dag_runs:
+        record = '%-3s | %-20s | %-10s | %-20s | %-20s |' % (dag_run['id'],
+                                                             dag_run['run_id'],
+                                                             dag_run['state'],
+                                                             dag_run['execution_date'],
+                                                             dag_run['start_date'])
+        print(record)
+
+
+@cli_utils.action_logging
+def sync_perm(args): # noqa
+    if settings.RBAC:
+        appbuilder = cached_appbuilder()
+        print('Update permission, view-menu for all existing roles')
+        appbuilder.sm.sync_roles()
+    else:
+        print('The sync_perm command only works for rbac UI.')
+
+
+class Arg(object):
+    def __init__(self, flags=None, help=None, action=None, default=None, nargs=None,
+                 type=None, choices=None, metavar=None):
+        self.flags = flags
+        self.help = help
+        self.action = action
+        self.default = default
+        self.nargs = nargs
+        self.type = type
+        self.choices = choices
+        self.metavar = metavar
 
 
 class CLIFactory(object):
@@ -1306,8 +1493,10 @@ class CLIFactory(object):
             "The regex to filter specific task_ids to backfill (optional)"),
         'subdir': Arg(
             ("-sd", "--subdir"),
-            "File location or directory from which to look for the dag",
-            default=settings.DAGS_FOLDER),
+            "File location or directory from which to look for the dag. "
+            "Defaults to '[AIRFLOW_HOME]/dags' where [AIRFLOW_HOME] is the "
+            "value you set for 'AIRFLOW_HOME' config you set in 'airflow.cfg' ",
+            default=DAGS_FOLDER),
         'start_date': Arg(
             ("-s", "--start_date"), "Override start_date YYYY-MM-DD",
             type=parsedate),
@@ -1334,6 +1523,19 @@ class CLIFactory(object):
             "Do not prompt to confirm reset. Use with care!",
             "store_true",
             default=False),
+        'username': Arg(
+            ('-u', '--username',),
+            help='Username of the user',
+            type=str),
+
+        # list_dag_runs
+        'no_backfill': Arg(
+            ("--no_backfill",),
+            "filter all the backfill dagruns given the dag id", "store_true"),
+        'state': Arg(
+            ("--state",),
+            "Only list the dag runs corresponding to the state"
+        ),
 
         # backfill
         'mark_success': Arg(
@@ -1388,6 +1590,13 @@ class CLIFactory(object):
                 "all the failed tasks for the backfill date range "
                 "instead of throwing exceptions"),
             "store_true"),
+        'run_backwards': Arg(
+            ("-B", "--run_backwards",),
+            (
+                "if set, the backfill will run tasks from the most "
+                "recent day first.  if there are tasks that depend_on_past "
+                "this option will throw an exception"),
+            "store_true"),
 
         # list_tasks
         'tree': Arg(("-t", "--tree"), "Tree view", "store_true"),
@@ -1409,6 +1618,10 @@ class CLIFactory(object):
         'exclude_subdags': Arg(
             ("-x", "--exclude_subdags"),
             "Exclude subdags", "store_true"),
+        'exclude_parentdag': Arg(
+            ("-xp", "--exclude_parentdag"),
+            "Exclude ParentDAGS if the task cleared is a part of a SubDAG",
+            "store_true"),
         'dag_regex': Arg(
             ("-dx", "--dag_regex"),
             "Search dag_id as regex instead of exact string", "store_true"),
@@ -1434,6 +1647,14 @@ class CLIFactory(object):
             ("-x", "--delete"),
             metavar="NAME",
             help="Delete a pool"),
+        'pool_import': Arg(
+            ("-i", "--import"),
+            metavar="FILEPATH",
+            help="Import pool from JSON file"),
+        'pool_export': Arg(
+            ("-e", "--export"),
+            metavar="FILEPATH",
+            help="Export pool to JSON file"),
         # variables
         'set': Arg(
             ("-s", "--set"),
@@ -1467,8 +1688,7 @@ class CLIFactory(object):
             help="Delete a variable"),
         # kerberos
         'principal': Arg(
-            ("principal",), "kerberos principal",
-            nargs='?', default=conf.get('kerberos', 'principal')),
+            ("principal",), "kerberos principal", nargs='?'),
         'keytab': Arg(
             ("-kt", "--keytab"), "keytab",
             nargs='?', default=conf.get('kerberos', 'keytab')),
@@ -1616,9 +1836,20 @@ class CLIFactory(object):
             ("-u", "--url_prefix"),
             default=conf.get('celery', 'FLOWER_URL_PREFIX'),
             help="URL prefix for Flower"),
+        'flower_basic_auth': Arg(
+            ("-ba", "--basic_auth"),
+            default=conf.get('celery', 'FLOWER_BASIC_AUTH'),
+            help=("Securing Flower with Basic Authentication. "
+                  "Accepts user:password pairs separated by a comma. "
+                  "Example: flower_basic_auth = user1:password1,user2:password2")),
         'task_params': Arg(
             ("-tp", "--task_params"),
             help="Sends a JSON params dict to the task"),
+        'post_mortem': Arg(
+            ("-pm", "--post_mortem"),
+            action="store_true",
+            help="Open debugger on uncaught exception",
+        ),
         # connections
         'list_connections': Arg(
             ('-l', '--list'),
@@ -1686,10 +1917,6 @@ class CLIFactory(object):
             ('-e', '--email',),
             help='Email of the user',
             type=str),
-        'username': Arg(
-            ('-u', '--username',),
-            help='Username of the user',
-            type=str),
         'password': Arg(
             ('-p', '--password',),
             help='Password of the user',
@@ -1699,6 +1926,9 @@ class CLIFactory(object):
             help='Do not prompt for password.  Use random string instead',
             default=False,
             action='store_true'),
+        'autoscale': Arg(
+            ('-a', '--autoscale'),
+            help="Minimum and Maximum number of worker to autoscale"),
     }
     subparsers = (
         {
@@ -1707,7 +1937,7 @@ class CLIFactory(object):
                     "If reset_dag_run option is used,"
                     " backfill will first prompt users whether airflow "
                     "should clear all the previous dag_run and task_instances "
-                    "within the backfill date range."
+                    "within the backfill date range. "
                     "If rerun_failed_tasks is used, backfill "
                     "will auto re-run the previous failed task instances"
                     " within the backfill date range.",
@@ -1716,7 +1946,16 @@ class CLIFactory(object):
                 'mark_success', 'local', 'donot_pickle',
                 'bf_ignore_dependencies', 'bf_ignore_first_depends_on_past',
                 'subdir', 'pool', 'delay_on_limit', 'dry_run', 'verbose', 'conf',
-                'reset_dag_run', 'rerun_failed_tasks',
+                'reset_dag_run', 'rerun_failed_tasks', 'run_backwards'
+            )
+        }, {
+            'func': list_dag_runs,
+            'help': "List dag runs given a DAG id. If state option is given, it will only"
+                    "search for all the dagruns with the given state. "
+                    "If no_backfill option is given, it will filter out"
+                    "all backfill dagruns for given dag id.",
+            'args': (
+                'dag_id', 'no_backfill', 'state'
             )
         }, {
             'func': list_tasks,
@@ -1728,7 +1967,7 @@ class CLIFactory(object):
             'args': (
                 'dag_id', 'task_regex', 'start_date', 'end_date', 'subdir',
                 'upstream', 'downstream', 'no_confirm', 'only_failed',
-                'only_running', 'exclude_subdags', 'dag_regex'),
+                'only_running', 'exclude_subdags', 'exclude_parentdag', 'dag_regex'),
         }, {
             'func': pause,
             'help': "Pause a DAG",
@@ -1748,7 +1987,7 @@ class CLIFactory(object):
         }, {
             'func': pool,
             'help': "CRUD operations on pools",
-            "args": ('pool_set', 'pool_get', 'pool_delete'),
+            "args": ('pool_set', 'pool_get', 'pool_delete', 'pool_import', 'pool_export'),
         }, {
             'func': variables,
             'help': "CRUD operations on variables",
@@ -1806,7 +2045,7 @@ class CLIFactory(object):
                 "dependencies or recording its state in the database."),
             'args': (
                 'dag_id', 'task_id', 'execution_date', 'subdir', 'dry_run',
-                'task_params'),
+                'task_params', 'post_mortem'),
         }, {
             'func': webserver,
             'help': "Start a Airflow webserver instance",
@@ -1831,12 +2070,12 @@ class CLIFactory(object):
             'func': worker,
             'help': "Start a Celery worker node",
             'args': ('do_pickle', 'queues', 'concurrency', 'celery_hostname',
-                     'pid', 'daemon', 'stdout', 'stderr', 'log_file'),
+                     'pid', 'daemon', 'stdout', 'stderr', 'log_file', 'autoscale'),
         }, {
             'func': flower,
             'help': "Start a Celery Flower",
             'args': ('flower_hostname', 'flower_port', 'flower_conf', 'flower_url_prefix',
-                     'broker_api', 'pid', 'daemon', 'stdout', 'stderr', 'log_file'),
+                     'flower_basic_auth', 'broker_api', 'pid', 'daemon', 'stdout', 'stderr', 'log_file'),
         }, {
             'func': version,
             'help': "Show the version",
@@ -1848,14 +2087,39 @@ class CLIFactory(object):
                      'conn_id', 'conn_uri', 'conn_extra') + tuple(alternative_conn_specs),
         }, {
             'func': create_user,
-            'help': "Create an admin account",
+            'help': "Create an account for the Web UI (FAB-based)",
             'args': ('role', 'username', 'email', 'firstname', 'lastname',
                      'password', 'use_random_password'),
+        }, {
+            'func': delete_user,
+            'help': "Delete an account for the Web UI",
+            'args': ('username',),
+        }, {
+            'func': list_users,
+            'help': "List accounts for the Web UI",
+            'args': tuple(),
+        },
+        {
+            'func': sync_perm,
+            'help': "Update existing role's permissions.",
+            'args': tuple(),
+        },
+        {
+            'func': next_execution,
+            'help': "Get the next execution datetime of a DAG.",
+            'args': ('dag_id', 'subdir')
+        },
+        {
+            'func': rotate_fernet_key,
+            'help': 'Rotate all encrypted connection credentials and variables; see '
+                    'https://airflow.readthedocs.io/en/stable/howto/secure-connections.html'
+                    '#rotating-encryption-keys.',
+            'args': (),
         },
     )
     subparsers_dict = {sp['func'].__name__: sp for sp in subparsers}
     dag_subparsers = (
-        'list_tasks', 'backfill', 'test', 'run', 'pause', 'unpause')
+        'list_tasks', 'backfill', 'test', 'run', 'pause', 'unpause', 'list_dag_runs')
 
     @classmethod
     def get_parser(cls, dag_parser=False):
@@ -1873,8 +2137,8 @@ def get_parser(cls, dag_parser=False):
                     continue
                 arg = cls.args[arg]
                 kwargs = {
-                    f: getattr(arg, f)
-                    for f in arg._fields if f != 'flags' and getattr(arg, f)}
+                    f: v
+                    for f, v in vars(arg).items() if f != 'flags' and v}
                 sp.add_argument(*arg.flags, **kwargs)
             sp.set_defaults(func=sub['func'])
         return parser
diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py
index 95150ab3bbbef..a25f56d076647 100644
--- a/airflow/config_templates/airflow_local_settings.py
+++ b/airflow/config_templates/airflow_local_settings.py
@@ -18,8 +18,10 @@
 # under the License.
 
 import os
+from typing import Dict, Any
 
 from airflow import configuration as conf
+from airflow.utils.file import mkdirs
 
 # TODO: Logging format and level should be configured
 # in this file instead of from airflow.cfg. Currently
@@ -38,7 +40,11 @@
 
 PROCESSOR_LOG_FOLDER = conf.get('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY')
 
+DAG_PROCESSOR_MANAGER_LOG_LOCATION = \
+    conf.get('core', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION')
+
 FILENAME_TEMPLATE = conf.get('core', 'LOG_FILENAME_TEMPLATE')
+
 PROCESSOR_FILENAME_TEMPLATE = conf.get('core', 'LOG_PROCESSOR_FILENAME_TEMPLATE')
 
 # Storage bucket url for remote logging
@@ -79,7 +85,7 @@
             'formatter': 'airflow',
             'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
             'filename_template': PROCESSOR_FILENAME_TEMPLATE,
-        },
+        }
     },
     'loggers': {
         'airflow.processor': {
@@ -102,6 +108,26 @@
         'handlers': ['console'],
         'level': LOG_LEVEL,
     }
+}  # type: Dict[str, Any]
+
+DEFAULT_DAG_PARSING_LOGGING_CONFIG = {
+    'handlers': {
+        'processor_manager': {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'formatter': 'airflow',
+            'filename': DAG_PROCESSOR_MANAGER_LOG_LOCATION,
+            'mode': 'a',
+            'maxBytes': 104857600,  # 100MB
+            'backupCount': 5
+        }
+    },
+    'loggers': {
+        'airflow.processor_manager': {
+            'handlers': ['processor_manager'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        }
+    }
 }
 
 REMOTE_HANDLERS = {
@@ -172,11 +198,27 @@
 
 REMOTE_LOGGING = conf.get('core', 'remote_logging')
 
+# Only update the handlers and loggers when CONFIG_PROCESSOR_MANAGER_LOGGER is set.
+# This is to avoid exceptions when initializing RotatingFileHandler multiple times
+# in multiple processes.
+if os.environ.get('CONFIG_PROCESSOR_MANAGER_LOGGER') == 'True':
+    DEFAULT_LOGGING_CONFIG['handlers'] \
+        .update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'])
+    DEFAULT_LOGGING_CONFIG['loggers'] \
+        .update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['loggers'])
+
+    # Manually create log directory for processor_manager handler as RotatingFileHandler
+    # will only create file but not the directory.
+    processor_manager_handler_config = DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'][
+        'processor_manager']
+    directory = os.path.dirname(processor_manager_handler_config['filename'])
+    mkdirs(directory, 0o755)
+
 if REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('s3://'):
-        DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['s3'])
+    DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['s3'])
 elif REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('gs://'):
-        DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['gcs'])
+    DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['gcs'])
 elif REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('wasb'):
-        DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['wasb'])
+    DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['wasb'])
 elif REMOTE_LOGGING and ELASTICSEARCH_HOST:
-        DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['elasticsearch'])
+    DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['elasticsearch'])
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 2f6130c8975f7..2f990131367b5 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -30,9 +30,6 @@
 # ----------------------- TEMPLATE BEGINS HERE -----------------------
 
 [core]
-# The home folder for airflow, default is ~/airflow
-airflow_home = {AIRFLOW_HOME}
-
 # The folder where your airflow pipelines live, most likely a
 # subfolder in a code repository
 # This path must be absolute
@@ -62,14 +59,13 @@ fab_logging_level = WARN
 logging_config_class =
 
 # Log format
-# we need to escape the curly braces by adding an additional curly brace
 log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s
 simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
 
 # Log filename format
-# we need to escape the curly braces by adding an additional curly brace
 log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log
 log_processor_filename_template = {{{{ filename }}}}.log
+dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log
 
 # Hostname by providing a path to a callable, which will resolve the hostname
 hostname_callable = socket:getfqdn
@@ -79,7 +75,7 @@ hostname_callable = socket:getfqdn
 default_timezone = utc
 
 # The executor class that airflow should use. Choices include
-# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor
+# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor
 executor = SequentialExecutor
 
 # The SqlAlchemy connection string to the metadata database.
@@ -87,6 +83,9 @@ executor = SequentialExecutor
 # their website
 sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/airflow.db
 
+# The encoding for the databases
+sql_engine_encoding = utf-8
+
 # If SqlAlchemy should pool database connections.
 sql_alchemy_pool_enabled = True
 
@@ -104,6 +103,10 @@ sql_alchemy_pool_recycle = 1800
 # disconnects. Setting this to 0 disables retries.
 sql_alchemy_reconnect_timeout = 300
 
+# The schema to use for the metadata database
+# SqlAlchemy supports databases with the concept of multiple schemas.
+sql_alchemy_schema =
+
 # The amount of parallelism as a setting to the executor. This defines
 # the max number of task instances that should run simultaneously
 # on this airflow installation
@@ -140,7 +143,7 @@ donot_pickle = False
 dagbag_import_timeout = 30
 
 # The class to use for running task instances in a subprocess
-task_runner = BashTaskRunner
+task_runner = StandardTaskRunner
 
 # If set, tasks without a `run_as_user` argument will be run with this user
 # Can be used to de-elevate a sudo user running Airflow when executing tasks
@@ -173,6 +176,13 @@ killed_task_cleanup_time = 60
 # `airflow trigger_dag -c`, the key-value pairs will override the existing ones in params.
 dag_run_conf_overrides_params = False
 
+# Worker initialisation check to validate Metadata Database connection
+worker_precheck = False
+
+# When discovering DAGs, ignore any files that don't contain the strings `DAG` and `airflow`.
+dag_discovery_safe_mode = True
+
+
 [cli]
 # In what way should the cli access the API. The LocalClient will use the
 # database directly, while the json_client will use the api running on the
@@ -258,10 +268,13 @@ access_logfile = -
 error_logfile = -
 
 # Expose the configuration file in the web server
+# This is only applicable for the flask-admin based web UI (non FAB-based).
+# In the FAB-based web UI with RBAC feature,
+# access to configuration is controlled by role permissions.
 expose_config = False
 
 # Set to true to turn on authentication:
-# https://airflow.incubator.apache.org/security.html#web-authentication
+# https://airflow.apache.org/security.html#web-authentication
 authenticate = False
 
 # Filter the list of dags by owner name (requires authentication to be enabled)
@@ -306,6 +319,15 @@ navbar_color = #007A87
 # Default dagrun to show in UI
 default_dag_run_display_number = 25
 
+# Enable werkzeug `ProxyFix` middleware
+enable_proxy_fix = False
+
+# Set secure flag on session cookie
+cookie_secure = False
+
+# Set samesite policy on session cookie
+cookie_samesite =
+
 
 [email]
 email_backend = airflow.utils.email.send_email_smtp
@@ -338,6 +360,14 @@ celery_app_name = airflow.executors.celery_executor
 # your worker box and the nature of your tasks
 worker_concurrency = 16
 
+# The maximum and minimum concurrency that will be used when starting workers with the
+# "airflow worker" command (always keep minimum processes, but grow to maximum if necessary).
+# Note the value should be "max_concurrency,min_concurrency"
+# Pick these numbers based on resources on worker box and the nature of the task.
+# If autoscale option is available, worker_concurrency will be ignored.
+# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale
+# worker_autoscale = 16,12
+
 # When you start an airflow worker, airflow starts a tiny web server
 # subprocess to serve the workers local log files to the airflow main
 # web server, who then builds pages and sends them to users. This defines
@@ -370,9 +400,18 @@ flower_url_prefix =
 # This defines the port that Celery Flower runs on
 flower_port = 5555
 
+# Securing Flower with Basic Authentication
+# Accepts user:password pairs separated by a comma
+# Example: flower_basic_auth = user1:password1,user2:password2
+flower_basic_auth =
+
 # Default queue that tasks get assigned to and that worker listen on.
 default_queue = default
 
+# How many processes CeleryExecutor uses to sync task state.
+# 0 means to use max(1, number of cores - 1) processes.
+sync_parallelism = 0
+
 # Import path for celery configuration options
 celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
 
@@ -425,17 +464,20 @@ scheduler_heartbeat_sec = 5
 # -1 indicates to run continuously (see also num_runs)
 run_duration = -1
 
-# after how much time a new DAGs should be picked up from the filesystem
+# after how much time (seconds) a new DAGs should be picked up from the filesystem
 min_file_process_interval = 0
 
-# How many seconds to wait between file-parsing loops to prevent the logs from being spammed.
-min_file_parsing_loop_time = 1
-
+# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes.
 dag_dir_list_interval = 300
 
 # How often should stats be printed to the logs
 print_stats_interval = 30
 
+# If the last scheduler heartbeat happened more than scheduler_health_check_threshold ago (in seconds),
+# scheduler is considered unhealthy.
+# This is used by the health check in the "/health" endpoint
+scheduler_health_check_threshold = 30
+
 child_process_log_directory = {AIRFLOW_HOME}/logs/scheduler
 
 # Local task jobs periodically heartbeat to the DB. If the job has
@@ -475,6 +517,10 @@ max_threads = 2
 
 authenticate = False
 
+# Turn off scheduler use of cron intervals by setting this to False.
+# DAGs submitted manually in the web UI or with trigger_dag will still run.
+use_job_schedule = True
+
 [ldap]
 # set this to ldaps://<your.ldap.server>:<port>
 uri =
@@ -489,6 +535,10 @@ basedn = dc=example,dc=com
 cacert = /etc/ca/ldap_ca.crt
 search_scope = LEVEL
 
+# This setting allows the use of LDAP servers that either return a 
+# broken schema, or do not return a schema. 
+ignore_malformed_schema = False
+
 [mesos]
 # Mesos master address which MesosExecutor will connect to.
 master = localhost:5050
@@ -549,28 +599,35 @@ hide_sensitive_variable_fields = True
 
 [elasticsearch]
 elasticsearch_host =
-# we need to escape the curly braces by adding an additional curly brace
 elasticsearch_log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
 elasticsearch_end_of_log_mark = end_of_log
 
 [kubernetes]
-# The repository and tag of the Kubernetes Image for the Worker to Run
+# The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run
 worker_container_repository =
 worker_container_tag =
+worker_container_image_pull_policy = IfNotPresent
 
 # If True (default), worker pods will be deleted upon termination
 delete_worker_pods = True
 
+# Number of Kubernetes Worker Pod creation calls per scheduler loop
+worker_pods_creation_batch_size = 1
+
 # The Kubernetes namespace where airflow workers should be created. Defaults to `default`
 namespace = default
 
 # The name of the Kubernetes ConfigMap Containing the Airflow Configuration (this file)
 airflow_configmap =
 
+# For docker image already contains DAGs, this is set to `True`, and the worker will search for dags in dags_folder,
+# otherwise use git sync or dags volume claim to mount DAGs
+dags_in_image = False
+
 # For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs
 dags_volume_subpath =
 
-# For DAGs mounted via a volume claim (mutually exclusive with volume claim)
+# For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path)
 dags_volume_claim =
 
 # For volume mounted logs, the worker will look in this subpath for logs
@@ -579,16 +636,67 @@ logs_volume_subpath =
 # A shared volume claim for the logs
 logs_volume_claim =
 
+# For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync)
+# Useful in local environment, discouraged in production
+dags_volume_host =
+
+# A hostPath volume for the logs
+# Useful in local environment, discouraged in production
+logs_volume_host =
+
+# A list of configMapsRefs to envFrom. If more than one configMap is
+# specified, provide a comma separated list: configmap_a,configmap_b
+env_from_configmap_ref =
+
+# A list of secretRefs to envFrom. If more than one secret is
+# specified, provide a comma separated list: secret_a,secret_b
+env_from_secret_ref =
+
 # Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim)
 git_repo =
 git_branch =
+git_subpath =
+# Use git_user and git_password for user authentication or git_ssh_key_secret_name and git_ssh_key_secret_key
+# for SSH authentication
 git_user =
 git_password =
-git_subpath =
+git_sync_root = /git
+git_sync_dest = repo
+# Mount point of the volume if git-sync is being used.
+# i.e. {AIRFLOW_HOME}/dags
+git_dags_folder_mount_point =
+
+# To get Git-sync SSH authentication set up follow this format
+#
+# airflow-secrets.yaml:
+# ---
+# apiVersion: v1
+# kind: Secret
+# metadata:
+#   name: airflow-secrets
+# data:
+#   # key needs to be gitSshKey
+#   gitSshKey: <base64_encoded_data>
+# ---
+# airflow-configmap.yaml:
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: airflow-configmap
+# data:
+#   known_hosts: |
+#       github.com ssh-rsa <...>
+#   airflow.cfg: |
+#       ...
+#
+# git_ssh_key_secret_name = airflow-secrets
+# git_ssh_known_hosts_configmap_name = airflow-configmap
+git_ssh_key_secret_name =
+git_ssh_known_hosts_configmap_name =
 
 # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
-git_sync_container_repository = gcr.io/google-containers/git-sync-amd64
-git_sync_container_tag = v2.0.5
+git_sync_container_repository = k8s.gcr.io/git-sync
+git_sync_container_tag = v3.1.1
 git_sync_init_container_name = git-sync-clone
 
 # The name of the Kubernetes service account to be associated with airflow workers, if any.
@@ -606,21 +714,71 @@ image_pull_secrets =
 gcp_service_account_keys =
 
 # Use the service account kubernetes gives to pods to connect to kubernetes cluster.
-# It’s intended for clients that expect to be running inside a pod running on kubernetes.
+# It's intended for clients that expect to be running inside a pod running on kubernetes.
 # It will raise an exception if called from a process not running in a kubernetes environment.
 in_cluster = True
 
+# When running with in_cluster=False change the default cluster_context or config_file
+# options to Kubernetes client. Leave blank these to use default behaviour like `kubectl` has.
+# cluster_context =
+# config_file =
+
+
+# Affinity configuration as a single line formatted JSON object.
+# See the affinity model for top-level key names (e.g. `nodeAffinity`, etc.):
+#   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core
+affinity =
+
+# A list of toleration objects as a single line formatted JSON array
+# See:
+#   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core
+tolerations =
+
+# Worker pods security context options
+# See:
+#   https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+
+# Specifies the uid to run the first process of the worker pods containers as
+run_as_user =
+
+# Specifies a gid to associate with all containers in the worker pods
+# if using a git_ssh_key_secret_name use an fs_group
+# that allows for the key to be read, e.g. 65533
+fs_group =
+
+[kubernetes_node_selectors]
+# The Key-value pairs to be given to worker pods.
+# The worker pods will be scheduled to the nodes of the specified key-value pairs.
+# Should be supplied in the format: key = value
+
+[kubernetes_annotations]
+# The Key-value annotations pairs to be given to worker pods.
+# Should be supplied in the format: key = value
+
+[kubernetes_environment_variables]
+# The scheduler sets the following environment variables into your workers. You may define as
+# many environment variables as needed and the kubernetes launcher will set them in the launched workers.
+# Environment variables in this section are defined as follows
+#     <environment_variable_key> = <environment_variable_value>
+#
+# For example if you wanted to set an environment variable with value `prod` and key
+# `ENVIRONMENT` you would follow the following format:
+#     ENVIRONMENT = prod
+#
+# Additionally you may override worker airflow settings with the AIRFLOW__<SECTION>__<KEY>
+# formatting as supported by airflow normally.
+
 [kubernetes_secrets]
 # The scheduler mounts the following secrets into your workers as they are launched by the
 # scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the
 # defined secrets and mount them as secret environment variables in the launched workers.
 # Secrets in this section are defined as follows
-#     <environment_variable_mount> = <kubernetes_secret_object>:<kubernetes_secret_key>
+#     <environment_variable_mount> = <kubernetes_secret_object>=<kubernetes_secret_key>
 #
 # For example if you wanted to mount a kubernetes secret key named `postgres_password` from the
 # kubernetes secret object `airflow-secret` as the environment variable `POSTGRES_PASSWORD` into
 # your workers you would follow the following format:
-#     POSTGRES_PASSWORD = airflow-secret:postgres_credentials
+#     POSTGRES_PASSWORD = airflow-secret=postgres_credentials
 #
 # Additionally you may override worker airflow settings with the AIRFLOW__<SECTION>__<KEY>
 # formatting as supported by airflow normally.
diff --git a/airflow/config_templates/default_celery.py b/airflow/config_templates/default_celery.py
index d44f2b3448aff..7a9fd25064b8a 100644
--- a/airflow/config_templates/default_celery.py
+++ b/airflow/config_templates/default_celery.py
@@ -37,7 +37,7 @@ def _broker_supports_visibility_timeout(url):
 )
 if 'visibility_timeout' not in broker_transport_options:
     if _broker_supports_visibility_timeout(broker_url):
-        broker_transport_options = {'visibility_timeout': 21600}
+        broker_transport_options['visibility_timeout'] = 21600
 
 DEFAULT_CELERY_CONFIG = {
     'accept_content': ['json', 'pickle'],
@@ -55,17 +55,27 @@ def _broker_supports_visibility_timeout(url):
 celery_ssl_active = False
 try:
     celery_ssl_active = configuration.conf.getboolean('celery', 'SSL_ACTIVE')
-except AirflowConfigException as e:
+except AirflowConfigException:
     log.warning("Celery Executor will run without SSL")
 
 try:
     if celery_ssl_active:
-        broker_use_ssl = {'keyfile': configuration.conf.get('celery', 'SSL_KEY'),
-                          'certfile': configuration.conf.get('celery', 'SSL_CERT'),
-                          'ca_certs': configuration.conf.get('celery', 'SSL_CACERT'),
-                          'cert_reqs': ssl.CERT_REQUIRED}
+        if 'amqp://' in broker_url:
+            broker_use_ssl = {'keyfile': configuration.conf.get('celery', 'SSL_KEY'),
+                              'certfile': configuration.conf.get('celery', 'SSL_CERT'),
+                              'ca_certs': configuration.conf.get('celery', 'SSL_CACERT'),
+                              'cert_reqs': ssl.CERT_REQUIRED}
+        elif 'redis://' in broker_url:
+            broker_use_ssl = {'ssl_keyfile': configuration.conf.get('celery', 'SSL_KEY'),
+                              'ssl_certfile': configuration.conf.get('celery', 'SSL_CERT'),
+                              'ssl_ca_certs': configuration.conf.get('celery', 'SSL_CACERT'),
+                              'ssl_cert_reqs': ssl.CERT_REQUIRED}
+        else:
+            raise AirflowException('The broker you configured does not support SSL_ACTIVE to be True. '
+                                   'Please use RabbitMQ or Redis if you would like to use SSL for broker.')
+
         DEFAULT_CELERY_CONFIG['broker_use_ssl'] = broker_use_ssl
-except AirflowConfigException as e:
+except AirflowConfigException:
     raise AirflowException('AirflowConfigException: SSL_ACTIVE is True, '
                            'please ensure SSL_KEY, '
                            'SSL_CERT and SSL_CACERT are set')
@@ -75,6 +85,6 @@ def _broker_supports_visibility_timeout(url):
                            'SSL and/or have all necessary certs and key ({}).'.format(e))
 
 result_backend = DEFAULT_CELERY_CONFIG['result_backend']
-if 'amqp' in result_backend or 'redis' in result_backend or 'rpc' in result_backend:
+if 'amqp://' in result_backend or 'redis://' in result_backend or 'rpc://' in result_backend:
     log.warning("You have configured a result_backend of %s, it is highly recommended "
                 "to use an alternative result_backend (i.e. a database).", result_backend)
diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg
index cd4bd32e68fe9..10281f6106e0d 100644
--- a/airflow/config_templates/default_test.cfg
+++ b/airflow/config_templates/default_test.cfg
@@ -31,7 +31,6 @@
 
 [core]
 unit_test_mode = True
-airflow_home = {AIRFLOW_HOME}
 dags_folder = {TEST_DAGS_FOLDER}
 plugins_folder = {TEST_PLUGINS_FOLDER}
 base_log_folder = {AIRFLOW_HOME}/logs
@@ -39,6 +38,7 @@ logging_level = INFO
 fab_logging_level = WARN
 log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log
 log_processor_filename_template = {{{{ filename }}}}.log
+dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log
 executor = SequentialExecutor
 sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/unittests.db
 load_examples = True
@@ -51,6 +51,7 @@ enable_xcom_pickling = False
 killed_task_cleanup_time = 5
 secure_mode = False
 hostname_callable = socket:getfqdn
+worker_precheck = False
 
 [cli]
 api_client = airflow.api.client.local_client
@@ -95,6 +96,7 @@ result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
 flower_host = 0.0.0.0
 flower_port = 5555
 default_queue = default
+sync_parallelism = 0
 
 [mesos]
 master = localhost:5050
@@ -108,6 +110,7 @@ docker_image_slave = test/docker-airflow
 [scheduler]
 job_heartbeat_sec = 1
 scheduler_heartbeat_sec = 5
+scheduler_health_check_threshold = 30
 authenticate = true
 max_threads = 2
 catchup_by_default = True
@@ -122,3 +125,6 @@ hide_sensitive_variable_fields = True
 elasticsearch_host =
 elasticsearch_log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
 elasticsearch_end_of_log_mark = end_of_log
+
+[kubernetes]
+dags_volume_claim = default
diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py
index ba33d81dec23b..e61c7e1a45cb2 100644
--- a/airflow/config_templates/default_webserver_config.py
+++ b/airflow/config_templates/default_webserver_config.py
@@ -89,3 +89,13 @@
 #    { 'name': 'AOL', 'url': 'http://openid.aol.com/<username>' },
 #    { 'name': 'Flickr', 'url': 'http://www.flickr.com/<username>' },
 #    { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }]
+
+# ----------------------------------------------------
+# Theme CONFIG
+# ----------------------------------------------------
+# Flask App Builder comes up with a number of predefined themes
+# that you can use for Apache Airflow.
+# http://flask-appbuilder.readthedocs.io/en/latest/customizing.html#changing-themes
+# Please make sure to remove "navbar_color" configuration from airflow.cfg
+# in order to fully utilize the theme. (or use that property in conjunction with theme)
+# APP_THEME = "bootstrap-theme.css"  # default bootstrap
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 2ee453fd7d93f..ad03b310073bb 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -35,7 +35,7 @@
 import sys
 import warnings
 
-from backports.configparser import ConfigParser
+from backports.configparser import ConfigParser, _UNSET, NoOptionError
 from zope.deprecation import deprecated as _deprecated
 
 from airflow.exceptions import AirflowConfigException
@@ -56,12 +56,9 @@ def generate_fernet_key():
     try:
         from cryptography.fernet import Fernet
     except ImportError:
-        pass
-    try:
-        key = Fernet.generate_key().decode()
-    except NameError:
-        key = "cryptography_not_found_storing_passwords_in_plain_text"
-    return key
+        return ''
+    else:
+        return Fernet.generate_key().decode()
 
 
 def expand_env_var(env_var):
@@ -101,15 +98,20 @@ def run_command(command):
     return output
 
 
-_templates_dir = os.path.join(os.path.dirname(__file__), 'config_templates')
-with open(os.path.join(_templates_dir, 'default_airflow.cfg')) as f:
-    DEFAULT_CONFIG = f.read()
-    if six.PY2:
-        DEFAULT_CONFIG = DEFAULT_CONFIG.decode('utf-8')
-with open(os.path.join(_templates_dir, 'default_test.cfg')) as f:
-    TEST_CONFIG = f.read()
+def _read_default_config_file(file_name):
+    templates_dir = os.path.join(os.path.dirname(__file__), 'config_templates')
+    file_path = os.path.join(templates_dir, file_name)
     if six.PY2:
-        TEST_CONFIG = TEST_CONFIG.decode('utf-8')
+        with open(file_path) as f:
+            config = f.read()
+            return config.decode('utf-8')
+    else:
+        with open(file_path, encoding='utf-8') as f:
+            return f.read()
+
+
+DEFAULT_CONFIG = _read_default_config_file('default_airflow.cfg')
+TEST_CONFIG = _read_default_config_file('default_test.cfg')
 
 
 class AirflowConfigParser(ConfigParser):
@@ -121,15 +123,53 @@ class AirflowConfigParser(ConfigParser):
         ('core', 'sql_alchemy_conn'),
         ('core', 'fernet_key'),
         ('celery', 'broker_url'),
-        ('celery', 'result_backend')
+        ('celery', 'result_backend'),
+        # Todo: remove this in Airflow 1.11
+        ('celery', 'celery_result_backend'),
+        ('atlas', 'password'),
+        ('smtp', 'smtp_password'),
+        ('ldap', 'bind_password'),
+        ('kubernetes', 'git_password'),
     }
 
+    # A two-level mapping of (section -> new_name -> old_name). When reading
+    # new_name, the old_name will be checked to see if it exists. If it does a
+    # DeprecationWarning will be issued and the old name will be used instead
+    deprecated_options = {
+        'celery': {
+            # Remove these keys in Airflow 1.11
+            'worker_concurrency': 'celeryd_concurrency',
+            'result_backend': 'celery_result_backend',
+            'broker_url': 'celery_broker_url',
+            'ssl_active': 'celery_ssl_active',
+            'ssl_cert': 'celery_ssl_cert',
+            'ssl_key': 'celery_ssl_key',
+        }
+    }
+    deprecation_format_string = (
+        'The {old} option in [{section}] has been renamed to {new} - the old '
+        'setting has been used, but please update your config.'
+    )
+
+    # A mapping of old default values that we want to change and warn the user
+    # about. Mapping of section -> setting -> { old, replace, by_version }
+    deprecated_values = {
+        'core': {
+            'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
+        },
+    }
+    deprecation_value_format_string = (
+        'The {name} setting in [{section}] has the old default value of {old!r}. This '
+        'value has been changed to {new!r} in the running config, but please '
+        'update your config before Apache Airflow {version}.'
+    )
+
     def __init__(self, default_config=None, *args, **kwargs):
         super(AirflowConfigParser, self).__init__(*args, **kwargs)
 
-        self.defaults = ConfigParser(*args, **kwargs)
+        self.airflow_defaults = ConfigParser(*args, **kwargs)
         if default_config is not None:
-            self.defaults.read_string(default_config)
+            self.airflow_defaults.read_string(default_config)
 
         self.is_validated = False
 
@@ -159,11 +199,30 @@ def _validate(self):
                 "error: attempt at using ldapgroup "
                 "filtering without using the Ldap backend")
 
+        for section, replacement in self.deprecated_values.items():
+            for name, info in replacement.items():
+                old, new, version = info
+                if self.get(section, name, fallback=None) == old:
+                    # Make sure the env var option is removed, otherwise it
+                    # would be read and used instead of the value we set
+                    env_var = self._env_var_name(section, name)
+                    os.environ.pop(env_var, None)
+
+                    self.set(section, name, new)
+                    warnings.warn(
+                        self.deprecation_value_format_string.format(**locals()),
+                        FutureWarning,
+                    )
+
         self.is_validated = True
 
+    @staticmethod
+    def _env_var_name(section, key):
+        return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
+
     def _get_env_var_option(self, section, key):
         # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
-        env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
+        env_var = self._env_var_name(section, key)
         if env_var in os.environ:
             return expand_env_var(os.environ[env_var])
 
@@ -181,10 +240,17 @@ def get(self, section, key, **kwargs):
         section = str(section).lower()
         key = str(key).lower()
 
+        deprecated_name = self.deprecated_options.get(section, {}).get(key, None)
+
         # first check environment variables
         option = self._get_env_var_option(section, key)
         if option is not None:
             return option
+        if deprecated_name:
+            option = self._get_env_var_option(section, deprecated_name)
+            if option is not None:
+                self._warn_deprecate(section, key, deprecated_name)
+                return option
 
         # ...then the config file
         if super(AirflowConfigParser, self).has_option(section, key):
@@ -192,56 +258,74 @@ def get(self, section, key, **kwargs):
             # separate the config from default config.
             return expand_env_var(
                 super(AirflowConfigParser, self).get(section, key, **kwargs))
+        if deprecated_name:
+            if super(AirflowConfigParser, self).has_option(section, deprecated_name):
+                self._warn_deprecate(section, key, deprecated_name)
+                return expand_env_var(super(AirflowConfigParser, self).get(
+                    section,
+                    deprecated_name,
+                    **kwargs
+                ))
 
         # ...then commands
         option = self._get_cmd_option(section, key)
         if option:
             return option
+        if deprecated_name:
+            option = self._get_cmd_option(section, deprecated_name)
+            if option:
+                self._warn_deprecate(section, key, deprecated_name)
+                return option
 
         # ...then the default config
-        if self.defaults.has_option(section, key):
+        if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs:
             return expand_env_var(
-                self.defaults.get(section, key, **kwargs))
+                self.airflow_defaults.get(section, key, **kwargs))
 
         else:
             log.warning(
-                "section/key [{section}/{key}] not found in config".format(**locals())
+                "section/key [%s/%s] not found in config", section, key
             )
 
             raise AirflowConfigException(
                 "section/key [{section}/{key}] not found "
-                "in config".format(**locals()))
+                "in config".format(section=section, key=key))
 
-    def getboolean(self, section, key):
-        val = str(self.get(section, key)).lower().strip()
+    def getboolean(self, section, key, **kwargs):
+        val = str(self.get(section, key, **kwargs)).lower().strip()
         if '#' in val:
             val = val.split('#')[0].strip()
-        if val.lower() in ('t', 'true', '1'):
+        if val in ('t', 'true', '1'):
             return True
-        elif val.lower() in ('f', 'false', '0'):
+        elif val in ('f', 'false', '0'):
             return False
         else:
             raise AirflowConfigException(
                 'The value for configuration option "{}:{}" is not a '
                 'boolean (received "{}").'.format(section, key, val))
 
-    def getint(self, section, key):
-        return int(self.get(section, key))
+    def getint(self, section, key, **kwargs):
+        return int(self.get(section, key, **kwargs))
+
+    def getfloat(self, section, key, **kwargs):
+        return float(self.get(section, key, **kwargs))
 
-    def getfloat(self, section, key):
-        return float(self.get(section, key))
+    def read(self, filenames, **kwargs):
+        super(AirflowConfigParser, self).read(filenames, **kwargs)
+        self._validate()
 
-    def read(self, filenames):
-        super(AirflowConfigParser, self).read(filenames)
+    def read_dict(self, *args, **kwargs):
+        super(AirflowConfigParser, self).read_dict(*args, **kwargs)
         self._validate()
 
     def has_option(self, section, option):
         try:
             # Using self.get() to avoid reimplementing the priority order
             # of config variables (env, config, cmd, defaults)
-            self.get(section, option)
+            # UNSET to avoid logging a warning about missing values
+            self.get(section, option, fallback=_UNSET)
             return True
-        except AirflowConfigException:
+        except NoOptionError:
             return False
 
     def remove_option(self, section, option, remove_default=True):
@@ -253,24 +337,32 @@ def remove_option(self, section, option, remove_default=True):
         if super(AirflowConfigParser, self).has_option(section, option):
             super(AirflowConfigParser, self).remove_option(section, option)
 
-        if self.defaults.has_option(section, option) and remove_default:
-            self.defaults.remove_option(section, option)
+        if self.airflow_defaults.has_option(section, option) and remove_default:
+            self.airflow_defaults.remove_option(section, option)
 
     def getsection(self, section):
         """
         Returns the section as a dict. Values are converted to int, float, bool
         as required.
+
         :param section: section from the config
-        :return: dict
+        :rtype: dict
         """
-        if section not in self._sections and section not in self.defaults._sections:
+        if (section not in self._sections and
+                section not in self.airflow_defaults._sections):
             return None
 
-        _section = copy.deepcopy(self.defaults._sections[section])
+        _section = copy.deepcopy(self.airflow_defaults._sections[section])
 
         if section in self._sections:
             _section.update(copy.deepcopy(self._sections[section]))
 
+        section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
+        for env_var in sorted(os.environ.keys()):
+            if env_var.startswith(section_prefix):
+                key = env_var.replace(section_prefix, '').lower()
+                _section[key] = self._get_env_var_option(section, key)
+
         for key, val in iteritems(_section):
             try:
                 val = int(val)
@@ -285,30 +377,35 @@ def getsection(self, section):
             _section[key] = val
         return _section
 
-    def as_dict(self, display_source=False, display_sensitive=False):
+    def as_dict(
+            self, display_source=False, display_sensitive=False, raw=False):
         """
         Returns the current configuration as an OrderedDict of OrderedDicts.
         :param display_source: If False, the option value is returned. If True,
             a tuple of (option_value, source) is returned. Source is either
-            'airflow.cfg' or 'default'.
+            'airflow.cfg', 'default', 'env var', or 'cmd'.
         :type display_source: bool
         :param display_sensitive: If True, the values of options set by env
             vars and bash commands will be displayed. If False, those options
             are shown as '< hidden >'
         :type display_sensitive: bool
+        :param raw: Should the values be output as interpolated values, or the
+            "raw" form that can be fed back in to ConfigParser
+        :type raw: bool
         """
-        cfg = copy.deepcopy(self.defaults._sections)
-        cfg.update(copy.deepcopy(self._sections))
-
-        # remove __name__ (affects Python 2 only)
-        for options in cfg.values():
-            options.pop('__name__', None)
-
-        # add source
-        if display_source:
-            for section in cfg:
-                for k, v in cfg[section].items():
-                    cfg[section][k] = (v, 'airflow config')
+        cfg = {}
+        configs = [
+            ('default', self.airflow_defaults),
+            ('airflow.cfg', self),
+        ]
+
+        for (source_name, config) in configs:
+            for section in config.sections():
+                sect = cfg.setdefault(section, OrderedDict())
+                for (k, val) in config.items(section=section, raw=raw):
+                    if display_source:
+                        val = (val, source_name)
+                    sect[k] = val
 
         # add env vars and overwrite because they have priority
         for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
@@ -316,16 +413,15 @@ def as_dict(self, display_source=False, display_sensitive=False):
                 _, section, key = ev.split('__')
                 opt = self._get_env_var_option(section, key)
             except ValueError:
-                opt = None
-            if opt:
-                if (
-                    not display_sensitive and
-                        ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
-                    opt = '< hidden >'
-                if display_source:
-                    opt = (opt, 'env var')
-                cfg.setdefault(section.lower(), OrderedDict()).update(
-                    {key.lower(): opt})
+                continue
+            if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
+                opt = '< hidden >'
+            elif raw:
+                opt = opt.replace('%', '%%')
+            if display_source:
+                opt = (opt, 'env var')
+            cfg.setdefault(section.lower(), OrderedDict()).update(
+                {key.lower(): opt})
 
         # add bash commands
         for (section, key) in self.as_command_stdout:
@@ -334,8 +430,11 @@ def as_dict(self, display_source=False, display_sensitive=False):
                 if not display_sensitive:
                     opt = '< hidden >'
                 if display_source:
-                    opt = (opt, 'bash cmd')
+                    opt = (opt, 'cmd')
+                elif raw:
+                    opt = opt.replace('%', '%%')
                 cfg.setdefault(section, OrderedDict()).update({key: opt})
+                del cfg[section][key + '_cmd']
 
         return cfg
 
@@ -352,6 +451,17 @@ def load_test_config(self):
         # then read any "custom" test settings
         self.read(TEST_CONFIG_FILE)
 
+    def _warn_deprecate(self, section, key, deprecated_name):
+        warnings.warn(
+            self.deprecation_format_string.format(
+                old=deprecated_name,
+                new=key,
+                section=section,
+            ),
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
 
 def mkdir_p(path):
     try:
@@ -364,23 +474,23 @@ def mkdir_p(path):
                 'Error creating {}: {}'.format(path, exc.strerror))
 
 
-# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using
-# "~/airflow" and "~/airflow/airflow.cfg" respectively as defaults.
+def get_airflow_home():
+    return expand_env_var(os.environ.get('AIRFLOW_HOME', '~/airflow'))
 
-if 'AIRFLOW_HOME' not in os.environ:
-    AIRFLOW_HOME = expand_env_var('~/airflow')
-else:
-    AIRFLOW_HOME = expand_env_var(os.environ['AIRFLOW_HOME'])
 
+def get_airflow_config(airflow_home):
+    if 'AIRFLOW_CONFIG' not in os.environ:
+        return os.path.join(airflow_home, 'airflow.cfg')
+    return expand_env_var(os.environ['AIRFLOW_CONFIG'])
+
+
+# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using
+# "~/airflow" and "$AIRFLOW_HOME/airflow.cfg" respectively as defaults.
+
+AIRFLOW_HOME = get_airflow_home()
+AIRFLOW_CONFIG = get_airflow_config(AIRFLOW_HOME)
 mkdir_p(AIRFLOW_HOME)
 
-if 'AIRFLOW_CONFIG' not in os.environ:
-    if os.path.isfile(expand_env_var('~/airflow.cfg')):
-        AIRFLOW_CONFIG = expand_env_var('~/airflow.cfg')
-    else:
-        AIRFLOW_CONFIG = AIRFLOW_HOME + '/airflow.cfg'
-else:
-    AIRFLOW_CONFIG = expand_env_var(os.environ['AIRFLOW_CONFIG'])
 
 # Set up dags folder for unit tests
 # this directory won't exist if users install via pip
@@ -449,12 +559,42 @@ def parameterized_config(template):
 
 conf.read(AIRFLOW_CONFIG)
 
+if conf.has_option('core', 'AIRFLOW_HOME'):
+    msg = (
+        'Specifying both AIRFLOW_HOME environment variable and airflow_home '
+        'in the config file is deprecated. Please use only the AIRFLOW_HOME '
+        'environment variable and remove the config file entry.'
+    )
+    if 'AIRFLOW_HOME' in os.environ:
+        warnings.warn(msg, category=DeprecationWarning)
+    elif conf.get('core', 'airflow_home') == AIRFLOW_HOME:
+        warnings.warn(
+            'Specifying airflow_home in the config file is deprecated. As you '
+            'have left it at the default value you should remove the setting '
+            'from your airflow.cfg and suffer no change in behaviour.',
+            category=DeprecationWarning,
+        )
+    else:
+        AIRFLOW_HOME = conf.get('core', 'airflow_home')
+        warnings.warn(msg, category=DeprecationWarning)
+
+# Warn about old config file. We used to read ~/airflow/airflow.cfg even if
+# that AIRFLOW_HOME was set to something else
+_old_config_file = os.path.expanduser("~/airflow/airflow.cfg")
+if _old_config_file != AIRFLOW_CONFIG and os.path.isfile(_old_config_file):
+    warnings.warn(
+        'You have two airflow.cfg files: {old} and {new}. Airflow used to look '
+        'at ~/airflow/airflow.cfg, even when AIRFLOW_HOME was set to a different '
+        'value. Airflow will now only read {new}, and you should remove the '
+        'other file'.format(old=_old_config_file, new=AIRFLOW_CONFIG),
+        category=DeprecationWarning,
+    )
+
 
-if conf.getboolean('webserver', 'rbac'):
-    with open(os.path.join(_templates_dir, 'default_webserver_config.py')) as f:
-        DEFAULT_WEBSERVER_CONFIG = f.read()
+WEBSERVER_CONFIG = AIRFLOW_HOME + '/webserver_config.py'
 
-    WEBSERVER_CONFIG = AIRFLOW_HOME + '/webserver_config.py'
+if conf.getboolean('webserver', 'rbac'):
+    DEFAULT_WEBSERVER_CONFIG = _read_default_config_file('default_webserver_config.py')
 
     if not os.path.isfile(WEBSERVER_CONFIG):
         log.info('Creating new FAB webserver config file in: %s', WEBSERVER_CONFIG)
diff --git a/airflow/contrib/auth/backends/github_enterprise_auth.py b/airflow/contrib/auth/backends/github_enterprise_auth.py
index 5196725156393..99c01ed2794ce 100644
--- a/airflow/contrib/auth/backends/github_enterprise_auth.py
+++ b/airflow/contrib/auth/backends/github_enterprise_auth.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Matthew Pelland (matt@pelland.io)
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -19,18 +19,14 @@
 import flask_login
 
 # Need to expose these downstream
-# pylint: disable=unused-import
-from flask_login import (current_user,
-                         logout_user,
-                         login_required,
-                         login_user)
-# pylint: enable=unused-import
+# flake8: noqa: F401
+from flask_login import current_user, logout_user, login_required, login_user
 
 from flask import url_for, redirect, request
 
 from flask_oauthlib.client import OAuth
 
-from airflow import models, configuration, settings
+from airflow import models, configuration
 from airflow.configuration import AirflowConfigException
 from airflow.utils.db import provide_session
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -47,14 +43,17 @@ class GHEUser(models.User):
     def __init__(self, user):
         self.user = user
 
+    @property
     def is_active(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_authenticated(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_anonymous(self):
         """Required by flask_login"""
         return False
diff --git a/airflow/contrib/auth/backends/google_auth.py b/airflow/contrib/auth/backends/google_auth.py
index d1a35791db888..ddbcb1222f2c0 100644
--- a/airflow/contrib/auth/backends/google_auth.py
+++ b/airflow/contrib/auth/backends/google_auth.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Ananya Mishra (am747@cornell.edu)
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -19,18 +19,14 @@
 import flask_login
 
 # Need to expose these downstream
-# pylint: disable=unused-import
-from flask_login import (current_user,
-                         logout_user,
-                         login_required,
-                         login_user)
-# pylint: enable=unused-import
+# flake8: noqa: F401
+from flask_login import current_user, logout_user, login_required, login_user
 
 from flask import url_for, redirect, request
 
 from flask_oauthlib.client import OAuth
 
-from airflow import models, configuration, settings
+from airflow import models, configuration
 from airflow.utils.db import provide_session
 from airflow.utils.log.logging_mixin import LoggingMixin
 
@@ -46,14 +42,17 @@ class GoogleUser(models.User):
     def __init__(self, user):
         self.user = user
 
+    @property
     def is_active(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_authenticated(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_anonymous(self):
         """Required by flask_login"""
         return False
@@ -113,8 +112,7 @@ def login(self, request):
         log.debug('Redirecting user to Google login')
         return self.google_oauth.authorize(callback=url_for(
             'google_oauth_callback',
-            _external=True,
-            _scheme='https'),
+            _external=True),
             state=request.args.get('next') or request.referrer or None)
 
     def get_google_user_profile_info(self, google_token):
diff --git a/airflow/contrib/auth/backends/kerberos_auth.py b/airflow/contrib/auth/backends/kerberos_auth.py
index 08be299a197dd..17b8b387c14ef 100644
--- a/airflow/contrib/auth/backends/kerberos_auth.py
+++ b/airflow/contrib/auth/backends/kerberos_auth.py
@@ -19,10 +19,10 @@
 
 import logging
 import flask_login
+from airflow.exceptions import AirflowConfigException
 from flask_login import current_user
 from flask import flash
-from wtforms import (
-    Form, PasswordField, StringField)
+from wtforms import Form, PasswordField, StringField
 from wtforms.validators import InputRequired
 
 # pykerberos should be used as it verifies the KDC, the "kerberos" module does not do so
@@ -32,7 +32,6 @@
 
 from flask import url_for, redirect
 
-from airflow import settings
 from airflow import models
 from airflow import configuration
 from airflow.utils.db import provide_session
@@ -58,7 +57,13 @@ def authenticate(username, password):
             utils.get_fqdn()
         )
         realm = configuration.conf.get("kerberos", "default_realm")
-        user_principal = utils.principal_from_username(username)
+
+        try:
+            user_realm = configuration.conf.get("security", "default_realm")
+        except AirflowConfigException:
+            user_realm = realm
+
+        user_principal = utils.principal_from_username(username, user_realm)
 
         try:
             # this is pykerberos specific, verify = True is needed to prevent KDC spoofing
@@ -68,19 +73,23 @@ def authenticate(username, password):
                 raise AuthenticationError()
         except kerberos.KrbError as e:
             logging.error(
-                'Password validation for principal %s failed %s', user_principal, e)
+                'Password validation for user '
+                '%s in realm %s failed %s', user_principal, realm, e)
             raise AuthenticationError(e)
 
         return
 
+    @property
     def is_active(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_authenticated(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_anonymous(self):
         """Required by flask_login"""
         return False
@@ -110,7 +119,7 @@ def load_user(userid, session=None):
 
 @provide_session
 def login(self, request, session=None):
-    if current_user.is_authenticated():
+    if current_user.is_authenticated:
         flash("You are already logged in")
         return redirect(url_for('index'))
 
diff --git a/airflow/contrib/auth/backends/ldap_auth.py b/airflow/contrib/auth/backends/ldap_auth.py
index eefaa1263b250..1639f43aec307 100644
--- a/airflow/contrib/auth/backends/ldap_auth.py
+++ b/airflow/contrib/auth/backends/ldap_auth.py
@@ -19,13 +19,12 @@
 from future.utils import native
 
 import flask_login
-from flask_login import login_required, current_user, logout_user
+from flask_login import login_required, current_user, logout_user  # noqa: F401
 from flask import flash
-from wtforms import (
-    Form, PasswordField, StringField)
+from wtforms import Form, PasswordField, StringField
 from wtforms.validators import InputRequired
 
-from ldap3 import Server, Connection, Tls, LEVEL, SUBTREE, BASE
+from ldap3 import Server, Connection, Tls, set_config_parameter, LEVEL, SUBTREE
 import ssl
 
 from flask import url_for, redirect
@@ -56,16 +55,26 @@ class LdapException(Exception):
 
 
 def get_ldap_connection(dn=None, password=None):
-    tls_configuration = None
-    use_ssl = False
     try:
         cacert = configuration.conf.get("ldap", "cacert")
-        tls_configuration = Tls(validate=ssl.CERT_REQUIRED, ca_certs_file=cacert)
-        use_ssl = True
-    except:
+    except AirflowConfigException:
         pass
 
-    server = Server(configuration.conf.get("ldap", "uri"), use_ssl, tls_configuration)
+    try:
+        ignore_malformed_schema = configuration.conf.get("ldap", "ignore_malformed_schema")
+    except AirflowConfigException:
+        pass
+
+    if ignore_malformed_schema:
+        set_config_parameter('IGNORE_MALFORMED_SCHEMA', ignore_malformed_schema)
+
+    tls_configuration = Tls(validate=ssl.CERT_REQUIRED,
+                            ca_certs_file=cacert)
+
+    server = Server(configuration.conf.get("ldap", "uri"),
+                    use_ssl=True,
+                    tls=tls_configuration)
+
     conn = Connection(server, native(dn), native(password))
 
     if not conn.bind():
@@ -94,7 +103,7 @@ def groups_user(conn, search_base, user_filter, user_name_att, username):
     search_filter = "(&({0})({1}={2}))".format(user_filter, user_name_att, username)
     try:
         memberof_attr = configuration.conf.get("ldap", "group_member_attr")
-    except:
+    except Exception:
         memberof_attr = "memberOf"
     res = conn.search(native(search_base), native(search_filter),
                       attributes=[native(memberof_attr)])
@@ -226,7 +235,7 @@ def try_login(username, password):
             Unable to parse LDAP structure. If you're using Active Directory
             and not specifying an OU, you must set search_scope=SUBTREE in airflow.cfg.
             %s
-            """ % traceback.format_exc())
+            """, traceback.format_exc())
             raise LdapException(
                 "Could not parse LDAP structure. "
                 "Try setting search_scope in airflow.cfg, or check logs"
@@ -236,14 +245,17 @@ def try_login(username, password):
             log.info("Password incorrect for user %s", username)
             raise AuthenticationError("Invalid username or password")
 
+    @property
     def is_active(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_authenticated(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_anonymous(self):
         """Required by flask_login"""
         return False
@@ -274,7 +286,7 @@ def load_user(userid, session=None):
 
 @provide_session
 def login(self, request, session=None):
-    if current_user.is_authenticated():
+    if current_user.is_authenticated:
         flash("You are already logged in")
         return redirect(url_for('admin.index'))
 
diff --git a/airflow/contrib/auth/backends/password_auth.py b/airflow/contrib/auth/backends/password_auth.py
index 879aaa142a4db..9d6a3ccbe6327 100644
--- a/airflow/contrib/auth/backends/password_auth.py
+++ b/airflow/contrib/auth/backends/password_auth.py
@@ -35,9 +35,8 @@
 from sqlalchemy import Column, String
 from sqlalchemy.ext.hybrid import hybrid_property
 
-from airflow import settings
 from airflow import models
-from airflow.utils.db import provide_session
+from airflow.utils.db import provide_session, create_session
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 login_manager = flask_login.LoginManager()
@@ -48,6 +47,9 @@
 PY3 = version_info[0] == 3
 
 
+client_auth = None
+
+
 class AuthenticationError(Exception):
     pass
 
@@ -71,14 +73,17 @@ def password(self, plaintext):
     def authenticate(self, plaintext):
         return check_password_hash(self._password, plaintext)
 
+    @property
     def is_active(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_authenticated(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_anonymous(self):
         """Required by flask_login"""
         return False
@@ -92,8 +97,7 @@ def data_profiling(self):
         return True
 
     def is_superuser(self):
-        """Access all the things"""
-        return True
+        return hasattr(self, 'user') and self.user.is_superuser()
 
 
 @login_manager.user_loader
@@ -137,7 +141,7 @@ def authenticate(session, username, password):
 
 @provide_session
 def login(self, request, session=None):
-    if current_user.is_authenticated():
+    if current_user.is_authenticated:
         flash("You are already logged in")
         return redirect(url_for('admin.index'))
 
@@ -160,9 +164,6 @@ def login(self, request, session=None):
         return self.render('airflow/login.html',
                            title="Airflow - Login",
                            form=form)
-    finally:
-        session.commit()
-        session.close()
 
 
 class LoginForm(Form):
@@ -196,19 +197,16 @@ def decorated(*args, **kwargs):
             userpass = ''.join(header.split()[1:])
             username, password = base64.b64decode(userpass).decode("utf-8").split(":", 1)
 
-            session = settings.Session()
-            try:
-                authenticate(session, username, password)
+            with create_session() as session:
+                try:
+                    authenticate(session, username, password)
 
-                response = function(*args, **kwargs)
-                response = make_response(response)
-                return response
+                    response = function(*args, **kwargs)
+                    response = make_response(response)
+                    return response
 
-            except AuthenticationError:
-                return _forbidden()
+                except AuthenticationError:
+                    return _forbidden()
 
-            finally:
-                session.commit()
-                session.close()
         return _unauthorized()
     return decorated
diff --git a/scripts/ci/kadm5.acl b/airflow/contrib/example_dags/__init__.py
similarity index 96%
rename from scripts/ci/kadm5.acl
rename to airflow/contrib/example_dags/__init__.py
index 691dce6c2bbdf..114d189da14ab 100644
--- a/scripts/ci/kadm5.acl
+++ b/airflow/contrib/example_dags/__init__.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -6,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-*/admin@TEST.LOCAL    *
diff --git a/airflow/contrib/example_dags/example_azure_container_instances_operator.py b/airflow/contrib/example_dags/example_azure_container_instances_operator.py
new file mode 100644
index 0000000000000..181a30b50e62f
--- /dev/null
+++ b/airflow/contrib/example_dags/example_azure_container_instances_operator.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow import DAG
+from airflow.contrib.operators.azure_container_instances_operator import AzureContainerInstancesOperator
+from datetime import datetime, timedelta
+
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'start_date': datetime(2018, 11, 1),
+    'email': ['airflow@example.com'],
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 1,
+    'retry_delay': timedelta(minutes=5),
+}
+
+dag = DAG(
+    'aci_example',
+    default_args=default_args,
+    schedule_interval=timedelta(1)
+)
+
+t1 = AzureContainerInstancesOperator(
+    ci_conn_id='azure_container_instances_default',
+    registry_conn_id=None,
+    resource_group='resource-group',
+    name='aci-test-{{ ds }}',
+    image='hello-world',
+    region='WestUS2',
+    environment_variables={},
+    volumes=[],
+    memory_in_gb=4.0,
+    cpu=1.0,
+    task_id='start_container',
+    dag=dag
+)
diff --git a/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py b/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py
new file mode 100644
index 0000000000000..dd0b83e811d1f
--- /dev/null
+++ b/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+This is only an example DAG to highlight usage of AzureCosmosDocumentSensor to detect
+if a document now exists.
+
+You can trigger this manually with `airflow trigger_dag example_cosmosdb_sensor`.
+
+*Note: Make sure that connection `azure_cosmos_default` is properly set before running
+this example.*
+"""
+
+from airflow import DAG
+from airflow.contrib.sensors.azure_cosmos_sensor import AzureCosmosDocumentSensor
+from airflow.contrib.operators.azure_cosmos_operator import AzureCosmosInsertDocumentOperator
+from airflow.utils import dates
+
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'start_date': dates.days_ago(2),
+    'email': ['airflow@example.com'],
+    'email_on_failure': False,
+    'email_on_retry': False
+}
+
+dag = DAG('example_azure_cosmosdb_sensor', default_args=default_args)
+
+dag.doc_md = __doc__
+
+t1 = AzureCosmosDocumentSensor(
+    task_id='check_cosmos_file',
+    database_name='airflow_example_db',
+    collection_name='airflow_example_coll',
+    document_id='airflow_checkid',
+    azure_cosmos_conn_id='azure_cosmos_default',
+    dag=dag)
+
+t2 = AzureCosmosInsertDocumentOperator(
+    task_id='insert_cosmos_file',
+    dag=dag,
+    database_name='airflow_example_db',
+    collection_name='new-collection',
+    document={"id": "someuniqueid", "param1": "value1", "param2": "value2"},
+    azure_cosmos_conn_id='azure_cosmos_default')
+
+t1 >> t2
diff --git a/airflow/contrib/example_dags/example_databricks_operator.py b/airflow/contrib/example_dags/example_databricks_operator.py
index bc827d465bd3a..79f947ba1c123 100644
--- a/airflow/contrib/example_dags/example_databricks_operator.py
+++ b/airflow/contrib/example_dags/example_databricks_operator.py
@@ -32,7 +32,7 @@
 # the spark jar task will NOT run until the notebook task completes
 # successfully.
 #
-# The definition of a succesful run is if the run has a result_state of "SUCCESS".
+# The definition of a successful run is if the run has a result_state of "SUCCESS".
 # For more information about the state of a run refer to
 # https://docs.databricks.com/api/latest/jobs.html#runstate
 
diff --git a/airflow/contrib/example_dags/example_dingding_operator.py b/airflow/contrib/example_dags/example_dingding_operator.py
new file mode 100644
index 0000000000000..c6d8cca6fdaad
--- /dev/null
+++ b/airflow/contrib/example_dags/example_dingding_operator.py
@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datetime import timedelta
+
+import airflow
+from airflow.contrib.operators.dingding_operator import DingdingOperator
+from airflow.models import DAG
+
+args = {
+    'owner': 'airflow',
+    'retries': 3,
+    'start_date': airflow.utils.dates.days_ago(2),
+}
+
+
+# [START howto_operator_dingding_failure_callback]
+def failure_callback(context):
+    message = 'AIRFLOW TASK FAILURE TIPS:\n' \
+              'DAG:    {}\n' \
+              'TASKS:  {}\n' \
+              'Reason: {}\n' \
+        .format(context['task_instance'].dag_id,
+                context['task_instance'].task_id,
+                context['exception'])
+    return DingdingOperator(
+        task_id='dingding_success_callback',
+        dingding_conn_id='dingding_default',
+        message_type='text',
+        message=message,
+        at_all=True,
+    ).execute(context)
+
+
+args['on_failure_callback'] = failure_callback
+# [END howto_operator_dingding_failure_callback]
+
+dag = DAG(
+    dag_id='example_dingding_operator',
+    default_args=args,
+    schedule_interval='@once',
+    dagrun_timeout=timedelta(minutes=60),
+)
+
+# [START howto_operator_dingding]
+text_msg_remind_none = DingdingOperator(
+    task_id='text_msg_remind_none',
+    dingding_conn_id='dingding_default',
+    message_type='text',
+    message='Airflow dingding text message remind none',
+    at_mobiles=None,
+    at_all=False,
+    dag=dag,
+)
+# [END howto_operator_dingding]
+
+text_msg_remind_specific = DingdingOperator(
+    task_id='text_msg_remind_specific',
+    dingding_conn_id='dingding_default',
+    message_type='text',
+    message='Airflow dingding text message remind specific users',
+    at_mobiles=['156XXXXXXXX', '130XXXXXXXX'],
+    at_all=False,
+    dag=dag,
+)
+
+text_msg_remind_include_invalid = DingdingOperator(
+    task_id='text_msg_remind_include_invalid',
+    dingding_conn_id='dingding_default',
+    message_type='text',
+    message='Airflow dingding text message remind users including invalid',
+    # 123 is invalid user or user not in the group
+    at_mobiles=['156XXXXXXXX', '123'],
+    at_all=False,
+    dag=dag,
+)
+
+# [START howto_operator_dingding_remind_users]
+text_msg_remind_all = DingdingOperator(
+    task_id='text_msg_remind_all',
+    dingding_conn_id='dingding_default',
+    message_type='text',
+    message='Airflow dingding text message remind all users in group',
+    # list of user phone/email here in the group
+    # when at_all is specific will cover at_mobiles
+    at_mobiles=['156XXXXXXXX', '130XXXXXXXX'],
+    at_all=True,
+    dag=dag,
+)
+# [END howto_operator_dingding_remind_users]
+
+link_msg = DingdingOperator(
+    task_id='link_msg',
+    dingding_conn_id='dingding_default',
+    message_type='link',
+    message={
+        'title': 'Airflow dingding link message',
+        'text': 'Airflow official documentation link',
+        'messageUrl': 'http://airflow.apache.org',
+        'picURL': 'http://airflow.apache.org/_images/pin_large.png'
+    },
+    dag=dag,
+)
+
+# [START howto_operator_dingding_rich_text]
+markdown_msg = DingdingOperator(
+    task_id='markdown_msg',
+    dingding_conn_id='dingding_default',
+    message_type='markdown',
+    message={
+        'title': 'Airflow dingding markdown message',
+        'text': '# Markdown message title\n'
+                'content content .. \n'
+                '### sub-title\n'
+                '![logo](http://airflow.apache.org/_images/pin_large.png)'
+    },
+    at_mobiles=['156XXXXXXXX'],
+    at_all=False,
+    dag=dag,
+)
+# [END howto_operator_dingding_rich_text]
+
+single_action_card_msg = DingdingOperator(
+    task_id='single_action_card_msg',
+    dingding_conn_id='dingding_default',
+    message_type='actionCard',
+    message={
+        'title': 'Airflow dingding single actionCard message',
+        'text': 'Airflow dingding single actionCard message\n'
+                '![logo](http://airflow.apache.org/_images/pin_large.png)\n'
+                'This is a official logo in Airflow website.',
+        'hideAvatar': '0',
+        'btnOrientation': '0',
+        'singleTitle': 'read more',
+        'singleURL': 'http://airflow.apache.org'
+    },
+    dag=dag,
+)
+
+multi_action_card_msg = DingdingOperator(
+    task_id='multi_action_card_msg',
+    dingding_conn_id='dingding_default',
+    message_type='actionCard',
+    message={
+        'title': 'Airflow dingding multi actionCard message',
+        'text': 'Airflow dingding multi actionCard message\n'
+                '![logo](http://airflow.apache.org/_images/pin_large.png)\n'
+                'Airflow documentation and github',
+        'hideAvatar': '0',
+        'btnOrientation': '0',
+        'btns': [
+            {
+                'title': 'Airflow Documentation',
+                'actionURL': 'http://airflow.apache.org'
+            },
+            {
+                'title': 'Airflow Github',
+                'actionURL': 'https://github.com/apache/airflow'
+            }
+        ]
+    },
+    dag=dag,
+)
+
+feed_card_msg = DingdingOperator(
+    task_id='feed_card_msg',
+    dingding_conn_id='dingding_default',
+    message_type='feedCard',
+    message={
+        "links": [
+            {
+                "title": "Airflow DAG feed card",
+                "messageURL": "https://airflow.readthedocs.io/en/latest/ui.html",
+                "picURL": "http://airflow.apache.org/_images/dags.png"
+            },
+            {
+                "title": "Airflow tree feed card",
+                "messageURL": "https://airflow.readthedocs.io/en/latest/ui.html",
+                "picURL": "http://airflow.apache.org/_images/tree.png"
+            },
+            {
+                "title": "Airflow graph feed card",
+                "messageURL": "https://airflow.readthedocs.io/en/latest/ui.html",
+                "picURL": "http://airflow.apache.org/_images/graph.png"
+            }
+        ]
+    },
+    dag=dag,
+)
+
+msg_failure_callback = DingdingOperator(
+    task_id='msg_failure_callback',
+    dingding_conn_id='dingding_default',
+    message_type='not_support_msg_type',
+    message="",
+    dag=dag,
+)
+
+[
+    text_msg_remind_none,
+    text_msg_remind_specific,
+    text_msg_remind_include_invalid,
+    text_msg_remind_all
+] >> link_msg >> markdown_msg >> [
+    single_action_card_msg,
+    multi_action_card_msg
+] >> feed_card_msg >> msg_failure_callback
diff --git a/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py b/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py
index 48a178a2a954c..ba5b923007511 100644
--- a/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py
+++ b/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py
@@ -54,8 +54,7 @@
 ]
 
 JOB_FLOW_OVERRIDES = {
-    'Name': 'PiCalc',
-    'KeepJobFlowAliveWhenNoSteps': True
+    'Name': 'PiCalc'
 }
 
 dag = DAG(
diff --git a/airflow/contrib/example_dags/example_gcp_bigtable_operators.py b/airflow/contrib/example_dags/example_gcp_bigtable_operators.py
new file mode 100644
index 0000000000000..aca820a0ecb41
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_bigtable_operators.py
@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# 'License'); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# noinspection LongLine
+"""
+Example Airflow DAG that creates and performs following operations on Cloud Bigtable:
+- creates an Instance
+- creates a Table
+- updates Cluster
+- waits for Table replication completeness
+- deletes the Table
+- deletes the Instance
+
+This DAG relies on the following environment variables:
+
+* GCP_PROJECT_ID - Google Cloud Platform project
+* CBT_INSTANCE_ID - desired ID of a Cloud Bigtable instance
+* CBT_INSTANCE_DISPLAY_NAME - desired human-readable display name of the Instance
+* CBT_INSTANCE_TYPE - type of the Instance, e.g. 1 for DEVELOPMENT
+    See https://googleapis.github.io/google-cloud-python/latest/bigtable/instance.html#google.cloud.bigtable.instance.Instance # noqa E501
+* CBT_INSTANCE_LABELS - labels to add for the Instance
+* CBT_CLUSTER_ID - desired ID of the main Cluster created for the Instance
+* CBT_CLUSTER_ZONE - zone in which main Cluster will be created. e.g. europe-west1-b
+    See available zones: https://cloud.google.com/bigtable/docs/locations
+* CBT_CLUSTER_NODES - initial amount of nodes of the Cluster
+* CBT_CLUSTER_NODES_UPDATED - amount of nodes for BigtableClusterUpdateOperator
+* CBT_CLUSTER_STORAGE_TYPE - storage for the Cluster, e.g. 1 for SSD
+    See https://googleapis.github.io/google-cloud-python/latest/bigtable/instance.html#google.cloud.bigtable.instance.Instance.cluster # noqa E501
+* CBT_TABLE_ID - desired ID of the Table
+* CBT_POKE_INTERVAL - number of seconds between every attempt of Sensor check
+
+"""
+import json
+
+from os import getenv
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_bigtable_operator import \
+    BigtableInstanceCreateOperator, \
+    BigtableInstanceDeleteOperator, \
+    BigtableClusterUpdateOperator, \
+    BigtableTableCreateOperator, \
+    BigtableTableWaitForReplicationSensor, \
+    BigtableTableDeleteOperator
+
+# [START howto_operator_gcp_bigtable_args]
+GCP_PROJECT_ID = getenv('GCP_PROJECT_ID', 'example-project')
+CBT_INSTANCE_ID = getenv('CBT_INSTANCE_ID', 'some-instance-id')
+CBT_INSTANCE_DISPLAY_NAME = getenv('CBT_INSTANCE_DISPLAY_NAME', 'Human-readable name')
+CBT_INSTANCE_TYPE = getenv('CBT_INSTANCE_TYPE', '2')
+CBT_INSTANCE_LABELS = getenv('CBT_INSTANCE_LABELS', '{}')
+CBT_CLUSTER_ID = getenv('CBT_CLUSTER_ID', 'some-cluster-id')
+CBT_CLUSTER_ZONE = getenv('CBT_CLUSTER_ZONE', 'europe-west1-b')
+CBT_CLUSTER_NODES = getenv('CBT_CLUSTER_NODES', '3')
+CBT_CLUSTER_NODES_UPDATED = getenv('CBT_CLUSTER_NODES_UPDATED', '5')
+CBT_CLUSTER_STORAGE_TYPE = getenv('CBT_CLUSTER_STORAGE_TYPE', '2')
+CBT_TABLE_ID = getenv('CBT_TABLE_ID', 'some-table-id')
+CBT_POKE_INTERVAL = getenv('CBT_POKE_INTERVAL', '60')
+# [END howto_operator_gcp_bigtable_args]
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+
+with models.DAG(
+    'example_gcp_bigtable_operators',
+    default_args=default_args,
+    schedule_interval=None  # Override to match your needs
+) as dag:
+    # [START howto_operator_gcp_bigtable_instance_create]
+    create_instance_task = BigtableInstanceCreateOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=CBT_INSTANCE_ID,
+        main_cluster_id=CBT_CLUSTER_ID,
+        main_cluster_zone=CBT_CLUSTER_ZONE,
+        instance_display_name=CBT_INSTANCE_DISPLAY_NAME,
+        instance_type=int(CBT_INSTANCE_TYPE),
+        instance_labels=json.loads(CBT_INSTANCE_LABELS),
+        cluster_nodes=int(CBT_CLUSTER_NODES),
+        cluster_storage_type=int(CBT_CLUSTER_STORAGE_TYPE),
+        task_id='create_instance_task',
+    )
+    create_instance_task2 = BigtableInstanceCreateOperator(
+        instance_id=CBT_INSTANCE_ID,
+        main_cluster_id=CBT_CLUSTER_ID,
+        main_cluster_zone=CBT_CLUSTER_ZONE,
+        instance_display_name=CBT_INSTANCE_DISPLAY_NAME,
+        instance_type=int(CBT_INSTANCE_TYPE),
+        instance_labels=json.loads(CBT_INSTANCE_LABELS),
+        cluster_nodes=int(CBT_CLUSTER_NODES),
+        cluster_storage_type=int(CBT_CLUSTER_STORAGE_TYPE),
+        task_id='create_instance_task2',
+    )
+    create_instance_task >> create_instance_task2
+    # [END howto_operator_gcp_bigtable_instance_create]
+
+    # [START howto_operator_gcp_bigtable_cluster_update]
+    cluster_update_task = BigtableClusterUpdateOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=CBT_INSTANCE_ID,
+        cluster_id=CBT_CLUSTER_ID,
+        nodes=int(CBT_CLUSTER_NODES_UPDATED),
+        task_id='update_cluster_task',
+    )
+    cluster_update_task2 = BigtableClusterUpdateOperator(
+        instance_id=CBT_INSTANCE_ID,
+        cluster_id=CBT_CLUSTER_ID,
+        nodes=int(CBT_CLUSTER_NODES_UPDATED),
+        task_id='update_cluster_task2',
+    )
+    cluster_update_task >> cluster_update_task2
+    # [END howto_operator_gcp_bigtable_cluster_update]
+
+    # [START howto_operator_gcp_bigtable_instance_delete]
+    delete_instance_task = BigtableInstanceDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=CBT_INSTANCE_ID,
+        task_id='delete_instance_task',
+    )
+    delete_instance_task2 = BigtableInstanceDeleteOperator(
+        instance_id=CBT_INSTANCE_ID,
+        task_id='delete_instance_task2',
+    )
+    # [END howto_operator_gcp_bigtable_instance_delete]
+
+    # [START howto_operator_gcp_bigtable_table_create]
+    create_table_task = BigtableTableCreateOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=CBT_INSTANCE_ID,
+        table_id=CBT_TABLE_ID,
+        task_id='create_table',
+    )
+    create_table_task2 = BigtableTableCreateOperator(
+        instance_id=CBT_INSTANCE_ID,
+        table_id=CBT_TABLE_ID,
+        task_id='create_table_task2',
+    )
+    create_table_task >> create_table_task2
+    # [END howto_operator_gcp_bigtable_table_create]
+
+    # [START howto_operator_gcp_bigtable_table_wait_for_replication]
+    wait_for_table_replication_task = BigtableTableWaitForReplicationSensor(
+        project_id=GCP_PROJECT_ID,
+        instance_id=CBT_INSTANCE_ID,
+        table_id=CBT_TABLE_ID,
+        poke_interval=int(CBT_POKE_INTERVAL),
+        timeout=180,
+        task_id='wait_for_table_replication_task',
+    )
+    wait_for_table_replication_task2 = BigtableTableWaitForReplicationSensor(
+        instance_id=CBT_INSTANCE_ID,
+        table_id=CBT_TABLE_ID,
+        poke_interval=int(CBT_POKE_INTERVAL),
+        timeout=180,
+        task_id='wait_for_table_replication_task2',
+    )
+    # [END howto_operator_gcp_bigtable_table_wait_for_replication]
+
+    # [START howto_operator_gcp_bigtable_table_delete]
+    delete_table_task = BigtableTableDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=CBT_INSTANCE_ID,
+        table_id=CBT_TABLE_ID,
+        task_id='delete_table_task',
+    )
+    delete_table_task2 = BigtableTableDeleteOperator(
+        instance_id=CBT_INSTANCE_ID,
+        table_id=CBT_TABLE_ID,
+        task_id='delete_table_task2',
+    )
+    # [END howto_operator_gcp_bigtable_table_delete]
+
+    wait_for_table_replication_task >> delete_table_task
+    wait_for_table_replication_task2 >> delete_table_task
+    wait_for_table_replication_task >> delete_table_task2
+    wait_for_table_replication_task2 >> delete_table_task2
+    create_instance_task \
+        >> create_table_task \
+        >> cluster_update_task \
+        >> delete_table_task
+    create_instance_task2 \
+        >> create_table_task2 \
+        >> cluster_update_task2 \
+        >> delete_table_task2
+
+    # Only delete instances after all tables are deleted
+    [delete_table_task, delete_table_task2] >> \
+        delete_instance_task >> delete_instance_task2
diff --git a/airflow/contrib/example_dags/example_gcp_compute.py b/airflow/contrib/example_dags/example_gcp_compute.py
new file mode 100644
index 0000000000000..dde0f5975ae6f
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_compute.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that starts, stops and sets the machine type of a Google Compute
+Engine instance.
+
+This DAG relies on the following OS environment variables
+
+* GCP_PROJECT_ID - Google Cloud Platform project where the Compute Engine instance exists.
+* GCE_ZONE - Google Cloud Platform zone where the instance exists.
+* GCE_INSTANCE - Name of the Compute Engine instance.
+* GCE_SHORT_MACHINE_TYPE_NAME - Machine type resource name to set, e.g. 'n1-standard-1'.
+    See https://cloud.google.com/compute/docs/machine-types
+"""
+import os
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_compute_operator import GceInstanceStartOperator, \
+    GceInstanceStopOperator, GceSetMachineTypeOperator
+
+# [START howto_operator_gce_args_common]
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCE_ZONE = os.environ.get('GCE_ZONE', 'europe-west1-b')
+GCE_INSTANCE = os.environ.get('GCE_INSTANCE', 'testinstance')
+# [END howto_operator_gce_args_common]
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1),
+}
+
+# [START howto_operator_gce_args_set_machine_type]
+GCE_SHORT_MACHINE_TYPE_NAME = os.environ.get('GCE_SHORT_MACHINE_TYPE_NAME', 'n1-standard-1')
+SET_MACHINE_TYPE_BODY = {
+    'machineType': 'zones/{}/machineTypes/{}'.format(GCE_ZONE, GCE_SHORT_MACHINE_TYPE_NAME)
+}
+# [END howto_operator_gce_args_set_machine_type]
+
+
+with models.DAG(
+    'example_gcp_compute',
+    default_args=default_args,
+    schedule_interval=None  # Override to match your needs
+) as dag:
+    # [START howto_operator_gce_start]
+    gce_instance_start = GceInstanceStartOperator(
+        project_id=GCP_PROJECT_ID,
+        zone=GCE_ZONE,
+        resource_id=GCE_INSTANCE,
+        task_id='gcp_compute_start_task'
+    )
+    # [END howto_operator_gce_start]
+    # Duplicate start for idempotence testing
+    # [START howto_operator_gce_start_no_project_id]
+    gce_instance_start2 = GceInstanceStartOperator(
+        zone=GCE_ZONE,
+        resource_id=GCE_INSTANCE,
+        task_id='gcp_compute_start_task2'
+    )
+    # [END howto_operator_gce_start_no_project_id]
+    # [START howto_operator_gce_stop]
+    gce_instance_stop = GceInstanceStopOperator(
+        project_id=GCP_PROJECT_ID,
+        zone=GCE_ZONE,
+        resource_id=GCE_INSTANCE,
+        task_id='gcp_compute_stop_task'
+    )
+    # [END howto_operator_gce_stop]
+    # Duplicate stop for idempotence testing
+    # [START howto_operator_gce_stop_no_project_id]
+    gce_instance_stop2 = GceInstanceStopOperator(
+        zone=GCE_ZONE,
+        resource_id=GCE_INSTANCE,
+        task_id='gcp_compute_stop_task2'
+    )
+    # [END howto_operator_gce_stop_no_project_id]
+    # [START howto_operator_gce_set_machine_type]
+    gce_set_machine_type = GceSetMachineTypeOperator(
+        project_id=GCP_PROJECT_ID,
+        zone=GCE_ZONE,
+        resource_id=GCE_INSTANCE,
+        body=SET_MACHINE_TYPE_BODY,
+        task_id='gcp_compute_set_machine_type'
+    )
+    # [END howto_operator_gce_set_machine_type]
+    # Duplicate set machine type for idempotence testing
+    # [START howto_operator_gce_set_machine_type_no_project_id]
+    gce_set_machine_type2 = GceSetMachineTypeOperator(
+        zone=GCE_ZONE,
+        resource_id=GCE_INSTANCE,
+        body=SET_MACHINE_TYPE_BODY,
+        task_id='gcp_compute_set_machine_type2'
+    )
+    # [END howto_operator_gce_set_machine_type_no_project_id]
+
+    gce_instance_start >> gce_instance_start2 >> gce_instance_stop >> \
+        gce_instance_stop2 >> gce_set_machine_type >> gce_set_machine_type2
diff --git a/airflow/contrib/example_dags/example_gcp_compute_igm.py b/airflow/contrib/example_dags/example_gcp_compute_igm.py
new file mode 100644
index 0000000000000..103cbfd590c15
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_compute_igm.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that uses IGM-type compute operations:
+* copy of Instance Template
+* update template in Instance Group Manager
+
+This DAG relies on the following OS environment variables
+
+* GCP_PROJECT_ID - the Google Cloud Platform project where the Compute Engine instance exists
+* GCE_ZONE - the zone where the Compute Engine instance exists
+
+Variables for copy template operator:
+* GCE_TEMPLATE_NAME - name of the template to copy
+* GCE_NEW_TEMPLATE_NAME - name of the new template
+* GCE_NEW_DESCRIPTION - description added to the template
+
+Variables for update template in Group Manager:
+
+* GCE_INSTANCE_GROUP_MANAGER_NAME - name of the Instance Group Manager
+* SOURCE_TEMPLATE_URL - url of the template to replace in the Instance Group Manager
+* DESTINATION_TEMPLATE_URL - url of the new template to set in the Instance Group Manager
+"""
+
+import os
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_compute_operator import \
+    GceInstanceTemplateCopyOperator, GceInstanceGroupManagerUpdateTemplateOperator
+
+# [START howto_operator_compute_igm_common_args]
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCE_ZONE = os.environ.get('GCE_ZONE', 'europe-west1-b')
+# [END howto_operator_compute_igm_common_args]
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+
+# [START howto_operator_compute_template_copy_args]
+GCE_TEMPLATE_NAME = os.environ.get('GCE_TEMPLATE_NAME', 'instance-template-test')
+GCE_NEW_TEMPLATE_NAME = os.environ.get('GCE_NEW_TEMPLATE_NAME',
+                                       'instance-template-test-new')
+GCE_NEW_DESCRIPTION = os.environ.get('GCE_NEW_DESCRIPTION', 'Test new description')
+GCE_INSTANCE_TEMPLATE_BODY_UPDATE = {
+    "name": GCE_NEW_TEMPLATE_NAME,
+    "description": GCE_NEW_DESCRIPTION,
+    "properties": {
+        "machineType": "n1-standard-2"
+    }
+}
+# [END howto_operator_compute_template_copy_args]
+
+# [START howto_operator_compute_igm_update_template_args]
+GCE_INSTANCE_GROUP_MANAGER_NAME = os.environ.get('GCE_INSTANCE_GROUP_MANAGER_NAME',
+                                                 'instance-group-test')
+
+SOURCE_TEMPLATE_URL = os.environ.get(
+    'SOURCE_TEMPLATE_URL',
+    "https://www.googleapis.com/compute/beta/projects/" + GCP_PROJECT_ID +
+    "/global/instanceTemplates/instance-template-test")
+
+DESTINATION_TEMPLATE_URL = os.environ.get(
+    'DESTINATION_TEMPLATE_URL',
+    "https://www.googleapis.com/compute/beta/projects/" + GCP_PROJECT_ID +
+    "/global/instanceTemplates/" + GCE_NEW_TEMPLATE_NAME)
+
+UPDATE_POLICY = {
+    "type": "OPPORTUNISTIC",
+    "minimalAction": "RESTART",
+    "maxSurge": {
+        "fixed": 1
+    },
+    "minReadySec": 1800
+}
+
+# [END howto_operator_compute_igm_update_template_args]
+
+
+with models.DAG(
+    'example_gcp_compute_igm',
+    default_args=default_args,
+    schedule_interval=None  # Override to match your needs
+) as dag:
+    # [START howto_operator_gce_igm_copy_template]
+    gce_instance_template_copy = GceInstanceTemplateCopyOperator(
+        project_id=GCP_PROJECT_ID,
+        resource_id=GCE_TEMPLATE_NAME,
+        body_patch=GCE_INSTANCE_TEMPLATE_BODY_UPDATE,
+        task_id='gcp_compute_igm_copy_template_task'
+    )
+    # [END howto_operator_gce_igm_copy_template]
+    # Added to check for idempotence
+    # [START howto_operator_gce_igm_copy_template_no_project_id]
+    gce_instance_template_copy2 = GceInstanceTemplateCopyOperator(
+        resource_id=GCE_TEMPLATE_NAME,
+        body_patch=GCE_INSTANCE_TEMPLATE_BODY_UPDATE,
+        task_id='gcp_compute_igm_copy_template_task_2'
+    )
+    # [END howto_operator_gce_igm_copy_template_no_project_id]
+    # [START howto_operator_gce_igm_update_template]
+    gce_instance_group_manager_update_template = \
+        GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            zone=GCE_ZONE,
+            source_template=SOURCE_TEMPLATE_URL,
+            destination_template=DESTINATION_TEMPLATE_URL,
+            update_policy=UPDATE_POLICY,
+            task_id='gcp_compute_igm_group_manager_update_template'
+        )
+    # [END howto_operator_gce_igm_update_template]
+    # Added to check for idempotence (and without UPDATE_POLICY)
+    # [START howto_operator_gce_igm_update_template_no_project_id]
+    gce_instance_group_manager_update_template2 = \
+        GceInstanceGroupManagerUpdateTemplateOperator(
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            zone=GCE_ZONE,
+            source_template=SOURCE_TEMPLATE_URL,
+            destination_template=DESTINATION_TEMPLATE_URL,
+            task_id='gcp_compute_igm_group_manager_update_template_2'
+        )
+    # [END howto_operator_gce_igm_update_template_no_project_id]
+    gce_instance_template_copy >> gce_instance_template_copy2 >> \
+        gce_instance_group_manager_update_template >> \
+        gce_instance_group_manager_update_template2
diff --git a/airflow/contrib/example_dags/example_gcp_function.py b/airflow/contrib/example_dags/example_gcp_function.py
new file mode 100644
index 0000000000000..75dcf2ece3816
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_function.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that displays interactions with Google Cloud Functions.
+It creates a function and then deletes it.
+
+This DAG relies on the following OS environment variables
+https://airflow.apache.org/concepts.html#variables
+
+* GCP_PROJECT_ID - Google Cloud Project to use for the Cloud Function.
+* GCP_LOCATION - Google Cloud Functions region where the function should be
+  created.
+* GCF_ENTRYPOINT - Name of the executable function in the source code.
+* and one of the below:
+
+    * GCF_SOURCE_ARCHIVE_URL - Path to the zipped source in Google Cloud Storage
+
+    * GCF_SOURCE_UPLOAD_URL - Generated upload URL for the zipped source and GCF_ZIP_PATH - Local path to
+      the zipped source archive
+
+    * GCF_SOURCE_REPOSITORY - The URL pointing to the hosted repository where the function
+      is defined in a supported Cloud Source Repository URL format
+      https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#SourceRepository
+
+"""
+
+import os
+
+from airflow import models
+from airflow.contrib.operators.gcp_function_operator \
+    import GcfFunctionDeployOperator, GcfFunctionDeleteOperator
+from airflow.utils import dates
+
+# [START howto_operator_gcf_common_variables]
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_LOCATION = os.environ.get('GCP_LOCATION', 'europe-west1')
+GCF_SHORT_FUNCTION_NAME = os.environ.get('GCF_SHORT_FUNCTION_NAME', 'hello').\
+    replace("-", "_")  # make sure there are no dashes in function name (!)
+FUNCTION_NAME = 'projects/{}/locations/{}/functions/{}'.format(GCP_PROJECT_ID,
+                                                               GCP_LOCATION,
+                                                               GCF_SHORT_FUNCTION_NAME)
+# [END howto_operator_gcf_common_variables]
+# [START howto_operator_gcf_deploy_variables]
+GCF_SOURCE_ARCHIVE_URL = os.environ.get('GCF_SOURCE_ARCHIVE_URL', '')
+GCF_SOURCE_UPLOAD_URL = os.environ.get('GCF_SOURCE_UPLOAD_URL', '')
+GCF_SOURCE_REPOSITORY = os.environ.get(
+    'GCF_SOURCE_REPOSITORY',
+    'https://source.developers.google.com/'
+    'projects/{}/repos/hello-world/moveable-aliases/master'.format(GCP_PROJECT_ID))
+GCF_ZIP_PATH = os.environ.get('GCF_ZIP_PATH', '')
+GCF_ENTRYPOINT = os.environ.get('GCF_ENTRYPOINT', 'helloWorld')
+GCF_RUNTIME = 'nodejs6'
+GCP_VALIDATE_BODY = os.environ.get('GCP_VALIDATE_BODY', True)
+# [END howto_operator_gcf_deploy_variables]
+
+# [START howto_operator_gcf_deploy_body]
+body = {
+    "name": FUNCTION_NAME,
+    "entryPoint": GCF_ENTRYPOINT,
+    "runtime": GCF_RUNTIME,
+    "httpsTrigger": {}
+}
+# [END howto_operator_gcf_deploy_body]
+
+# [START howto_operator_gcf_default_args]
+default_args = {
+    'start_date': dates.days_ago(1)
+}
+# [END howto_operator_gcf_default_args]
+
+# [START howto_operator_gcf_deploy_variants]
+if GCF_SOURCE_ARCHIVE_URL:
+    body['sourceArchiveUrl'] = GCF_SOURCE_ARCHIVE_URL
+elif GCF_SOURCE_REPOSITORY:
+    body['sourceRepository'] = {
+        'url': GCF_SOURCE_REPOSITORY
+    }
+elif GCF_ZIP_PATH:
+    body['sourceUploadUrl'] = ''
+    default_args['zip_path'] = GCF_ZIP_PATH
+elif GCF_SOURCE_UPLOAD_URL:
+    body['sourceUploadUrl'] = GCF_SOURCE_UPLOAD_URL
+else:
+    raise Exception("Please provide one of the source_code parameters")
+# [END howto_operator_gcf_deploy_variants]
+
+
+with models.DAG(
+    'example_gcp_function',
+    default_args=default_args,
+    schedule_interval=None  # Override to match your needs
+) as dag:
+    # [START howto_operator_gcf_deploy]
+    deploy_task = GcfFunctionDeployOperator(
+        task_id="gcf_deploy_task",
+        project_id=GCP_PROJECT_ID,
+        location=GCP_LOCATION,
+        body=body,
+        validate_body=GCP_VALIDATE_BODY
+    )
+    # [END howto_operator_gcf_deploy]
+    # [START howto_operator_gcf_deploy_no_project_id]
+    deploy2_task = GcfFunctionDeployOperator(
+        task_id="gcf_deploy2_task",
+        location=GCP_LOCATION,
+        body=body,
+        validate_body=GCP_VALIDATE_BODY
+    )
+    # [END howto_operator_gcf_deploy_no_project_id]
+    # [START howto_operator_gcf_delete]
+    delete_task = GcfFunctionDeleteOperator(
+        task_id="gcf_delete_task",
+        name=FUNCTION_NAME
+    )
+    # [END howto_operator_gcf_delete]
+    deploy_task >> deploy2_task >> delete_task
diff --git a/airflow/contrib/example_dags/example_gcp_natural_language.py b/airflow/contrib/example_dags/example_gcp_natural_language.py
new file mode 100644
index 0000000000000..48ace4030bf17
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_natural_language.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google Cloud Natural Language service
+"""
+
+
+from google.cloud.language_v1.proto.language_service_pb2 import Document
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_natural_language_operator import (
+    CloudLanguageAnalyzeEntitiesOperator,
+    CloudLanguageAnalyzeEntitySentimentOperator,
+    CloudLanguageAnalyzeSentimentOperator,
+    CloudLanguageClassifyTextOperator,
+)
+from airflow.operators.bash_operator import BashOperator
+
+# [START howto_operator_gcp_natural_language_document_text]
+TEXT = """
+Airflow is a platform to programmatically author, schedule and monitor workflows.
+
+Use Airflow to author workflows as Directed Acyclic Graphs (DAGs) of tasks. The Airflow scheduler executes
+ your tasks on an array of workers while following the specified dependencies. Rich command line utilities
+ make performing complex surgeries on DAGs a snap. The rich user interface makes it easy to visualize
+ pipelines running in production, monitor progress, and troubleshoot issues when needed.
+"""
+document = Document(content=TEXT, type="PLAIN_TEXT")
+# [END howto_operator_gcp_natural_language_document_text]
+
+# [START howto_operator_gcp_natural_language_document_gcs]
+GCS_CONTENT_URI = "gs://my-text-bucket/sentiment-me.txt"
+document_gcs = Document(gcs_content_uri=GCS_CONTENT_URI, type="PLAIN_TEXT")
+# [END howto_operator_gcp_natural_language_document_gcs]
+
+
+default_args = {"start_date": airflow.utils.dates.days_ago(1)}
+
+with models.DAG(
+    "example_gcp_natural_language",
+    default_args=default_args,
+    schedule_interval=None,  # Override to match your needs
+) as dag:
+
+    # [START howto_operator_gcp_natural_language_analyze_entities]
+    analyze_entities = CloudLanguageAnalyzeEntitiesOperator(document=document, task_id="analyze_entities")
+    # [END howto_operator_gcp_natural_language_analyze_entities]
+
+    # [START howto_operator_gcp_natural_language_analyze_entities_result]
+    analyze_entities_result = BashOperator(
+        bash_command="echo \"{{ task_instance.xcom_pull('analyze_entities') }}\"",
+        task_id="analyze_entities_result",
+    )
+    # [END howto_operator_gcp_natural_language_analyze_entities_result]
+
+    # [START howto_operator_gcp_natural_language_analyze_entity_sentiment]
+    analyze_entity_sentiment = CloudLanguageAnalyzeEntitySentimentOperator(
+        document=document, task_id="analyze_entity_sentiment"
+    )
+    # [END howto_operator_gcp_natural_language_analyze_entity_sentiment]
+
+    # [START howto_operator_gcp_natural_language_analyze_entity_sentiment_result]
+    analyze_entity_sentiment_result = BashOperator(
+        bash_command="echo \"{{ task_instance.xcom_pull('analyze_entity_sentiment') }}\"",
+        task_id="analyze_entity_sentiment_result",
+    )
+    # [END howto_operator_gcp_natural_language_analyze_entity_sentiment_result]
+
+    # [START howto_operator_gcp_natural_language_analyze_sentiment]
+    analyze_sentiment = CloudLanguageAnalyzeSentimentOperator(document=document, task_id="analyze_sentiment")
+    # [END howto_operator_gcp_natural_language_analyze_sentiment]
+
+    # [START howto_operator_gcp_natural_language_analyze_sentiment_result]
+    analyze_sentiment_result = BashOperator(
+        bash_command="echo \"{{ task_instance.xcom_pull('analyze_sentiment') }}\"",
+        task_id="analyze_sentiment_result",
+    )
+    # [END howto_operator_gcp_natural_language_analyze_sentiment_result]
+
+    # [START howto_operator_gcp_natural_language_analyze_classify_text]
+    analyze_classify_text = CloudLanguageClassifyTextOperator(
+        document=document, task_id="analyze_classify_text"
+    )
+    # [END howto_operator_gcp_natural_language_analyze_classify_text]
+
+    # [START howto_operator_gcp_natural_language_analyze_classify_text_result]
+    analyze_classify_text_result = BashOperator(
+        bash_command="echo \"{{ task_instance.xcom_pull('analyze_classify_text') }}\"",
+        task_id="analyze_classify_text_result",
+    )
+    # [END howto_operator_gcp_natural_language_analyze_classify_text_result]
+
+    analyze_entities >> analyze_entities_result
+    analyze_entity_sentiment >> analyze_entity_sentiment_result
+    analyze_sentiment >> analyze_sentiment_result
+    analyze_classify_text >> analyze_classify_text_result
diff --git a/airflow/contrib/example_dags/example_gcp_spanner.py b/airflow/contrib/example_dags/example_gcp_spanner.py
new file mode 100644
index 0000000000000..4d77b3abf096b
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_spanner.py
@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that creates, updates, queries and deletes a Cloud Spanner instance.
+
+This DAG relies on the following environment variables
+* GCP_PROJECT_ID - Google Cloud Platform project for the Cloud Spanner instance.
+* GCP_SPANNER_INSTANCE_ID - Cloud Spanner instance ID.
+* GCP_SPANNER_DATABASE_ID - Cloud Spanner database ID.
+* GCP_SPANNER_CONFIG_NAME - The name of the instance's configuration. Values are of the
+  form ``projects/<gcp_project>/instanceConfigs/<configuration>``. See also:
+  https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instanceConfigs#InstanceConfig
+  https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instanceConfigs/list#google.spanner.admin.instance.v1.InstanceAdmin.ListInstanceConfigs
+* GCP_SPANNER_NODE_COUNT - Number of nodes allocated to the instance.
+* GCP_SPANNER_DISPLAY_NAME - The descriptive name for this instance as it appears in UIs.
+  Must be unique per project and between 4 and 30 characters in length.
+"""
+
+import os
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_spanner_operator import \
+    CloudSpannerInstanceDeployOperator, \
+    CloudSpannerInstanceDatabaseQueryOperator, \
+    CloudSpannerInstanceDeleteOperator, \
+    CloudSpannerInstanceDatabaseDeployOperator, \
+    CloudSpannerInstanceDatabaseUpdateOperator, \
+    CloudSpannerInstanceDatabaseDeleteOperator
+
+# [START howto_operator_spanner_arguments]
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_SPANNER_INSTANCE_ID = os.environ.get('GCP_SPANNER_INSTANCE_ID', 'testinstance')
+GCP_SPANNER_DATABASE_ID = os.environ.get('GCP_SPANNER_DATABASE_ID', 'testdatabase')
+GCP_SPANNER_CONFIG_NAME = os.environ.get('GCP_SPANNER_CONFIG_NAME',
+                                         'projects/example-project/instanceConfigs/eur3')
+GCP_SPANNER_NODE_COUNT = os.environ.get('GCP_SPANNER_NODE_COUNT', '1')
+GCP_SPANNER_DISPLAY_NAME = os.environ.get('GCP_SPANNER_DISPLAY_NAME', 'Test Instance')
+# OPERATION_ID should be unique per operation
+OPERATION_ID = 'unique_operation_id'
+# [END howto_operator_spanner_arguments]
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+
+with models.DAG(
+    'example_gcp_spanner',
+    default_args=default_args,
+    schedule_interval=None  # Override to match your needs
+) as dag:
+    # Create
+    # [START howto_operator_spanner_deploy]
+    spanner_instance_create_task = CloudSpannerInstanceDeployOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        configuration_name=GCP_SPANNER_CONFIG_NAME,
+        node_count=int(GCP_SPANNER_NODE_COUNT),
+        display_name=GCP_SPANNER_DISPLAY_NAME,
+        task_id='spanner_instance_create_task'
+    )
+    spanner_instance_update_task = CloudSpannerInstanceDeployOperator(
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        configuration_name=GCP_SPANNER_CONFIG_NAME,
+        node_count=int(GCP_SPANNER_NODE_COUNT) + 1,
+        display_name=GCP_SPANNER_DISPLAY_NAME + '_updated',
+        task_id='spanner_instance_update_task'
+    )
+    # [END howto_operator_spanner_deploy]
+
+    # [START howto_operator_spanner_database_deploy]
+    spanner_database_deploy_task = CloudSpannerInstanceDatabaseDeployOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        ddl_statements=[
+            "CREATE TABLE my_table1 (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+            "CREATE TABLE my_table2 (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+        ],
+        task_id='spanner_database_deploy_task'
+    )
+    spanner_database_deploy_task2 = CloudSpannerInstanceDatabaseDeployOperator(
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        ddl_statements=[
+            "CREATE TABLE my_table1 (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+            "CREATE TABLE my_table2 (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+        ],
+        task_id='spanner_database_deploy_task2'
+    )
+    # [END howto_operator_spanner_database_deploy]
+
+    # [START howto_operator_spanner_database_update]
+    spanner_database_update_task = CloudSpannerInstanceDatabaseUpdateOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        ddl_statements=[
+            "CREATE TABLE my_table3 (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+        ],
+        task_id='spanner_database_update_task'
+    )
+    # [END howto_operator_spanner_database_update]
+
+    # [START howto_operator_spanner_database_update_idempotent]
+    spanner_database_update_idempotent1_task = CloudSpannerInstanceDatabaseUpdateOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        operation_id=OPERATION_ID,
+        ddl_statements=[
+            "CREATE TABLE my_table_unique (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+        ],
+        task_id='spanner_database_update_idempotent1_task'
+    )
+    spanner_database_update_idempotent2_task = CloudSpannerInstanceDatabaseUpdateOperator(
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        operation_id=OPERATION_ID,
+        ddl_statements=[
+            "CREATE TABLE my_table_unique (id INT64, name STRING(MAX)) PRIMARY KEY (id)",
+        ],
+        task_id='spanner_database_update_idempotent2_task'
+    )
+    # [END howto_operator_spanner_database_update_idempotent]
+
+    # [START howto_operator_spanner_query]
+    spanner_instance_query_task = CloudSpannerInstanceDatabaseQueryOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        query=["DELETE FROM my_table2 WHERE true"],
+        task_id='spanner_instance_query_task'
+    )
+    spanner_instance_query_task2 = CloudSpannerInstanceDatabaseQueryOperator(
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        query=["DELETE FROM my_table2 WHERE true"],
+        task_id='spanner_instance_query_task2'
+    )
+    # [END howto_operator_spanner_query]
+
+    # [START howto_operator_spanner_database_delete]
+    spanner_database_delete_task = CloudSpannerInstanceDatabaseDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        task_id='spanner_database_delete_task'
+    )
+    spanner_database_delete_task2 = CloudSpannerInstanceDatabaseDeleteOperator(
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        database_id=GCP_SPANNER_DATABASE_ID,
+        task_id='spanner_database_delete_task2'
+    )
+    # [END howto_operator_spanner_database_delete]
+
+    # [START howto_operator_spanner_delete]
+    spanner_instance_delete_task = CloudSpannerInstanceDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        task_id='spanner_instance_delete_task'
+    )
+    spanner_instance_delete_task2 = CloudSpannerInstanceDeleteOperator(
+        instance_id=GCP_SPANNER_INSTANCE_ID,
+        task_id='spanner_instance_delete_task2'
+    )
+    # [END howto_operator_spanner_delete]
+
+    spanner_instance_create_task \
+        >> spanner_instance_update_task \
+        >> spanner_database_deploy_task \
+        >> spanner_database_deploy_task2 \
+        >> spanner_database_update_task \
+        >> spanner_database_update_idempotent1_task \
+        >> spanner_database_update_idempotent2_task \
+        >> spanner_instance_query_task \
+        >> spanner_instance_query_task2 \
+        >> spanner_database_delete_task \
+        >> spanner_database_delete_task2 \
+        >> spanner_instance_delete_task \
+        >> spanner_instance_delete_task2
diff --git a/airflow/contrib/example_dags/example_gcp_spanner.sql b/airflow/contrib/example_dags/example_gcp_spanner.sql
new file mode 100644
index 0000000000000..5d5f238022a52
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_spanner.sql
@@ -0,0 +1,3 @@
+INSERT my_table2 (id, name) VALUES (7, 'Seven');
+INSERT my_table2 (id, name)
+    VALUES (8, 'Eight');
diff --git a/airflow/contrib/example_dags/example_gcp_sql.py b/airflow/contrib/example_dags/example_gcp_sql.py
new file mode 100644
index 0000000000000..6c357f4b07e2f
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_sql.py
@@ -0,0 +1,423 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that creates, patches and deletes a Cloud SQL instance, and also
+creates, patches and deletes a database inside the instance, in Google Cloud Platform.
+
+This DAG relies on the following OS environment variables
+https://airflow.apache.org/concepts.html#variables
+* GCP_PROJECT_ID - Google Cloud Platform project for the Cloud SQL instance.
+* INSTANCE_NAME - Name of the Cloud SQL instance.
+* DB_NAME - Name of the database inside a Cloud SQL instance.
+"""
+
+import os
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_sql_operator import CloudSqlInstanceCreateOperator, \
+    CloudSqlInstancePatchOperator, CloudSqlInstanceDeleteOperator, \
+    CloudSqlInstanceDatabaseCreateOperator, CloudSqlInstanceDatabasePatchOperator, \
+    CloudSqlInstanceDatabaseDeleteOperator, CloudSqlInstanceExportOperator, \
+    CloudSqlInstanceImportOperator
+from airflow.contrib.operators.gcs_acl_operator import \
+    GoogleCloudStorageBucketCreateAclEntryOperator, \
+    GoogleCloudStorageObjectCreateAclEntryOperator
+
+from six.moves.urllib.parse import urlsplit
+
+# [START howto_operator_cloudsql_arguments]
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+INSTANCE_NAME = os.environ.get('GCSQL_MYSQL_INSTANCE_NAME', 'test-mysql')
+INSTANCE_NAME2 = os.environ.get('GCSQL_MYSQL_INSTANCE_NAME2', 'test-mysql2')
+DB_NAME = os.environ.get('GCSQL_MYSQL_DATABASE_NAME', 'testdb')
+# [END howto_operator_cloudsql_arguments]
+
+# [START howto_operator_cloudsql_export_import_arguments]
+EXPORT_URI = os.environ.get('GCSQL_MYSQL_EXPORT_URI', 'gs://bucketName/fileName')
+IMPORT_URI = os.environ.get('GCSQL_MYSQL_IMPORT_URI', 'gs://bucketName/fileName')
+# [END howto_operator_cloudsql_export_import_arguments]
+
+# Bodies below represent Cloud SQL instance resources:
+# https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances
+
+# [START howto_operator_cloudsql_create_arguments]
+FAILOVER_REPLICA_NAME = INSTANCE_NAME + "-failover-replica"
+READ_REPLICA_NAME = INSTANCE_NAME + "-read-replica"
+# [END howto_operator_cloudsql_create_arguments]
+
+# [START howto_operator_cloudsql_create_body]
+body = {
+    "name": INSTANCE_NAME,
+    "settings": {
+        "tier": "db-n1-standard-1",
+        "backupConfiguration": {
+            "binaryLogEnabled": True,
+            "enabled": True,
+            "startTime": "05:00"
+        },
+        "activationPolicy": "ALWAYS",
+        "dataDiskSizeGb": 30,
+        "dataDiskType": "PD_SSD",
+        "databaseFlags": [],
+        "ipConfiguration": {
+            "ipv4Enabled": True,
+            "requireSsl": True,
+        },
+        "locationPreference": {
+            "zone": "europe-west4-a"
+        },
+        "maintenanceWindow": {
+            "hour": 5,
+            "day": 7,
+            "updateTrack": "canary"
+        },
+        "pricingPlan": "PER_USE",
+        "replicationType": "ASYNCHRONOUS",
+        "storageAutoResize": True,
+        "storageAutoResizeLimit": 0,
+        "userLabels": {
+            "my-key": "my-value"
+        }
+    },
+    "failoverReplica": {
+        "name": FAILOVER_REPLICA_NAME
+    },
+    "databaseVersion": "MYSQL_5_7",
+    "region": "europe-west4",
+}
+# [END howto_operator_cloudsql_create_body]
+
+body2 = {
+    "name": INSTANCE_NAME2,
+    "settings": {
+        "tier": "db-n1-standard-1",
+    },
+    "databaseVersion": "MYSQL_5_7",
+    "region": "europe-west4",
+}
+
+# [START howto_operator_cloudsql_create_replica]
+read_replica_body = {
+    "name": READ_REPLICA_NAME,
+    "settings": {
+        "tier": "db-n1-standard-1",
+    },
+    "databaseVersion": "MYSQL_5_7",
+    "region": "europe-west4",
+    "masterInstanceName": INSTANCE_NAME,
+}
+# [END howto_operator_cloudsql_create_replica]
+
+
+# [START howto_operator_cloudsql_patch_body]
+patch_body = {
+    "name": INSTANCE_NAME,
+    "settings": {
+        "dataDiskSizeGb": 35,
+        "maintenanceWindow": {
+            "hour": 3,
+            "day": 6,
+            "updateTrack": "canary"
+        },
+        "userLabels": {
+            "my-key-patch": "my-value-patch"
+        }
+    }
+}
+# [END howto_operator_cloudsql_patch_body]
+# [START howto_operator_cloudsql_export_body]
+export_body = {
+    "exportContext": {
+        "fileType": "sql",
+        "uri": EXPORT_URI,
+        "sqlExportOptions": {
+            "schemaOnly": False
+        }
+    }
+}
+# [END howto_operator_cloudsql_export_body]
+# [START howto_operator_cloudsql_import_body]
+import_body = {
+    "importContext": {
+        "fileType": "sql",
+        "uri": IMPORT_URI
+    }
+}
+# [END howto_operator_cloudsql_import_body]
+# [START howto_operator_cloudsql_db_create_body]
+db_create_body = {
+    "instance": INSTANCE_NAME,
+    "name": DB_NAME,
+    "project": GCP_PROJECT_ID
+}
+# [END howto_operator_cloudsql_db_create_body]
+# [START howto_operator_cloudsql_db_patch_body]
+db_patch_body = {
+    "charset": "utf16",
+    "collation": "utf16_general_ci"
+}
+# [END howto_operator_cloudsql_db_patch_body]
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+
+with models.DAG(
+    'example_gcp_sql',
+    default_args=default_args,
+    schedule_interval=None  # Override to match your needs
+) as dag:
+
+    def next_dep(task, prev):
+        prev >> task
+        return task
+
+    # ############################################## #
+    # ### INSTANCES SET UP ######################### #
+    # ############################################## #
+
+    # [START howto_operator_cloudsql_create]
+    sql_instance_create_task = CloudSqlInstanceCreateOperator(
+        project_id=GCP_PROJECT_ID,
+        body=body,
+        instance=INSTANCE_NAME,
+        task_id='sql_instance_create_task'
+    )
+    # [END howto_operator_cloudsql_create]
+    prev_task = sql_instance_create_task
+
+    sql_instance_create_2_task = CloudSqlInstanceCreateOperator(
+        project_id=GCP_PROJECT_ID,
+        body=body2,
+        instance=INSTANCE_NAME2,
+        task_id='sql_instance_create_task2'
+    )
+    # [END howto_operator_cloudsql_create]
+
+    prev_task = sql_instance_create_task
+    prev_task = next_dep(sql_instance_create_2_task, prev_task)
+
+    sql_instance_read_replica_create = CloudSqlInstanceCreateOperator(
+        project_id=GCP_PROJECT_ID,
+        body=read_replica_body,
+        instance=INSTANCE_NAME2,
+        task_id='sql_instance_read_replica_create'
+    )
+    prev_task = next_dep(sql_instance_read_replica_create, prev_task)
+
+    # ############################################## #
+    # ### MODIFYING INSTANCE AND ITS DATABASE ###### #
+    # ############################################## #
+
+    # [START howto_operator_cloudsql_patch]
+    sql_instance_patch_task = CloudSqlInstancePatchOperator(
+        project_id=GCP_PROJECT_ID,
+        body=patch_body,
+        instance=INSTANCE_NAME,
+        task_id='sql_instance_patch_task'
+    )
+
+    sql_instance_patch_task2 = CloudSqlInstancePatchOperator(
+        body=patch_body,
+        instance=INSTANCE_NAME,
+        task_id='sql_instance_patch_task2'
+    )
+    # [END howto_operator_cloudsql_patch]
+    prev_task = next_dep(sql_instance_patch_task, prev_task)
+    prev_task = next_dep(sql_instance_patch_task2, prev_task)
+
+    # [START howto_operator_cloudsql_db_create]
+    sql_db_create_task = CloudSqlInstanceDatabaseCreateOperator(
+        project_id=GCP_PROJECT_ID,
+        body=db_create_body,
+        instance=INSTANCE_NAME,
+        task_id='sql_db_create_task'
+    )
+    sql_db_create_task2 = CloudSqlInstanceDatabaseCreateOperator(
+        body=db_create_body,
+        instance=INSTANCE_NAME,
+        task_id='sql_db_create_task2'
+    )
+    # [END howto_operator_cloudsql_db_create]
+    prev_task = next_dep(sql_db_create_task, prev_task)
+    prev_task = next_dep(sql_db_create_task2, prev_task)
+
+    # [START howto_operator_cloudsql_db_patch]
+    sql_db_patch_task = CloudSqlInstanceDatabasePatchOperator(
+        project_id=GCP_PROJECT_ID,
+        body=db_patch_body,
+        instance=INSTANCE_NAME,
+        database=DB_NAME,
+        task_id='sql_db_patch_task'
+    )
+    sql_db_patch_task2 = CloudSqlInstanceDatabasePatchOperator(
+        body=db_patch_body,
+        instance=INSTANCE_NAME,
+        database=DB_NAME,
+        task_id='sql_db_patch_task2'
+    )
+    # [END howto_operator_cloudsql_db_patch]
+    prev_task = next_dep(sql_db_patch_task, prev_task)
+    prev_task = next_dep(sql_db_patch_task2, prev_task)
+
+    # ############################################## #
+    # ### EXPORTING SQL FROM INSTANCE 1 ############ #
+    # ############################################## #
+    export_url_split = urlsplit(EXPORT_URI)
+
+    # For export to work we need to add the Cloud SQL instance's Service Account
+    # write access to the destination GCS bucket.
+    # [START howto_operator_cloudsql_export_gcs_permissions]
+    sql_gcp_add_bucket_permission_task = GoogleCloudStorageBucketCreateAclEntryOperator(
+        entity="user-{{ task_instance.xcom_pull("
+               "'sql_instance_create_task', key='service_account_email') "
+               "}}",
+        role="WRITER",
+        bucket=export_url_split[1],  # netloc (bucket)
+        task_id='sql_gcp_add_bucket_permission_task'
+    )
+    # [END howto_operator_cloudsql_export_gcs_permissions]
+    prev_task = next_dep(sql_gcp_add_bucket_permission_task, prev_task)
+
+    # [START howto_operator_cloudsql_export]
+    sql_export_task = CloudSqlInstanceExportOperator(
+        project_id=GCP_PROJECT_ID,
+        body=export_body,
+        instance=INSTANCE_NAME,
+        task_id='sql_export_task'
+    )
+    sql_export_task2 = CloudSqlInstanceExportOperator(
+        body=export_body,
+        instance=INSTANCE_NAME,
+        task_id='sql_export_task2'
+    )
+    # [END howto_operator_cloudsql_export]
+    prev_task = next_dep(sql_export_task, prev_task)
+    prev_task = next_dep(sql_export_task2, prev_task)
+
+    # ############################################## #
+    # ### IMPORTING SQL TO INSTANCE 2 ############## #
+    # ############################################## #
+    import_url_split = urlsplit(IMPORT_URI)
+
+    # For import to work we need to add the Cloud SQL instance's Service Account
+    # read access to the target GCS object.
+    # [START howto_operator_cloudsql_import_gcs_permissions]
+    sql_gcp_add_object_permission_task = GoogleCloudStorageObjectCreateAclEntryOperator(
+        entity="user-{{ task_instance.xcom_pull("
+               "'sql_instance_create_task2', key='service_account_email')"
+               " }}",
+        role="READER",
+        bucket=import_url_split[1],  # netloc (bucket)
+        object_name=import_url_split[2][1:],  # path (strip first '/')
+        task_id='sql_gcp_add_object_permission_task',
+    )
+    prev_task = next_dep(sql_gcp_add_object_permission_task, prev_task)
+
+    # For import to work we also need to add the Cloud SQL instance's Service Account
+    # write access to the whole bucket!.
+    sql_gcp_add_bucket_permission_2_task = GoogleCloudStorageBucketCreateAclEntryOperator(
+        entity="user-{{ task_instance.xcom_pull("
+               "'sql_instance_create_task2', key='service_account_email') "
+               "}}",
+        role="WRITER",
+        bucket=import_url_split[1],  # netloc
+        task_id='sql_gcp_add_bucket_permission_2_task',
+    )
+    # [END howto_operator_cloudsql_import_gcs_permissions]
+    prev_task = next_dep(sql_gcp_add_bucket_permission_2_task, prev_task)
+
+    # [START howto_operator_cloudsql_import]
+    sql_import_task = CloudSqlInstanceImportOperator(
+        project_id=GCP_PROJECT_ID,
+        body=import_body,
+        instance=INSTANCE_NAME2,
+        task_id='sql_import_task'
+    )
+    sql_import_task2 = CloudSqlInstanceImportOperator(
+        body=import_body,
+        instance=INSTANCE_NAME2,
+        task_id='sql_import_task2'
+    )
+    # [END howto_operator_cloudsql_import]
+    prev_task = next_dep(sql_import_task, prev_task)
+    prev_task = next_dep(sql_import_task2, prev_task)
+
+    # ############################################## #
+    # ### DELETING A DATABASE FROM AN INSTANCE ##### #
+    # ############################################## #
+
+    # [START howto_operator_cloudsql_db_delete]
+    sql_db_delete_task = CloudSqlInstanceDatabaseDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance=INSTANCE_NAME,
+        database=DB_NAME,
+        task_id='sql_db_delete_task'
+    )
+    sql_db_delete_task2 = CloudSqlInstanceDatabaseDeleteOperator(
+        instance=INSTANCE_NAME,
+        database=DB_NAME,
+        task_id='sql_db_delete_task2'
+    )
+    # [END howto_operator_cloudsql_db_delete]
+    prev_task = next_dep(sql_db_delete_task, prev_task)
+    prev_task = next_dep(sql_db_delete_task2, prev_task)
+
+    # ############################################## #
+    # ### INSTANCES TEAR DOWN ###################### #
+    # ############################################## #
+
+    # [START howto_operator_cloudsql_replicas_delete]
+    sql_instance_failover_replica_delete_task = CloudSqlInstanceDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance=FAILOVER_REPLICA_NAME,
+        task_id='sql_instance_failover_replica_delete_task'
+    )
+
+    sql_instance_read_replica_delete_task = CloudSqlInstanceDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance=READ_REPLICA_NAME,
+        task_id='sql_instance_read_replica_delete_task'
+    )
+    # [END howto_operator_cloudsql_replicas_delete]
+
+    prev_task = next_dep(sql_instance_failover_replica_delete_task, prev_task)
+    prev_task = next_dep(sql_instance_read_replica_delete_task, prev_task)
+
+    # [START howto_operator_cloudsql_delete]
+    sql_instance_delete_task = CloudSqlInstanceDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance=INSTANCE_NAME,
+        task_id='sql_instance_delete_task'
+    )
+    sql_instance_delete_task2 = CloudSqlInstanceDeleteOperator(
+        instance=INSTANCE_NAME2,
+        task_id='sql_instance_delete_task2'
+    )
+    # [END howto_operator_cloudsql_delete]
+    prev_task = next_dep(sql_instance_delete_task, prev_task)
+
+    sql_instance_delete_2_task = CloudSqlInstanceDeleteOperator(
+        project_id=GCP_PROJECT_ID,
+        instance=INSTANCE_NAME2,
+        task_id='sql_instance_delete_2_task'
+    )
+    prev_task = next_dep(sql_instance_delete_2_task, prev_task)
diff --git a/airflow/contrib/example_dags/example_gcp_sql_query.py b/airflow/contrib/example_dags/example_gcp_sql_query.py
new file mode 100644
index 0000000000000..f4fc5f392650c
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_sql_query.py
@@ -0,0 +1,294 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that performs query in a Cloud SQL instance.
+
+This DAG relies on the following OS environment variables
+
+* GCP_PROJECT_ID - Google Cloud Platform project for the Cloud SQL instance
+* GCP_REGION - Google Cloud region where the database is created
+*
+* GCSQL_POSTGRES_INSTANCE_NAME - Name of the postgres Cloud SQL instance
+* GCSQL_POSTGRES_USER - Name of the postgres database user
+* GCSQL_POSTGRES_PASSWORD - Password of the postgres database user
+* GCSQL_POSTGRES_PUBLIC_IP - Public IP of the Postgres database
+* GCSQL_POSTGRES_PUBLIC_PORT - Port of the postgres database
+*
+* GCSQL_MYSQL_INSTANCE_NAME - Name of the postgres Cloud SQL instance
+* GCSQL_MYSQL_USER - Name of the mysql database user
+* GCSQL_MYSQL_PASSWORD - Password of the mysql database user
+* GCSQL_MYSQL_PUBLIC_IP - Public IP of the mysql database
+* GCSQL_MYSQL_PUBLIC_PORT - Port of the mysql database
+"""
+
+import os
+import subprocess
+from os.path import expanduser
+
+from six.moves.urllib.parse import quote_plus
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_sql_operator import CloudSqlQueryOperator
+
+# [START howto_operator_cloudsql_query_arguments]
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_REGION = os.environ.get('GCP_REGION', 'europe-west-1b')
+
+GCSQL_POSTGRES_INSTANCE_NAME_QUERY = os.environ.get(
+    'GCSQL_POSTGRES_INSTANCE_NAME_QUERY',
+    'testpostgres')
+GCSQL_POSTGRES_DATABASE_NAME = os.environ.get('GCSQL_POSTGRES_DATABASE_NAME',
+                                              'postgresdb')
+GCSQL_POSTGRES_USER = os.environ.get('GCSQL_POSTGRES_USER', 'postgres_user')
+GCSQL_POSTGRES_PASSWORD = os.environ.get('GCSQL_POSTGRES_PASSWORD', 'password')
+GCSQL_POSTGRES_PUBLIC_IP = os.environ.get('GCSQL_POSTGRES_PUBLIC_IP', '0.0.0.0')
+GCSQL_POSTGRES_PUBLIC_PORT = os.environ.get('GCSQL_POSTGRES_PUBLIC_PORT', 5432)
+GCSQL_POSTGRES_CLIENT_CERT_FILE = os.environ.get('GCSQL_POSTGRES_CLIENT_CERT_FILE',
+                                                 ".key/postgres-client-cert.pem")
+GCSQL_POSTGRES_CLIENT_KEY_FILE = os.environ.get('GCSQL_POSTGRES_CLIENT_KEY_FILE',
+                                                ".key/postgres-client-key.pem")
+GCSQL_POSTGRES_SERVER_CA_FILE = os.environ.get('GCSQL_POSTGRES_SERVER_CA_FILE',
+                                               ".key/postgres-server-ca.pem")
+
+GCSQL_MYSQL_INSTANCE_NAME_QUERY = os.environ.get('GCSQL_MYSQL_INSTANCE_NAME_QUERY',
+                                                 'testmysql')
+GCSQL_MYSQL_DATABASE_NAME = os.environ.get('GCSQL_MYSQL_DATABASE_NAME', 'mysqldb')
+GCSQL_MYSQL_USER = os.environ.get('GCSQL_MYSQL_USER', 'mysql_user')
+GCSQL_MYSQL_PASSWORD = os.environ.get('GCSQL_MYSQL_PASSWORD', 'password')
+GCSQL_MYSQL_PUBLIC_IP = os.environ.get('GCSQL_MYSQL_PUBLIC_IP', '0.0.0.0')
+GCSQL_MYSQL_PUBLIC_PORT = os.environ.get('GCSQL_MYSQL_PUBLIC_PORT', 3306)
+GCSQL_MYSQL_CLIENT_CERT_FILE = os.environ.get('GCSQL_MYSQL_CLIENT_CERT_FILE',
+                                              ".key/mysql-client-cert.pem")
+GCSQL_MYSQL_CLIENT_KEY_FILE = os.environ.get('GCSQL_MYSQL_CLIENT_KEY_FILE',
+                                             ".key/mysql-client-key.pem")
+GCSQL_MYSQL_SERVER_CA_FILE = os.environ.get('GCSQL_MYSQL_SERVER_CA_FILE',
+                                            ".key/mysql-server-ca.pem")
+
+SQL = [
+    'CREATE TABLE IF NOT EXISTS TABLE_TEST (I INTEGER)',
+    'CREATE TABLE IF NOT EXISTS TABLE_TEST (I INTEGER)',  # shows warnings logged
+    'INSERT INTO TABLE_TEST VALUES (0)',
+    'CREATE TABLE IF NOT EXISTS TABLE_TEST2 (I INTEGER)',
+    'DROP TABLE TABLE_TEST',
+    'DROP TABLE TABLE_TEST2',
+]
+
+# [END howto_operator_cloudsql_query_arguments]
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+
+
+# [START howto_operator_cloudsql_query_connections]
+
+HOME_DIR = expanduser("~")
+
+
+def get_absolute_path(path):
+    if path.startswith("/"):
+        return path
+    else:
+        return os.path.join(HOME_DIR, path)
+
+
+postgres_kwargs = dict(
+    user=quote_plus(GCSQL_POSTGRES_USER),
+    password=quote_plus(GCSQL_POSTGRES_PASSWORD),
+    public_port=GCSQL_POSTGRES_PUBLIC_PORT,
+    public_ip=quote_plus(GCSQL_POSTGRES_PUBLIC_IP),
+    project_id=quote_plus(GCP_PROJECT_ID),
+    location=quote_plus(GCP_REGION),
+    instance=quote_plus(GCSQL_POSTGRES_INSTANCE_NAME_QUERY),
+    database=quote_plus(GCSQL_POSTGRES_DATABASE_NAME),
+    client_cert_file=quote_plus(get_absolute_path(GCSQL_POSTGRES_CLIENT_CERT_FILE)),
+    client_key_file=quote_plus(get_absolute_path(GCSQL_POSTGRES_CLIENT_KEY_FILE)),
+    server_ca_file=quote_plus(get_absolute_path(GCSQL_POSTGRES_SERVER_CA_FILE))
+)
+
+# The connections below are created using one of the standard approaches - via environment
+# variables named AIRFLOW_CONN_* . The connections can also be created in the database
+# of AIRFLOW (using command line or UI).
+
+# Postgres: connect via proxy over TCP
+os.environ['AIRFLOW_CONN_PROXY_POSTGRES_TCP'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=postgres&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=True&" \
+    "sql_proxy_use_tcp=True".format(**postgres_kwargs)
+
+# Postgres: connect via proxy over UNIX socket (specific proxy version)
+os.environ['AIRFLOW_CONN_PROXY_POSTGRES_SOCKET'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=postgres&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=True&" \
+    "sql_proxy_version=v1.13&" \
+    "sql_proxy_use_tcp=False".format(**postgres_kwargs)
+
+# Postgres: connect directly via TCP (non-SSL)
+os.environ['AIRFLOW_CONN_PUBLIC_POSTGRES_TCP'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=postgres&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=False&" \
+    "use_ssl=False".format(**postgres_kwargs)
+
+# Postgres: connect directly via TCP (SSL)
+os.environ['AIRFLOW_CONN_PUBLIC_POSTGRES_TCP_SSL'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=postgres&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=False&" \
+    "use_ssl=True&" \
+    "sslcert={client_cert_file}&" \
+    "sslkey={client_key_file}&" \
+    "sslrootcert={server_ca_file}"\
+    .format(**postgres_kwargs)
+
+mysql_kwargs = dict(
+    user=quote_plus(GCSQL_MYSQL_USER),
+    password=quote_plus(GCSQL_MYSQL_PASSWORD),
+    public_port=GCSQL_MYSQL_PUBLIC_PORT,
+    public_ip=quote_plus(GCSQL_MYSQL_PUBLIC_IP),
+    project_id=quote_plus(GCP_PROJECT_ID),
+    location=quote_plus(GCP_REGION),
+    instance=quote_plus(GCSQL_MYSQL_INSTANCE_NAME_QUERY),
+    database=quote_plus(GCSQL_MYSQL_DATABASE_NAME),
+    client_cert_file=quote_plus(get_absolute_path(GCSQL_MYSQL_CLIENT_CERT_FILE)),
+    client_key_file=quote_plus(get_absolute_path(GCSQL_MYSQL_CLIENT_KEY_FILE)),
+    server_ca_file=quote_plus(get_absolute_path(GCSQL_MYSQL_SERVER_CA_FILE))
+)
+
+# MySQL: connect via proxy over TCP (specific proxy version)
+os.environ['AIRFLOW_CONN_PROXY_MYSQL_TCP'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=mysql&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=True&" \
+    "sql_proxy_version=v1.13&" \
+    "sql_proxy_use_tcp=True".format(**mysql_kwargs)
+
+# MySQL: connect via proxy over UNIX socket using pre-downloaded Cloud Sql Proxy binary
+try:
+    sql_proxy_binary_path = subprocess.check_output(
+        ['which', 'cloud_sql_proxy']).decode('utf-8').rstrip()
+except subprocess.CalledProcessError:
+    sql_proxy_binary_path = "/tmp/anyhow_download_cloud_sql_proxy"
+
+os.environ['AIRFLOW_CONN_PROXY_MYSQL_SOCKET'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=mysql&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=True&" \
+    "sql_proxy_binary_path={sql_proxy_binary_path}&" \
+    "sql_proxy_use_tcp=False".format(
+        sql_proxy_binary_path=quote_plus(sql_proxy_binary_path), **mysql_kwargs)
+
+# MySQL: connect directly via TCP (non-SSL)
+os.environ['AIRFLOW_CONN_PUBLIC_MYSQL_TCP'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=mysql&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=False&" \
+    "use_ssl=False".format(**mysql_kwargs)
+
+# MySQL: connect directly via TCP (SSL) and with fixed Cloud Sql Proxy binary path
+os.environ['AIRFLOW_CONN_PUBLIC_MYSQL_TCP_SSL'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=mysql&" \
+    "project_id={project_id}&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=False&" \
+    "use_ssl=True&" \
+    "sslcert={client_cert_file}&" \
+    "sslkey={client_key_file}&" \
+    "sslrootcert={server_ca_file}".format(**mysql_kwargs)
+
+# Special case: MySQL: connect directly via TCP (SSL) and with fixed Cloud Sql
+# Proxy binary path AND with missing project_id
+
+os.environ['AIRFLOW_CONN_PUBLIC_MYSQL_TCP_SSL_NO_PROJECT_ID'] = \
+    "gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?" \
+    "database_type=mysql&" \
+    "location={location}&" \
+    "instance={instance}&" \
+    "use_proxy=False&" \
+    "use_ssl=True&" \
+    "sslcert={client_cert_file}&" \
+    "sslkey={client_key_file}&" \
+    "sslrootcert={server_ca_file}".format(**mysql_kwargs)
+
+
+# [END howto_operator_cloudsql_query_connections]
+
+# [START howto_operator_cloudsql_query_operators]
+
+connection_names = [
+    "proxy_postgres_tcp",
+    "proxy_postgres_socket",
+    "public_postgres_tcp",
+    "public_postgres_tcp_ssl",
+    "proxy_mysql_tcp",
+    "proxy_mysql_socket",
+    "public_mysql_tcp",
+    "public_mysql_tcp_ssl",
+    "public_mysql_tcp_ssl_no_project_id"
+]
+
+tasks = []
+
+
+with models.DAG(
+    dag_id='example_gcp_sql_query',
+    default_args=default_args,
+    schedule_interval=None
+) as dag:
+    prev_task = None
+
+    for connection_name in connection_names:
+        task = CloudSqlQueryOperator(
+            gcp_cloudsql_conn_id=connection_name,
+            task_id="example_gcp_sql_task_" + connection_name,
+            sql=SQL
+        )
+        tasks.append(task)
+        if prev_task:
+            prev_task >> task
+        prev_task = task
+
+# [END howto_operator_cloudsql_query_operators]
diff --git a/airflow/contrib/example_dags/example_gcp_transfer.py b/airflow/contrib/example_dags/example_gcp_transfer.py
new file mode 100644
index 0000000000000..a21bdbcc18a4d
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_transfer.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that demonstrates interactions with Google Cloud Transfer.
+
+
+This DAG relies on the following OS environment variables
+
+* GCP_PROJECT_ID - Google Cloud Project to use for the Google Cloud Transfer Service.
+* GCP_DESCRIPTION - Description of transfer job
+* GCP_TRANSFER_SOURCE_AWS_BUCKET - Amazon Web Services Storage bucket from which files are copied.
+  .. warning::
+    You need to provide a large enough set of data so that operations do not execute too quickly.
+    Otherwise, DAG will fail.
+* GCP_TRANSFER_FIRST_TARGET_BUCKET - Google Cloud Storage bucket to which files are copied from AWS.
+  It is also a source bucket in next step
+* GCP_TRANSFER_SECOND_TARGET_BUCKET - Google Cloud Storage bucket bucket to which files are copied
+* WAIT_FOR_OPERATION_POKE_INTERVAL - interval of what to check the status of the operation
+  A smaller value than the default value accelerates the system test and ensures its correct execution with
+  smaller quantities of files in the source bucket
+  Look at documentation of :class:`~airflow.operators.sensors.BaseSensorOperator` for more information
+
+"""
+import os
+from datetime import datetime, timedelta
+from typing import Any, Dict
+
+from airflow import models
+from airflow.contrib.hooks.gcp_transfer_hook import (
+    GcpTransferOperationStatus,
+    GcpTransferJobsStatus,
+    TRANSFER_OPTIONS,
+    PROJECT_ID,
+    BUCKET_NAME,
+    GCS_DATA_SINK,
+    STATUS,
+    DESCRIPTION,
+    GCS_DATA_SOURCE,
+    START_TIME_OF_DAY,
+    SCHEDULE_END_DATE,
+    SCHEDULE_START_DATE,
+    SCHEDULE,
+    AWS_S3_DATA_SOURCE,
+    TRANSFER_SPEC,
+    FILTER_PROJECT_ID,
+    FILTER_JOB_NAMES,
+    TRANSFER_JOB,
+    TRANSFER_JOB_FIELD_MASK,
+    ALREADY_EXISTING_IN_SINK,
+)
+from airflow.contrib.operators.gcp_transfer_operator import (
+    GcpTransferServiceJobCreateOperator,
+    GcpTransferServiceJobDeleteOperator,
+    GcpTransferServiceJobUpdateOperator,
+    GcpTransferServiceOperationsListOperator,
+    GcpTransferServiceOperationGetOperator,
+    GcpTransferServiceOperationPauseOperator,
+    GcpTransferServiceOperationResumeOperator,
+    GcpTransferServiceOperationCancelOperator,
+)
+from airflow.contrib.sensors.gcp_transfer_sensor import GCPTransferServiceWaitForJobStatusSensor
+from airflow.utils.dates import days_ago
+
+# [START howto_operator_gcp_transfer_common_variables]
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_DESCRIPTION = os.environ.get('GCP_DESCRIPTION', 'description')
+GCP_TRANSFER_TARGET_BUCKET = os.environ.get('GCP_TRANSFER_TARGET_BUCKET')
+WAIT_FOR_OPERATION_POKE_INTERVAL = int(os.environ.get('WAIT_FOR_OPERATION_POKE_INTERVAL', 5))
+
+GCP_TRANSFER_SOURCE_AWS_BUCKET = os.environ.get('GCP_TRANSFER_SOURCE_AWS_BUCKET')
+GCP_TRANSFER_FIRST_TARGET_BUCKET = os.environ.get(
+    'GCP_TRANSFER_FIRST_TARGET_BUCKET', 'gcp-transfer-first-target'
+)
+GCP_TRANSFER_SECOND_TARGET_BUCKET = os.environ.get(
+    'GCP_TRANSFER_SECOND_TARGET_BUCKET', 'gcp-transfer-second-target'
+)
+# [END howto_operator_gcp_transfer_common_variables]
+
+# [START howto_operator_gcp_transfer_create_job_body_aws]
+aws_to_gcs_transfer_body = {
+    DESCRIPTION: GCP_DESCRIPTION,
+    STATUS: GcpTransferJobsStatus.ENABLED,
+    PROJECT_ID: GCP_PROJECT_ID,
+    SCHEDULE: {
+        SCHEDULE_START_DATE: datetime(2015, 1, 1).date(),
+        SCHEDULE_END_DATE: datetime(2030, 1, 1).date(),
+        START_TIME_OF_DAY: (datetime.utcnow() + timedelta(minutes=2)).time(),
+    },
+    TRANSFER_SPEC: {
+        AWS_S3_DATA_SOURCE: {BUCKET_NAME: GCP_TRANSFER_SOURCE_AWS_BUCKET},
+        GCS_DATA_SINK: {BUCKET_NAME: GCP_TRANSFER_FIRST_TARGET_BUCKET},
+        TRANSFER_OPTIONS: {ALREADY_EXISTING_IN_SINK: True},
+    },
+}
+# [END howto_operator_gcp_transfer_create_job_body_aws]
+
+# [START howto_operator_gcp_transfer_create_job_body_gcp]
+gcs_to_gcs_transfer_body = {
+    DESCRIPTION: GCP_DESCRIPTION,
+    STATUS: GcpTransferJobsStatus.ENABLED,
+    PROJECT_ID: GCP_PROJECT_ID,
+    SCHEDULE: {
+        SCHEDULE_START_DATE: datetime(2015, 1, 1).date(),
+        SCHEDULE_END_DATE: datetime(2030, 1, 1).date(),
+        START_TIME_OF_DAY: (datetime.utcnow() + timedelta(minutes=2)).time(),
+    },
+    TRANSFER_SPEC: {
+        GCS_DATA_SOURCE: {BUCKET_NAME: GCP_TRANSFER_FIRST_TARGET_BUCKET},
+        GCS_DATA_SINK: {BUCKET_NAME: GCP_TRANSFER_SECOND_TARGET_BUCKET},
+        TRANSFER_OPTIONS: {ALREADY_EXISTING_IN_SINK: True},
+    },
+}  # type: Dict[str, Any]
+# [END howto_operator_gcp_transfer_create_job_body_gcp]
+
+# [START howto_operator_gcp_transfer_update_job_body]
+update_body = {
+    PROJECT_ID: GCP_PROJECT_ID,
+    TRANSFER_JOB: {DESCRIPTION: "{}_updated".format(GCP_DESCRIPTION)},
+    TRANSFER_JOB_FIELD_MASK: "description",
+}
+# [END howto_operator_gcp_transfer_update_job_body]
+
+list_filter_dict = {FILTER_PROJECT_ID: GCP_PROJECT_ID, FILTER_JOB_NAMES: []}
+
+# [START howto_operator_gcp_transfer_default_args]
+default_args = {'start_date': days_ago(1)}
+# [END howto_operator_gcp_transfer_default_args]
+
+with models.DAG(
+    'example_gcp_transfer', default_args=default_args, schedule_interval=None  # Override to match your needs
+) as dag:
+
+    # [START howto_operator_gcp_transfer_create_job]
+    create_transfer_job_from_aws = GcpTransferServiceJobCreateOperator(
+        task_id="create_transfer_job_from_aws", body=aws_to_gcs_transfer_body
+    )
+    # [END howto_operator_gcp_transfer_create_job]
+
+    wait_for_operation_to_start = GCPTransferServiceWaitForJobStatusSensor(
+        task_id="wait_for_operation_to_start",
+        job_name="{{task_instance.xcom_pull('create_transfer_job_from_aws')['name']}}",
+        project_id=GCP_PROJECT_ID,
+        expected_statuses={GcpTransferOperationStatus.IN_PROGRESS},
+        poke_interval=WAIT_FOR_OPERATION_POKE_INTERVAL,
+    )
+
+    # [START howto_operator_gcp_transfer_pause_operation]
+    pause_operation = GcpTransferServiceOperationPauseOperator(
+        task_id="pause_operation",
+        operation_name="{{task_instance.xcom_pull('wait_for_operation_to_start', "
+        "key='sensed_operations')[0]['name']}}",
+    )
+    # [END howto_operator_gcp_transfer_pause_operation]
+
+    # [START howto_operator_gcp_transfer_update_job]
+    update_job = GcpTransferServiceJobUpdateOperator(
+        task_id="update_job",
+        job_name="{{task_instance.xcom_pull('create_transfer_job_from_aws')['name']}}",
+        body=update_body,
+    )
+    # [END howto_operator_gcp_transfer_update_job]
+
+    # [START howto_operator_gcp_transfer_list_operations]
+    list_operations = GcpTransferServiceOperationsListOperator(
+        task_id="list_operations",
+        filter={
+            FILTER_PROJECT_ID: GCP_PROJECT_ID,
+            FILTER_JOB_NAMES: ["{{task_instance.xcom_pull('create_transfer_job_from_aws')['name']}}"],
+        },
+    )
+    # [END howto_operator_gcp_transfer_list_operations]
+
+    # [START howto_operator_gcp_transfer_get_operation]
+    get_operation = GcpTransferServiceOperationGetOperator(
+        task_id="get_operation", operation_name="{{task_instance.xcom_pull('list_operations')[0]['name']}}"
+    )
+    # [END howto_operator_gcp_transfer_get_operation]
+
+    # [START howto_operator_gcp_transfer_resume_operation]
+    resume_operation = GcpTransferServiceOperationResumeOperator(
+        task_id="resume_operation", operation_name="{{task_instance.xcom_pull('get_operation')['name']}}"
+    )
+    # [END howto_operator_gcp_transfer_resume_operation]
+
+    # [START howto_operator_gcp_transfer_wait_operation]
+    wait_for_operation_to_end = GCPTransferServiceWaitForJobStatusSensor(
+        task_id="wait_for_operation_to_end",
+        job_name="{{task_instance.xcom_pull('create_transfer_job_from_aws')['name']}}",
+        project_id=GCP_PROJECT_ID,
+        expected_statuses={GcpTransferOperationStatus.SUCCESS},
+        poke_interval=WAIT_FOR_OPERATION_POKE_INTERVAL,
+    )
+    # [END howto_operator_gcp_transfer_wait_operation]
+
+    job_time = datetime.utcnow() + timedelta(minutes=2)
+
+    gcs_to_gcs_transfer_body['schedule']['startTimeOfDay'] = (datetime.utcnow() + timedelta(minutes=2)).time()
+
+    create_transfer_job_from_gcp = GcpTransferServiceJobCreateOperator(
+        task_id="create_transfer_job_from_gcp", body=gcs_to_gcs_transfer_body
+    )
+
+    wait_for_second_operation_to_start = GCPTransferServiceWaitForJobStatusSensor(
+        task_id="wait_for_second_operation_to_start",
+        job_name="{{ task_instance.xcom_pull('create_transfer_job_from_gcp')['name'] }}",
+        project_id=GCP_PROJECT_ID,
+        expected_statuses={GcpTransferOperationStatus.IN_PROGRESS},
+        poke_interval=WAIT_FOR_OPERATION_POKE_INTERVAL,
+    )
+
+    # [START howto_operator_gcp_transfer_cancel_operation]
+    cancel_operation = GcpTransferServiceOperationCancelOperator(
+        task_id="cancel_operation",
+        operation_name="{{task_instance.xcom_pull("
+        "'wait_for_second_operation_to_start', key='sensed_operations')[0]['name']}}",
+    )
+    # [END howto_operator_gcp_transfer_cancel_operation]
+
+    # [START howto_operator_gcp_transfer_delete_job]
+    delete_transfer_from_aws_job = GcpTransferServiceJobDeleteOperator(
+        task_id="delete_transfer_from_aws_job",
+        job_name="{{task_instance.xcom_pull('create_transfer_job_from_aws')['name']}}",
+        project_id=GCP_PROJECT_ID,
+    )
+    # [END howto_operator_gcp_transfer_delete_job]
+
+    delete_transfer_from_gcp_job = GcpTransferServiceJobDeleteOperator(
+        task_id="delete_transfer_from_gcp_job",
+        job_name="{{task_instance.xcom_pull('create_transfer_job_from_gcp')['name']}}",
+        project_id=GCP_PROJECT_ID,
+    )
+
+    create_transfer_job_from_aws >> wait_for_operation_to_start >> pause_operation >> \
+        list_operations >> get_operation >> resume_operation >> wait_for_operation_to_end >> \
+        create_transfer_job_from_gcp >> wait_for_second_operation_to_start >> cancel_operation >> \
+        delete_transfer_from_aws_job >> delete_transfer_from_gcp_job
diff --git a/airflow/contrib/example_dags/example_gcp_translate.py b/airflow/contrib/example_dags/example_gcp_translate.py
new file mode 100644
index 0000000000000..bc7030cf1a084
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_translate.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that translates text in Google Cloud Translate
+service in the Google Cloud Platform.
+
+"""
+import airflow
+from airflow import models
+
+from airflow.contrib.operators.gcp_translate_operator import CloudTranslateTextOperator
+from airflow.operators.bash_operator import BashOperator
+
+default_args = {'start_date': airflow.utils.dates.days_ago(1)}
+
+with models.DAG(
+    'example_gcp_translate', default_args=default_args, schedule_interval=None  # Override to match your needs
+) as dag:
+    # [START howto_operator_translate_text]
+    product_set_create = CloudTranslateTextOperator(
+        task_id='translate',
+        values=['zażółć gęślą jaźń'],
+        target_language='en',
+        format_='text',
+        source_language=None,
+        model='base',
+    )
+    # [END howto_operator_translate_text]
+    # [START howto_operator_translate_access]
+    translation_access = BashOperator(
+        task_id='access',
+        bash_command="echo '{{ task_instance.xcom_pull(\"translate\")[0] }}'"
+    )
+    product_set_create >> translation_access
+    # [END howto_operator_translate_access]
diff --git a/airflow/contrib/example_dags/example_gcp_vision.py b/airflow/contrib/example_dags/example_gcp_vision.py
new file mode 100644
index 0000000000000..2a1facffcef78
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcp_vision.py
@@ -0,0 +1,415 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that creates, gets, updates and deletes Products and Product Sets in the Google Cloud
+Vision service in the Google Cloud Platform.
+
+This DAG relies on the following OS environment variables
+
+* GCP_VISION_LOCATION - Zone where the instance exists.
+* GCP_VISION_PRODUCT_SET_ID - Product Set ID.
+* GCP_VISION_PRODUCT_ID - Product  ID.
+* GCP_VISION_REFERENCE_IMAGE_ID - Reference Image ID.
+* GCP_VISION_REFERENCE_IMAGE_URL - A link to the bucket that contains the reference image.
+* GCP_VISION_ANNOTATE_IMAGE_URL - A link to the bucket that contains the file to be annotated.
+
+"""
+import os
+
+# [START howto_operator_vision_retry_import]
+from google.api_core.retry import Retry
+
+# [END howto_operator_vision_retry_import]
+# [START howto_operator_vision_product_set_import]
+from google.cloud.vision_v1.types import ProductSet
+# [END howto_operator_vision_product_set_import]
+# [START howto_operator_vision_product_import]
+from google.cloud.vision_v1.types import Product
+# [END howto_operator_vision_product_import]
+# [START howto_operator_vision_reference_image_import]
+from google.cloud.vision_v1.types import ReferenceImage
+# [END howto_operator_vision_reference_image_import]
+# [START howto_operator_vision_enums_import]
+from google.cloud.vision import enums
+# [END howto_operator_vision_enums_import]
+
+import airflow
+from airflow import models
+from airflow.operators.bash_operator import BashOperator
+
+from airflow.contrib.operators.gcp_vision_operator import (
+    CloudVisionProductSetCreateOperator,
+    CloudVisionProductSetGetOperator,
+    CloudVisionProductSetUpdateOperator,
+    CloudVisionProductSetDeleteOperator,
+    CloudVisionProductCreateOperator,
+    CloudVisionProductGetOperator,
+    CloudVisionProductUpdateOperator,
+    CloudVisionProductDeleteOperator,
+    CloudVisionReferenceImageCreateOperator,
+    CloudVisionAddProductToProductSetOperator,
+    CloudVisionRemoveProductFromProductSetOperator,
+    CloudVisionAnnotateImageOperator,
+)
+
+default_args = {'start_date': airflow.utils.dates.days_ago(1)}
+
+# [START howto_operator_vision_args_common]
+GCP_VISION_LOCATION = os.environ.get('GCP_VISION_LOCATION', 'europe-west1')
+# [END howto_operator_vision_args_common]
+
+# [START howto_operator_vision_product_set_explicit_id]
+GCP_VISION_PRODUCT_SET_ID = os.environ.get('GCP_VISION_PRODUCT_SET_ID', 'product_set_explicit_id')
+# [END howto_operator_vision_product_set_explicit_id]
+
+# [START howto_operator_vision_product_explicit_id]
+GCP_VISION_PRODUCT_ID = os.environ.get('GCP_VISION_PRODUCT_ID', 'product_explicit_id')
+# [END howto_operator_vision_product_explicit_id]
+
+# [START howto_operator_vision_reference_image_args]
+GCP_VISION_REFERENCE_IMAGE_ID = os.environ.get('GCP_VISION_REFERENCE_IMAGE_ID', 'reference_image_explicit_id')
+GCP_VISION_REFERENCE_IMAGE_URL = os.environ.get('GCP_VISION_REFERENCE_IMAGE_URL', 'gs://bucket/image1.jpg')
+# [END howto_operator_vision_reference_image_args]
+
+# [START howto_operator_vision_annotate_image_url]
+GCP_VISION_ANNOTATE_IMAGE_URL = os.environ.get('GCP_VISION_ANNOTATE_IMAGE_URL', 'gs://bucket/image2.jpg')
+# [END howto_operator_vision_annotate_image_url]
+
+# [START howto_operator_vision_product_set]
+product_set = ProductSet(display_name='My Product Set')
+# [END howto_operator_vision_product_set]
+
+# [START howto_operator_vision_product]
+product = Product(display_name='My Product 1', product_category='toys')
+# [END howto_operator_vision_product]
+
+# [START howto_operator_vision_reference_image]
+reference_image = ReferenceImage(uri=GCP_VISION_REFERENCE_IMAGE_URL)
+# [END howto_operator_vision_reference_image]
+
+# [START howto_operator_vision_annotate_image_request]
+annotate_image_request = {
+    'image': {'source': {'image_uri': GCP_VISION_ANNOTATE_IMAGE_URL}},
+    'features': [{'type': enums.Feature.Type.LOGO_DETECTION}],
+}
+# [END howto_operator_vision_annotate_image_request]
+
+with models.DAG(
+    'example_gcp_vision_autogenerated_id', default_args=default_args, schedule_interval=None
+) as dag_autogenerated_id:
+    # ################################## #
+    # ### Autogenerated IDs examples ### #
+    # ################################## #
+
+    # [START howto_operator_vision_product_set_create]
+    product_set_create = CloudVisionProductSetCreateOperator(
+        location=GCP_VISION_LOCATION,
+        product_set=product_set,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='product_set_create',
+    )
+    # [END howto_operator_vision_product_set_create]
+
+    # [START howto_operator_vision_product_set_get]
+    product_set_get = CloudVisionProductSetGetOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id="{{ task_instance.xcom_pull('product_set_create') }}",
+        task_id='product_set_get',
+    )
+    # [END howto_operator_vision_product_set_get]
+
+    # [START howto_operator_vision_product_set_update]
+    product_set_update = CloudVisionProductSetUpdateOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id="{{ task_instance.xcom_pull('product_set_create') }}",
+        product_set=ProductSet(display_name='My Product Set 2'),
+        task_id='product_set_update',
+    )
+    # [END howto_operator_vision_product_set_update]
+
+    # [START howto_operator_vision_product_set_delete]
+    product_set_delete = CloudVisionProductSetDeleteOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id="{{ task_instance.xcom_pull('product_set_create') }}",
+        task_id='product_set_delete',
+    )
+    # [END howto_operator_vision_product_set_delete]
+
+    # [START howto_operator_vision_product_create]
+    product_create = CloudVisionProductCreateOperator(
+        location=GCP_VISION_LOCATION,
+        product=product,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='product_create',
+    )
+    # [END howto_operator_vision_product_create]
+
+    # [START howto_operator_vision_product_get]
+    product_get = CloudVisionProductGetOperator(
+        location=GCP_VISION_LOCATION,
+        product_id="{{ task_instance.xcom_pull('product_create') }}",
+        task_id='product_get',
+    )
+    # [END howto_operator_vision_product_get]
+
+    # [START howto_operator_vision_product_update]
+    product_update = CloudVisionProductUpdateOperator(
+        location=GCP_VISION_LOCATION,
+        product_id="{{ task_instance.xcom_pull('product_create') }}",
+        product=Product(display_name='My Product 2', description='My updated description'),
+        task_id='product_update',
+    )
+    # [END howto_operator_vision_product_update]
+
+    # [START howto_operator_vision_product_delete]
+    product_delete = CloudVisionProductDeleteOperator(
+        location=GCP_VISION_LOCATION,
+        product_id="{{ task_instance.xcom_pull('product_create') }}",
+        task_id='product_delete',
+    )
+    # [END howto_operator_vision_product_delete]
+
+    # [START howto_operator_vision_reference_image_create]
+    reference_image_create = CloudVisionReferenceImageCreateOperator(
+        location=GCP_VISION_LOCATION,
+        reference_image=reference_image,
+        product_id="{{ task_instance.xcom_pull('product_create') }}",
+        reference_image_id=GCP_VISION_REFERENCE_IMAGE_ID,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='reference_image_create',
+    )
+    # [END howto_operator_vision_reference_image_create]
+
+    # [START howto_operator_vision_add_product_to_product_set]
+    add_product_to_product_set = CloudVisionAddProductToProductSetOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id="{{ task_instance.xcom_pull('product_set_create') }}",
+        product_id="{{ task_instance.xcom_pull('product_create') }}",
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='add_product_to_product_set',
+    )
+    # [END howto_operator_vision_add_product_to_product_set]
+
+    # [START howto_operator_vision_remove_product_from_product_set]
+    remove_product_from_product_set = CloudVisionRemoveProductFromProductSetOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id="{{ task_instance.xcom_pull('product_set_create') }}",
+        product_id="{{ task_instance.xcom_pull('product_create') }}",
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='remove_product_from_product_set',
+    )
+    # [END howto_operator_vision_remove_product_from_product_set]
+
+    # Product path
+    product_create >> product_get >> product_update >> product_delete
+
+    # ProductSet path
+    product_set_create >> product_set_get >> product_set_update >> product_set_delete
+
+    # ReferenceImage path
+    product_create >> reference_image_create >> product_delete
+
+    # Product/ProductSet path
+    product_create >> add_product_to_product_set
+    product_set_create >> add_product_to_product_set
+    add_product_to_product_set >> remove_product_from_product_set
+    remove_product_from_product_set >> product_delete
+    remove_product_from_product_set >> product_set_delete
+
+with models.DAG(
+    'example_gcp_vision_explicit_id', default_args=default_args, schedule_interval=None
+) as dag_explicit_id:
+    # ############################# #
+    # ### Explicit IDs examples ### #
+    # ############################# #
+
+    # [START howto_operator_vision_product_set_create_2]
+    product_set_create_2 = CloudVisionProductSetCreateOperator(
+        product_set_id=GCP_VISION_PRODUCT_SET_ID,
+        location=GCP_VISION_LOCATION,
+        product_set=product_set,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='product_set_create_2',
+    )
+    # [END howto_operator_vision_product_set_create_2]
+
+    # Second 'create' task with the same product_set_id to demonstrate idempotence
+    product_set_create_2_idempotence = CloudVisionProductSetCreateOperator(
+        product_set_id=GCP_VISION_PRODUCT_SET_ID,
+        location=GCP_VISION_LOCATION,
+        product_set=product_set,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='product_set_create_2_idempotence',
+    )
+
+    # [START howto_operator_vision_product_set_get_2]
+    product_set_get_2 = CloudVisionProductSetGetOperator(
+        location=GCP_VISION_LOCATION, product_set_id=GCP_VISION_PRODUCT_SET_ID, task_id='product_set_get_2'
+    )
+    # [END howto_operator_vision_product_set_get_2]
+
+    # [START howto_operator_vision_product_set_update_2]
+    product_set_update_2 = CloudVisionProductSetUpdateOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id=GCP_VISION_PRODUCT_SET_ID,
+        product_set=ProductSet(display_name='My Product Set 2'),
+        task_id='product_set_update_2',
+    )
+    # [END howto_operator_vision_product_set_update_2]
+
+    # [START howto_operator_vision_product_set_delete_2]
+    product_set_delete_2 = CloudVisionProductSetDeleteOperator(
+        location=GCP_VISION_LOCATION, product_set_id=GCP_VISION_PRODUCT_SET_ID, task_id='product_set_delete_2'
+    )
+    # [END howto_operator_vision_product_set_delete_2]
+
+    # [START howto_operator_vision_product_create_2]
+    product_create_2 = CloudVisionProductCreateOperator(
+        product_id=GCP_VISION_PRODUCT_ID,
+        location=GCP_VISION_LOCATION,
+        product=product,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='product_create_2',
+    )
+    # [END howto_operator_vision_product_create_2]
+
+    # Second 'create' task with the same product_id to demonstrate idempotence
+    product_create_2_idempotence = CloudVisionProductCreateOperator(
+        product_id=GCP_VISION_PRODUCT_ID,
+        location=GCP_VISION_LOCATION,
+        product=product,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='product_create_2_idempotence',
+    )
+
+    # [START howto_operator_vision_product_get_2]
+    product_get_2 = CloudVisionProductGetOperator(
+        location=GCP_VISION_LOCATION, product_id=GCP_VISION_PRODUCT_ID, task_id='product_get_2'
+    )
+    # [END howto_operator_vision_product_get_2]
+
+    # [START howto_operator_vision_product_update_2]
+    product_update_2 = CloudVisionProductUpdateOperator(
+        location=GCP_VISION_LOCATION,
+        product_id=GCP_VISION_PRODUCT_ID,
+        product=Product(display_name='My Product 2', description='My updated description'),
+        task_id='product_update_2',
+    )
+    # [END howto_operator_vision_product_update_2]
+
+    # [START howto_operator_vision_product_delete_2]
+    product_delete_2 = CloudVisionProductDeleteOperator(
+        location=GCP_VISION_LOCATION, product_id=GCP_VISION_PRODUCT_ID, task_id='product_delete_2'
+    )
+    # [END howto_operator_vision_product_delete_2]
+
+    # [START howto_operator_vision_reference_image_create_2]
+    reference_image_create_2 = CloudVisionReferenceImageCreateOperator(
+        location=GCP_VISION_LOCATION,
+        reference_image=reference_image,
+        product_id=GCP_VISION_PRODUCT_ID,
+        reference_image_id=GCP_VISION_REFERENCE_IMAGE_ID,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='reference_image_create_2',
+    )
+    # [END howto_operator_vision_reference_image_create_2]
+
+    # Second 'create' task with the same product_id to demonstrate idempotence
+    reference_image_create_2_idempotence = CloudVisionReferenceImageCreateOperator(
+        location=GCP_VISION_LOCATION,
+        reference_image=reference_image,
+        product_id=GCP_VISION_PRODUCT_ID,
+        reference_image_id=GCP_VISION_REFERENCE_IMAGE_ID,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='reference_image_create_2_idempotence',
+    )
+
+    # [START howto_operator_vision_add_product_to_product_set_2]
+    add_product_to_product_set_2 = CloudVisionAddProductToProductSetOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id=GCP_VISION_PRODUCT_SET_ID,
+        product_id=GCP_VISION_PRODUCT_ID,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='add_product_to_product_set_2',
+    )
+    # [END howto_operator_vision_add_product_to_product_set_2]
+
+    # [START howto_operator_vision_remove_product_from_product_set_2]
+    remove_product_from_product_set_2 = CloudVisionRemoveProductFromProductSetOperator(
+        location=GCP_VISION_LOCATION,
+        product_set_id=GCP_VISION_PRODUCT_SET_ID,
+        product_id=GCP_VISION_PRODUCT_ID,
+        retry=Retry(maximum=10.0),
+        timeout=5,
+        task_id='remove_product_from_product_set_2',
+    )
+    # [END howto_operator_vision_remove_product_from_product_set_2]
+
+    # Product path
+    product_create_2 >> product_create_2_idempotence >> product_get_2 >> product_update_2 >> product_delete_2
+
+    # ProductSet path
+    product_set_create_2 >> product_set_get_2 >> product_set_update_2 >> product_set_delete_2
+    product_set_create_2 >> product_set_create_2_idempotence >> product_set_delete_2
+
+    # ReferenceImage path
+    product_create_2 >> reference_image_create_2 >> reference_image_create_2_idempotence >> product_delete_2
+
+    # Product/ProductSet path
+    add_product_to_product_set_2 >> remove_product_from_product_set_2
+    product_set_create_2 >> add_product_to_product_set_2
+    product_create_2 >> add_product_to_product_set_2
+    remove_product_from_product_set_2 >> product_set_delete_2
+    remove_product_from_product_set_2 >> product_delete_2
+
+with models.DAG(
+    'example_gcp_vision_annotate_image', default_args=default_args, schedule_interval=None
+) as dag_annotate_image:
+    # ############################## #
+    # ### Annotate image example ### #
+    # ############################## #
+
+    # [START howto_operator_vision_annotate_image]
+    annotate_image = CloudVisionAnnotateImageOperator(
+        request=annotate_image_request, retry=Retry(maximum=10.0), timeout=5, task_id='annotate_image'
+    )
+    # [END howto_operator_vision_annotate_image]
+
+    # [START howto_operator_vision_annotate_image_result]
+    annotate_image_result = BashOperator(
+        bash_command="echo {{ task_instance.xcom_pull('annotate_image')"
+        "['logoAnnotations'][0]['description'] }}",
+        task_id='annotate_image_result',
+    )
+    # [END howto_operator_vision_annotate_image_result]
+
+    annotate_image >> annotate_image_result
diff --git a/airflow/contrib/example_dags/example_gcs_acl.py b/airflow/contrib/example_dags/example_gcs_acl.py
new file mode 100644
index 0000000000000..7247199a4f656
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcs_acl.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that creates a new ACL entry on the specified bucket and object.
+
+This DAG relies on the following OS environment variables
+
+* GCS_ACL_BUCKET - Name of a bucket.
+* GCS_ACL_OBJECT - Name of the object. For information about how to URL encode object
+    names to be path safe, see:
+    https://cloud.google.com/storage/docs/json_api/#encoding
+* GCS_ACL_ENTITY - The entity holding the permission.
+* GCS_ACL_BUCKET_ROLE - The access permission for the entity for the bucket.
+* GCS_ACL_OBJECT_ROLE - The access permission for the entity for the object.
+"""
+import os
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcs_acl_operator import \
+    GoogleCloudStorageBucketCreateAclEntryOperator, \
+    GoogleCloudStorageObjectCreateAclEntryOperator
+
+# [START howto_operator_gcs_acl_args_common]
+GCS_ACL_BUCKET = os.environ.get('GCS_ACL_BUCKET', 'example-bucket')
+GCS_ACL_OBJECT = os.environ.get('GCS_ACL_OBJECT', 'example-object')
+GCS_ACL_ENTITY = os.environ.get('GCS_ACL_ENTITY', 'example-entity')
+GCS_ACL_BUCKET_ROLE = os.environ.get('GCS_ACL_BUCKET_ROLE', 'example-bucket-role')
+GCS_ACL_OBJECT_ROLE = os.environ.get('GCS_ACL_OBJECT_ROLE', 'example-object-role')
+# [END howto_operator_gcs_acl_args_common]
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+
+with models.DAG(
+    'example_gcs_acl',
+    default_args=default_args,
+    schedule_interval=None  # Change to match your use case
+) as dag:
+    # [START howto_operator_gcs_bucket_create_acl_entry_task]
+    gcs_bucket_create_acl_entry_task = GoogleCloudStorageBucketCreateAclEntryOperator(
+        bucket=GCS_ACL_BUCKET,
+        entity=GCS_ACL_ENTITY,
+        role=GCS_ACL_BUCKET_ROLE,
+        task_id="gcs_bucket_create_acl_entry_task"
+    )
+    # [END howto_operator_gcs_bucket_create_acl_entry_task]
+    # [START howto_operator_gcs_object_create_acl_entry_task]
+    gcs_object_create_acl_entry_task = GoogleCloudStorageObjectCreateAclEntryOperator(
+        bucket=GCS_ACL_BUCKET,
+        object_name=GCS_ACL_OBJECT,
+        entity=GCS_ACL_ENTITY,
+        role=GCS_ACL_OBJECT_ROLE,
+        task_id="gcs_object_create_acl_entry_task"
+    )
+    # [END howto_operator_gcs_object_create_acl_entry_task]
+
+    gcs_bucket_create_acl_entry_task >> gcs_object_create_acl_entry_task
diff --git a/airflow/contrib/example_dags/example_gcs_to_bq_operator.py b/airflow/contrib/example_dags/example_gcs_to_bq_operator.py
new file mode 100644
index 0000000000000..b76d05eec138e
--- /dev/null
+++ b/airflow/contrib/example_dags/example_gcs_to_bq_operator.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any
+
+import airflow
+from airflow import models
+from airflow.operators import bash_operator
+
+gcs_to_bq = None  # type: Any
+try:
+    from airflow.contrib.operators import gcs_to_bq
+except ImportError:
+    pass
+
+
+if gcs_to_bq is not None:
+    args = {
+        'owner': 'airflow',
+        'start_date': airflow.utils.dates.days_ago(2)
+    }
+
+    dag = models.DAG(
+        dag_id='example_gcs_to_bq_operator', default_args=args,
+        schedule_interval=None)
+
+    create_test_dataset = bash_operator.BashOperator(
+        task_id='create_airflow_test_dataset',
+        bash_command='bq mk airflow_test',
+        dag=dag)
+
+    # [START howto_operator_gcs_to_bq]
+    load_csv = gcs_to_bq.GoogleCloudStorageToBigQueryOperator(
+        task_id='gcs_to_bq_example',
+        bucket='cloud-samples-data',
+        source_objects=['bigquery/us-states/us-states.csv'],
+        destination_project_dataset_table='airflow_test.gcs_to_bq_table',
+        schema_fields=[
+            {'name': 'name', 'type': 'STRING', 'mode': 'NULLABLE'},
+            {'name': 'post_abbr', 'type': 'STRING', 'mode': 'NULLABLE'},
+        ],
+        write_disposition='WRITE_TRUNCATE',
+        dag=dag)
+    # [END howto_operator_gcs_to_bq]
+
+    delete_test_dataset = bash_operator.BashOperator(
+        task_id='delete_airflow_test_dataset',
+        bash_command='bq rm -rf airflow_test',
+        dag=dag)
+
+    create_test_dataset >> load_csv >> delete_test_dataset
diff --git a/airflow/contrib/example_dags/example_kubernetes_annotation.py b/airflow/contrib/example_dags/example_kubernetes_annotation.py
new file mode 100644
index 0000000000000..058baf69901f7
--- /dev/null
+++ b/airflow/contrib/example_dags/example_kubernetes_annotation.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import print_function
+import airflow
+from airflow.operators.python_operator import PythonOperator
+from airflow.models import DAG
+
+args = {
+    'owner': 'airflow',
+    'start_date': airflow.utils.dates.days_ago(2)
+}
+
+dag = DAG(
+    dag_id='example_kubernetes_annotation', default_args=args,
+    schedule_interval=None
+)
+
+
+def print_stuff():
+    print("annotated!")
+
+
+# You can use annotations on your kubernetes pods!
+start_task = PythonOperator(
+    task_id="start_task", python_callable=print_stuff, dag=dag,
+    executor_config={
+        "KubernetesExecutor": {
+            "annotations": {"test": "annotation"}
+        }
+    }
+)
diff --git a/airflow/example_dags/example_kubernetes_executor.py b/airflow/contrib/example_dags/example_kubernetes_executor.py
similarity index 69%
rename from airflow/example_dags/example_kubernetes_executor.py
rename to airflow/contrib/example_dags/example_kubernetes_executor.py
index 1d9bb7304318b..d03e255ab3287 100644
--- a/airflow/example_dags/example_kubernetes_executor.py
+++ b/airflow/contrib/example_dags/example_kubernetes_executor.py
@@ -32,6 +32,31 @@
     schedule_interval=None
 )
 
+affinity = {
+    'podAntiAffinity': {
+        'requiredDuringSchedulingIgnoredDuringExecution': [
+            {
+                'topologyKey': 'kubernetes.io/hostname',
+                'labelSelector': {
+                    'matchExpressions': [
+                        {
+                            'key': 'app',
+                            'operator': 'In',
+                            'values': ['airflow']
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+}
+
+tolerations = [{
+    'key': 'dedicated',
+    'operator': 'Equal',
+    'value': 'airflow'
+}]
+
 
 def print_stuff():
     print("stuff!")
@@ -59,11 +84,14 @@ def use_zip_binary():
     executor_config={"KubernetesExecutor": {"image": "airflow/ci_zip:latest"}}
 )
 
-# Limit resources on this operator/task
+# Limit resources on this operator/task with node affinity & tolerations
 three_task = PythonOperator(
     task_id="three_task", python_callable=print_stuff, dag=dag,
     executor_config={
-        "KubernetesExecutor": {"request_memory": "128Mi", "limit_memory": "128Mi"}}
+        "KubernetesExecutor": {"request_memory": "128Mi",
+                               "limit_memory": "128Mi",
+                               "tolerations": tolerations,
+                               "affinity": affinity}}
 )
 
 start_task.set_downstream([one_task, two_task, three_task])
diff --git a/airflow/contrib/example_dags/example_kubernetes_executor_config.py b/airflow/contrib/example_dags/example_kubernetes_executor_config.py
new file mode 100644
index 0000000000000..1316b21803921
--- /dev/null
+++ b/airflow/contrib/example_dags/example_kubernetes_executor_config.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import print_function
+import airflow
+from airflow.operators.python_operator import PythonOperator
+from libs.helper import print_stuff
+from airflow.models import DAG
+import os
+
+args = {
+    'owner': 'airflow',
+    'start_date': airflow.utils.dates.days_ago(2)
+}
+
+dag = DAG(
+    dag_id='example_kubernetes_executor_config', default_args=args,
+    schedule_interval=None
+)
+
+
+def test_volume_mount():
+    with open('/foo/volume_mount_test.txt', 'w') as foo:
+        foo.write('Hello')
+
+    rc = os.system("cat /foo/volume_mount_test.txt")
+    assert rc == 0
+
+
+# You can use annotations on your kubernetes pods!
+start_task = PythonOperator(
+    task_id="start_task", python_callable=print_stuff, dag=dag,
+    executor_config={
+        "KubernetesExecutor": {
+            "annotations": {"test": "annotation"}
+        }
+    }
+)
+
+# You can mount volume or secret to the worker pod
+second_task = PythonOperator(
+    task_id="four_task", python_callable=test_volume_mount, dag=dag,
+    executor_config={
+        "KubernetesExecutor": {
+            "volumes": [
+                {
+                    "name": "example-kubernetes-test-volume",
+                    "hostPath": {"path": "/tmp/"},
+                },
+            ],
+            "volume_mounts": [
+                {
+                    "mountPath": "/foo/",
+                    "name": "example-kubernetes-test-volume",
+                },
+            ]
+        }
+    }
+)
+
+# Test that we can run tasks as a normal user
+third_task = PythonOperator(
+    task_id="non_root_task", python_callable=print_stuff, dag=dag,
+    executor_config={
+        "KubernetesExecutor": {
+            "securityContext": {
+                "runAsUser": 1000
+            }
+        }
+    }
+)
+
+start_task.set_downstream(second_task)
+second_task.set_downstream(third_task)
diff --git a/airflow/example_dags/example_kubernetes_operator.py b/airflow/contrib/example_dags/example_kubernetes_operator.py
similarity index 84%
rename from airflow/example_dags/example_kubernetes_operator.py
rename to airflow/contrib/example_dags/example_kubernetes_operator.py
index 92d73c5d3369d..4977890bc68f3 100644
--- a/airflow/example_dags/example_kubernetes_operator.py
+++ b/airflow/contrib/example_dags/example_kubernetes_operator.py
@@ -25,7 +25,7 @@
 
 try:
     # Kubernetes is optional, so not available in vanilla Airflow
-    # pip install airflow[kubernetes]
+    # pip install 'apache-airflow[kubernetes]'
     from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
 
     args = {
@@ -38,6 +38,14 @@
         default_args=args,
         schedule_interval=None)
 
+    tolerations = [
+        {
+            'key': "key",
+            'operator': 'Equal',
+            'value': 'value'
+        }
+    ]
+
     k = KubernetesPodOperator(
         namespace='default',
         image="ubuntu:16.04",
@@ -48,9 +56,12 @@
         in_cluster=False,
         task_id="task",
         get_logs=True,
-        dag=dag)
+        dag=dag,
+        is_delete_operator_pod=False,
+        tolerations=tolerations
+    )
 
 except ImportError as e:
     log.warn("Could not import KubernetesPodOperator: " + str(e))
     log.warn("Install kubernetes dependencies with: "
-             "    pip install airflow['kubernetes']")
+             "    pip install 'apache-airflow[kubernetes]'")
diff --git a/airflow/contrib/example_dags/example_qubole_operator.py b/airflow/contrib/example_dags/example_qubole_operator.py
index 826a50af99cd9..5f77d09ba1442 100644
--- a/airflow/contrib/example_dags/example_qubole_operator.py
+++ b/airflow/contrib/example_dags/example_qubole_operator.py
@@ -65,7 +65,7 @@ def compare_result(ds, **kwargs):
     fetch_logs=True,
     # If `fetch_logs`=true, will fetch qubole command logs and concatenate
     # them into corresponding airflow task logs
-    tags='aiflow_example_run',
+    tags='airflow_example_run',
     # To attach tags to qubole command, auto attach 3 tags - dag_id, task_id, run_id
     qubole_conn_id='qubole_default',
     # Connection id to submit commands inside QDS, if not set "qubole_default" is used
@@ -220,7 +220,7 @@ def main(args: Array[String]) {
     program=prog,
     language='scala',
     arguments='--class SparkPi',
-    tags='aiflow_example_run',
+    tags='airflow_example_run',
     dag=dag)
 
 t11.set_upstream(branching)
diff --git a/airflow/contrib/example_dags/example_twitter_README.md b/airflow/contrib/example_dags/example_twitter_README.md
index 319eac39f6953..7563b816656b6 100644
--- a/airflow/contrib/example_dags/example_twitter_README.md
+++ b/airflow/contrib/example_dags/example_twitter_README.md
@@ -1,3 +1,22 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Example Twitter DAG
 
 ***Introduction:*** This example dag depicts a typical ETL process and is a perfect use case automation scenario for Airflow. Please note that the main scripts associated with the tasks are returning None. The purpose of this DAG is to demonstrate how to write a functional DAG within Airflow.
@@ -31,6 +50,6 @@ CREATE TABLE toTwitter_A(id BIGINT, id_str STRING
 ```
 When you review the code for the DAG, you will notice that these tasks are generated using for loop. These two for loops could be combined into one loop. However, in most cases, you will be running different analysis on your incoming incoming and outgoing tweets, and hence they are kept separated in this example.
 Final step is a running the broker script, brokerapi.py, which will run queries in Hive and store the summarized data to MySQL in our case. To connect to Hive, pyhs2 library is extremely useful and easy to use. To insert data into MySQL from Python, sqlalchemy is also a good one to use.
-I hope you find this tutorial useful. If you have question feel free to ask me on [Twitter](https://twitter.com/EkhtiarSyed) or via the live Airflow chatroom room in [Gitter](https://gitter.im/airbnb/airflow).<p>
+I hope you find this tutorial useful. If you have question feel free to ask me on [Twitter](https://twitter.com/EkhtiarSyed) or via the live Airflow chatroom room in [Gitter](https://gitter.im/apache/airflow).<p>
 -Ekhtiar Syed
 Last Update: 8-April-2016
diff --git a/airflow/contrib/example_dags/example_winrm_operator.py b/airflow/contrib/example_dags/example_winrm_operator.py
index 195bf5d98d03c..83e1844ebb042 100644
--- a/airflow/contrib/example_dags/example_winrm_operator.py
+++ b/airflow/contrib/example_dags/example_winrm_operator.py
@@ -31,7 +31,7 @@
 from airflow.models import DAG
 from datetime import timedelta
 
-from airflow.contrib.hooks import WinRMHook
+from airflow.contrib.hooks.winrm_hook import WinRMHook
 from airflow.contrib.operators.winrm_operator import WinRMOperator
 
 
diff --git a/init.sh b/airflow/contrib/example_dags/libs/__init__.py
similarity index 92%
rename from init.sh
rename to airflow/contrib/example_dags/libs/__init__.py
index 6f4adcad06752..114d189da14ab 100644
--- a/init.sh
+++ b/airflow/contrib/example_dags/libs/__init__.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -8,14 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-source $AIRFLOW_HOME/env/bin/activate
diff --git a/run_tox.sh b/airflow/contrib/example_dags/libs/helper.py
old mode 100755
new mode 100644
similarity index 91%
rename from run_tox.sh
rename to airflow/contrib/example_dags/libs/helper.py
index b4f204d649a32..d7b62e65c7c88
--- a/run_tox.sh
+++ b/airflow/contrib/example_dags/libs/helper.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -6,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,6 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -o verbose
 
-python setup.py test --tox-args="-v -e $TOX_ENV"
+def print_stuff():
+    print("annotated!")
diff --git a/airflow/contrib/executors/kubernetes_executor.py b/airflow/contrib/executors/kubernetes_executor.py
index 65053bd56b2a2..ebd22076e078c 100644
--- a/airflow/contrib/executors/kubernetes_executor.py
+++ b/airflow/contrib/executors/kubernetes_executor.py
@@ -16,6 +16,9 @@
 # under the License.
 
 import base64
+import hashlib
+import re
+import json
 import multiprocessing
 from queue import Queue
 from dateutil import parser
@@ -29,8 +32,10 @@
 from airflow.contrib.kubernetes.worker_configuration import WorkerConfiguration
 from airflow.executors.base_executor import BaseExecutor
 from airflow.executors import Executors
-from airflow.models import TaskInstance, KubeResourceVersion, KubeWorkerIdentifier
+from airflow.models import TaskInstance
+from airflow.models.kubernetes import KubeResourceVersion, KubeWorkerIdentifier
 from airflow.utils.state import State
+from airflow.utils.db import provide_session, create_session
 from airflow import configuration, settings
 from airflow.exceptions import AirflowConfigException, AirflowException
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -39,7 +44,8 @@
 class KubernetesExecutorConfig:
     def __init__(self, image=None, image_pull_policy=None, request_memory=None,
                  request_cpu=None, limit_memory=None, limit_cpu=None,
-                 gcp_service_account_key=None):
+                 gcp_service_account_key=None, node_selectors=None, affinity=None,
+                 annotations=None, volumes=None, volume_mounts=None, tolerations=None):
         self.image = image
         self.image_pull_policy = image_pull_policy
         self.request_memory = request_memory
@@ -47,13 +53,23 @@ def __init__(self, image=None, image_pull_policy=None, request_memory=None,
         self.limit_memory = limit_memory
         self.limit_cpu = limit_cpu
         self.gcp_service_account_key = gcp_service_account_key
+        self.node_selectors = node_selectors
+        self.affinity = affinity
+        self.annotations = annotations
+        self.volumes = volumes
+        self.volume_mounts = volume_mounts
+        self.tolerations = tolerations
 
     def __repr__(self):
         return "{}(image={}, image_pull_policy={}, request_memory={}, request_cpu={}, " \
-               "limit_memory={}, limit_cpu={}, gcp_service_account_key={})" \
+               "limit_memory={}, limit_cpu={}, gcp_service_account_key={}, " \
+               "node_selectors={}, affinity={}, annotations={}, volumes={}, " \
+               "volume_mounts={}, tolerations={})" \
             .format(KubernetesExecutorConfig.__name__, self.image, self.image_pull_policy,
                     self.request_memory, self.request_cpu, self.limit_memory,
-                    self.limit_cpu, self.gcp_service_account_key)
+                    self.limit_cpu, self.gcp_service_account_key, self.node_selectors,
+                    self.affinity, self.annotations, self.volumes, self.volume_mounts,
+                    self.tolerations)
 
     @staticmethod
     def from_dict(obj):
@@ -73,7 +89,13 @@ def from_dict(obj):
             request_cpu=namespaced.get('request_cpu', None),
             limit_memory=namespaced.get('limit_memory', None),
             limit_cpu=namespaced.get('limit_cpu', None),
-            gcp_service_account_key=namespaced.get('gcp_service_account_key', None)
+            gcp_service_account_key=namespaced.get('gcp_service_account_key', None),
+            node_selectors=namespaced.get('node_selectors', None),
+            affinity=namespaced.get('affinity', None),
+            annotations=namespaced.get('annotations', {}),
+            volumes=namespaced.get('volumes', []),
+            volume_mounts=namespaced.get('volume_mounts', []),
+            tolerations=namespaced.get('tolerations', None),
         )
 
     def as_dict(self):
@@ -84,7 +106,13 @@ def as_dict(self):
             'request_cpu': self.request_cpu,
             'limit_memory': self.limit_memory,
             'limit_cpu': self.limit_cpu,
-            'gcp_service_account_key': self.gcp_service_account_key
+            'gcp_service_account_key': self.gcp_service_account_key,
+            'node_selectors': self.node_selectors,
+            'affinity': self.affinity,
+            'annotations': self.annotations,
+            'volumes': self.volumes,
+            'volume_mounts': self.volume_mounts,
+            'tolerations': self.tolerations,
         }
 
 
@@ -96,7 +124,12 @@ def __init__(self):
         configuration_dict = configuration.as_dict(display_sensitive=True)
         self.core_configuration = configuration_dict['core']
         self.kube_secrets = configuration_dict.get('kubernetes_secrets', {})
-        self.airflow_home = configuration.get(self.core_section, 'airflow_home')
+        self.kube_env_vars = configuration_dict.get('kubernetes_environment_variables', {})
+        self.env_from_configmap_ref = configuration.get(self.kubernetes_section,
+                                                        'env_from_configmap_ref')
+        self.env_from_secret_ref = configuration.get(self.kubernetes_section,
+                                                     'env_from_secret_ref')
+        self.airflow_home = settings.AIRFLOW_HOME
         self.dags_folder = configuration.get(self.core_section, 'dags_folder')
         self.parallelism = configuration.getint(self.core_section, 'PARALLELISM')
         self.worker_container_repository = configuration.get(
@@ -108,13 +141,24 @@ def __init__(self):
         self.kube_image_pull_policy = configuration.get(
             self.kubernetes_section, "worker_container_image_pull_policy"
         )
+        self.kube_node_selectors = configuration_dict.get('kubernetes_node_selectors', {})
+        self.kube_annotations = configuration_dict.get('kubernetes_annotations', {})
         self.delete_worker_pods = conf.getboolean(
             self.kubernetes_section, 'delete_worker_pods')
-
+        self.worker_pods_creation_batch_size = conf.getint(
+            self.kubernetes_section, 'worker_pods_creation_batch_size')
         self.worker_service_account_name = conf.get(
             self.kubernetes_section, 'worker_service_account_name')
         self.image_pull_secrets = conf.get(self.kubernetes_section, 'image_pull_secrets')
 
+        # NOTE: user can build the dags into the docker image directly,
+        # this will set to True if so
+        self.dags_in_image = conf.getboolean(self.kubernetes_section, 'dags_in_image')
+
+        # Run as user for pod security context
+        self.worker_run_as_user = conf.get(self.kubernetes_section, 'run_as_user')
+        self.worker_fs_group = conf.get(self.kubernetes_section, 'fs_group')
+
         # NOTE: `git_repo` and `git_branch` must be specified together as a pair
         # The http URL of the git repository to clone from
         self.git_repo = conf.get(self.kubernetes_section, 'git_repo')
@@ -122,11 +166,22 @@ def __init__(self):
         self.git_branch = conf.get(self.kubernetes_section, 'git_branch')
         # Optionally, the directory in the git repository containing the dags
         self.git_subpath = conf.get(self.kubernetes_section, 'git_subpath')
-
-        # Optionally a user may supply a `git_user` and `git_password` for private
-        # repositories
+        # Optionally, the root directory for git operations
+        self.git_sync_root = conf.get(self.kubernetes_section, 'git_sync_root')
+        # Optionally, the name at which to publish the checked-out files under --root
+        self.git_sync_dest = conf.get(self.kubernetes_section, 'git_sync_dest')
+        # Optionally, if git_dags_folder_mount_point is set the worker will use
+        # {git_dags_folder_mount_point}/{git_sync_dest}/{git_subpath} as dags_folder
+        self.git_dags_folder_mount_point = conf.get(self.kubernetes_section,
+                                                    'git_dags_folder_mount_point')
+
+        # Optionally a user may supply a (`git_user` AND `git_password`) OR
+        # (`git_ssh_key_secret_name` AND `git_ssh_key_secret_key`) for private repositories
         self.git_user = conf.get(self.kubernetes_section, 'git_user')
         self.git_password = conf.get(self.kubernetes_section, 'git_password')
+        self.git_ssh_key_secret_name = conf.get(self.kubernetes_section, 'git_ssh_key_secret_name')
+        self.git_ssh_known_hosts_configmap_name = conf.get(self.kubernetes_section,
+                                                           'git_ssh_known_hosts_configmap_name')
 
         # NOTE: The user may optionally use a volume claim to mount a PV containing
         # DAGs directly
@@ -145,6 +200,12 @@ def __init__(self):
         self.logs_volume_subpath = conf.get(
             self.kubernetes_section, 'logs_volume_subpath')
 
+        # Optionally, hostPath volume containing DAGs
+        self.dags_volume_host = conf.get(self.kubernetes_section, 'dags_volume_host')
+
+        # Optionally, write logs to a hostPath Volume
+        self.logs_volume_host = conf.get(self.kubernetes_section, 'logs_volume_host')
+
         # This prop may optionally be set for PV Claims and is used to write logs
         self.base_log_folder = configuration.get(self.core_section, 'base_log_folder')
 
@@ -179,13 +240,41 @@ def __init__(self):
         # configmap
         self.airflow_configmap = conf.get(self.kubernetes_section, 'airflow_configmap')
 
+        affinity_json = conf.get(self.kubernetes_section, 'affinity')
+        if affinity_json:
+            self.kube_affinity = json.loads(affinity_json)
+        else:
+            self.kube_affinity = None
+
+        tolerations_json = conf.get(self.kubernetes_section, 'tolerations')
+        if tolerations_json:
+            self.kube_tolerations = json.loads(tolerations_json)
+        else:
+            self.kube_tolerations = None
+
         self._validate()
 
     def _validate(self):
-        if not self.dags_volume_claim and (not self.git_repo or not self.git_branch):
+        # TODO: use XOR for dags_volume_claim and git_dags_folder_mount_point
+        if not self.dags_volume_claim \
+           and not self.dags_volume_host \
+           and not self.dags_in_image \
+           and (not self.git_repo or not self.git_branch or not self.git_dags_folder_mount_point):
             raise AirflowConfigException(
                 'In kubernetes mode the following must be set in the `kubernetes` '
-                'config section: `dags_volume_claim` or `git_repo and git_branch`')
+                'config section: `dags_volume_claim` '
+                'or `dags_volume_host` '
+                'or `dags_in_image` '
+                'or `git_repo and git_branch and git_dags_folder_mount_point`')
+        if self.git_repo \
+           and (self.git_user or self.git_password) \
+           and self.git_ssh_key_secret_name:
+            raise AirflowConfigException(
+                'In kubernetes mode, using `git_repo` to pull the DAGs: '
+                'for private repositories, either `git_user` and `git_password` '
+                'must be set for authentication through user credentials; '
+                'or `git_ssh_key_secret_name` must be set for authentication '
+                'through ssh key, but not both')
 
 
 class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin, object):
@@ -275,8 +364,7 @@ def process_status(self, pod_id, status, labels, resource_version):
 
 
 class AirflowKubernetesScheduler(LoggingMixin):
-    def __init__(self, kube_config, task_queue, result_queue, session,
-                 kube_client, worker_uuid):
+    def __init__(self, kube_config, task_queue, result_queue, kube_client, worker_uuid):
         self.log.debug("Creating Kubernetes executor")
         self.kube_config = kube_config
         self.task_queue = task_queue
@@ -287,12 +375,11 @@ def __init__(self, kube_config, task_queue, result_queue, session,
         self.launcher = PodLauncher(kube_client=self.kube_client)
         self.worker_configuration = WorkerConfiguration(kube_config=self.kube_config)
         self.watcher_queue = multiprocessing.Queue()
-        self._session = session
         self.worker_uuid = worker_uuid
         self.kube_watcher = self._make_kube_watcher()
 
     def _make_kube_watcher(self):
-        resource_version = KubeResourceVersion.get_current_resource_version(self._session)
+        resource_version = KubeResourceVersion.get_current_resource_version()
         watcher = KubernetesJobWatcher(self.namespace, self.watcher_queue,
                                        resource_version, self.worker_uuid)
         watcher.start()
@@ -317,13 +404,15 @@ def run_next(self, next_job):
         """
         self.log.info('Kubernetes job is %s', str(next_job))
         key, command, kube_executor_config = next_job
-        dag_id, task_id, execution_date = key
+        dag_id, task_id, execution_date, try_number = key
         self.log.debug("Kubernetes running for command %s", command)
         self.log.debug("Kubernetes launching image %s", self.kube_config.kube_image)
         pod = self.worker_configuration.make_pod(
             namespace=self.namespace, worker_uuid=self.worker_uuid,
             pod_id=self._create_pod_id(dag_id, task_id),
-            dag_id=dag_id, task_id=task_id,
+            dag_id=self._make_safe_label_value(dag_id),
+            task_id=self._make_safe_label_value(task_id),
+            try_number=try_number,
             execution_date=self._datetime_to_label_safe_datestring(execution_date),
             airflow_command=command, kube_executor_config=kube_executor_config
         )
@@ -382,7 +471,7 @@ def _strip_unsafe_kubernetes_special_chars(string):
 
     @staticmethod
     def _make_safe_pod_id(safe_dag_id, safe_task_id, safe_uuid):
-        """
+        r"""
         Kubernetes pod names must be <= 253 chars and must pass the following regex for
         validation
         "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"
@@ -400,6 +489,27 @@ def _make_safe_pod_id(safe_dag_id, safe_task_id, safe_uuid):
 
         return safe_pod_id
 
+    @staticmethod
+    def _make_safe_label_value(string):
+        """
+        Valid label values must be 63 characters or less and must be empty or begin and
+        end with an alphanumeric character ([a-z0-9A-Z]) with dashes (-), underscores (_),
+        dots (.), and alphanumerics between.
+
+        If the label value is then greater than 63 chars once made safe, or differs in any
+        way from the original value sent to this function, then we need to truncate to
+        53chars, and append it with a unique hash.
+        """
+        MAX_LABEL_LEN = 63
+
+        safe_label = re.sub(r'^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$', '', string)
+
+        if len(safe_label) > MAX_LABEL_LEN or string != safe_label:
+            safe_hash = hashlib.md5(string.encode()).hexdigest()[:9]
+            safe_label = safe_label[:MAX_LABEL_LEN - len(safe_hash) - 1] + "-" + safe_hash
+
+        return safe_label
+
     @staticmethod
     def _create_pod_id(dag_id, task_id):
         safe_dag_id = AirflowKubernetesScheduler._strip_unsafe_kubernetes_special_chars(
@@ -418,7 +528,7 @@ def _label_safe_datestring_to_datetime(string):
         "_", let's
         replace ":" with "_"
 
-        :param string: string
+        :param string: str
         :return: datetime.datetime object
         """
         return parser.parse(string.replace('_plus_', '+').replace("_", ":"))
@@ -435,30 +545,65 @@ def _datetime_to_label_safe_datestring(datetime_obj):
         return datetime_obj.isoformat().replace(":", "_").replace('+', '_plus_')
 
     def _labels_to_key(self, labels):
+        try_num = 1
+        try:
+            try_num = int(labels.get('try_number', '1'))
+        except ValueError:
+            self.log.warn("could not get try_number as an int: %s", labels.get('try_number', '1'))
+
         try:
-            return (
-                labels['dag_id'], labels['task_id'],
-                self._label_safe_datestring_to_datetime(labels['execution_date']))
+            dag_id = labels['dag_id']
+            task_id = labels['task_id']
+            ex_time = self._label_safe_datestring_to_datetime(labels['execution_date'])
         except Exception as e:
             self.log.warn(
-                'Error while converting labels to key; labels: %s; exception: %s',
+                'Error while retrieving labels; labels: %s; exception: %s',
                 labels, e
             )
             return None
 
+        with create_session() as session:
+            tasks = (
+                session
+                .query(TaskInstance)
+                .filter_by(execution_date=ex_time).all()
+            )
+            self.log.info(
+                'Checking %s task instances.',
+                len(tasks)
+            )
+            for task in tasks:
+                if (
+                    self._make_safe_label_value(task.dag_id) == dag_id and
+                    self._make_safe_label_value(task.task_id) == task_id and
+                    task.execution_date == ex_time
+                ):
+                    self.log.info(
+                        'Found matching task %s-%s (%s) with current state of %s',
+                        task.dag_id, task.task_id, task.execution_date, task.state
+                    )
+                    dag_id = task.dag_id
+                    task_id = task.task_id
+                    return (dag_id, task_id, ex_time, try_num)
+        self.log.warn(
+            'Failed to find and match task details to a pod; labels: %s',
+            labels
+        )
+        return None
+
 
 class KubernetesExecutor(BaseExecutor, LoggingMixin):
     def __init__(self):
         self.kube_config = KubeConfig()
         self.task_queue = None
-        self._session = None
         self.result_queue = None
         self.kube_scheduler = None
         self.kube_client = None
         self.worker_uuid = None
         super(KubernetesExecutor, self).__init__(parallelism=self.kube_config.parallelism)
 
-    def clear_not_launched_queued_tasks(self):
+    @provide_session
+    def clear_not_launched_queued_tasks(self, session=None):
         """
         If the airflow scheduler restarts with pending "Queued" tasks, the tasks may or
         may not
@@ -474,18 +619,25 @@ def clear_not_launched_queued_tasks(self):
         proper support
         for State.LAUNCHED
         """
-        queued_tasks = self._session.query(
-            TaskInstance).filter(TaskInstance.state == State.QUEUED).all()
+        queued_tasks = session\
+            .query(TaskInstance)\
+            .filter(TaskInstance.state == State.QUEUED).all()
         self.log.info(
             'When executor started up, found %s queued task instances',
             len(queued_tasks)
         )
 
         for task in queued_tasks:
-            dict_string = "dag_id={},task_id={},execution_date={},airflow-worker={}" \
-                .format(task.dag_id, task.task_id,
-                        AirflowKubernetesScheduler._datetime_to_label_safe_datestring(
-                            task.execution_date), self.worker_uuid)
+            dict_string = (
+                "dag_id={},task_id={},execution_date={},airflow-worker={}".format(
+                    AirflowKubernetesScheduler._make_safe_label_value(task.dag_id),
+                    AirflowKubernetesScheduler._make_safe_label_value(task.task_id),
+                    AirflowKubernetesScheduler._datetime_to_label_safe_datestring(
+                        task.execution_date
+                    ),
+                    self.worker_uuid
+                )
+            )
             kwargs = dict(label_selector=dict_string)
             pod_list = self.kube_client.list_namespaced_pod(
                 self.kube_config.kube_namespace, **kwargs)
@@ -494,14 +646,12 @@ def clear_not_launched_queued_tasks(self):
                     'TaskInstance: %s found in queued state but was not launched, '
                     'rescheduling', task
                 )
-                self._session.query(TaskInstance).filter(
+                session.query(TaskInstance).filter(
                     TaskInstance.dag_id == task.dag_id,
                     TaskInstance.task_id == task.task_id,
                     TaskInstance.execution_date == task.execution_date
                 ).update({TaskInstance.state: State.NONE})
 
-        self._session.commit()
-
     def _inject_secrets(self):
         def _create_or_update_secret(secret_name, secret_path):
             try:
@@ -538,20 +688,18 @@ def _create_or_update_secret(secret_name, secret_path):
 
     def start(self):
         self.log.info('Start Kubernetes executor')
-        self._session = settings.Session()
-        self.worker_uuid = KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid(
-            self._session)
+        self.worker_uuid = KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid()
         self.log.debug('Start with worker_uuid: %s', self.worker_uuid)
         # always need to reset resource version since we don't know
         # when we last started, note for behavior below
         # https://github.com/kubernetes-client/python/blob/master/kubernetes/docs
         # /CoreV1Api.md#list_namespaced_pod
-        KubeResourceVersion.reset_resource_version(self._session)
+        KubeResourceVersion.reset_resource_version()
         self.task_queue = Queue()
         self.result_queue = Queue()
         self.kube_client = get_kube_client()
         self.kube_scheduler = AirflowKubernetesScheduler(
-            self.kube_config, self.task_queue, self.result_queue, self._session,
+            self.kube_config, self.task_queue, self.result_queue,
             self.kube_client, self.worker_uuid
         )
         self._inject_secrets()
@@ -567,9 +715,9 @@ def execute_async(self, key, command, queue=None, executor_config=None):
 
     def sync(self):
         if self.running:
-            self.log.info('self.running: %s', self.running)
+            self.log.debug('self.running: %s', self.running)
         if self.queued_tasks:
-            self.log.info('self.queued: %s', self.queued_tasks)
+            self.log.debug('self.queued: %s', self.queued_tasks)
         self.kube_scheduler.sync()
 
         last_resource_version = None
@@ -578,14 +726,24 @@ def sync(self):
             key, state, pod_id, resource_version = results
             last_resource_version = resource_version
             self.log.info('Changing state of %s to %s', results, state)
-            self._change_state(key, state, pod_id)
+            try:
+                self._change_state(key, state, pod_id)
+            except Exception as e:
+                self.log.exception('Exception: %s when attempting ' +
+                                   'to change state of %s to %s, re-queueing.', e, results, state)
+                self.result_queue.put(results)
 
-        KubeResourceVersion.checkpoint_resource_version(
-            last_resource_version, session=self._session)
+        KubeResourceVersion.checkpoint_resource_version(last_resource_version)
 
-        if not self.task_queue.empty():
-            key, command, kube_executor_config = self.task_queue.get()
-            self.kube_scheduler.run_next((key, command, kube_executor_config))
+        for i in range(min((self.kube_config.worker_pods_creation_batch_size, self.task_queue.qsize()))):
+            task = self.task_queue.get()
+
+            try:
+                self.kube_scheduler.run_next(task)
+            except ApiException:
+                self.log.exception('ApiException when attempting ' +
+                                   'to run task, re-queueing.')
+                self.task_queue.put(task)
 
     def _change_state(self, key, state, pod_id):
         if state != State.RUNNING:
@@ -597,16 +755,16 @@ def _change_state(self, key, state, pod_id):
                 self.log.debug('Could not find key: %s', str(key))
                 pass
         self.event_buffer[key] = state
-        (dag_id, task_id, ex_time) = key
-        item = self._session.query(TaskInstance).filter_by(
-            dag_id=dag_id,
-            task_id=task_id,
-            execution_date=ex_time
-        ).one()
-        if state:
-            item.state = state
-            self._session.add(item)
-            self._session.commit()
+        (dag_id, task_id, ex_time, try_number) = key
+        with create_session() as session:
+            item = session.query(TaskInstance).filter_by(
+                dag_id=dag_id,
+                task_id=task_id,
+                execution_date=ex_time
+            ).one()
+            if state:
+                item.state = state
+                session.add(item)
 
     def end(self):
         self.log.info('Shutting down Kubernetes executor')
diff --git a/airflow/contrib/executors/mesos_executor.py b/airflow/contrib/executors/mesos_executor.py
index ff974ffc3c080..f101348acc1a1 100644
--- a/airflow/contrib/executors/mesos_executor.py
+++ b/airflow/contrib/executors/mesos_executor.py
@@ -19,10 +19,6 @@
 
 from future import standard_library
 
-from airflow.utils.log.logging_mixin import LoggingMixin
-from airflow.www.utils import LoginMixin
-
-
 from builtins import str
 from queue import Queue
 
@@ -49,7 +45,7 @@ def get_framework_name():
 
 # AirflowMesosScheduler, implements Mesos Scheduler interface
 # To schedule airflow jobs on mesos
-class AirflowMesosScheduler(mesos.interface.Scheduler, LoggingMixin):
+class AirflowMesosScheduler(mesos.interface.Scheduler):
     """
     Airflow Mesos scheduler implements mesos scheduler interface
     to schedule airflow tasks on mesos.
@@ -80,7 +76,7 @@ def registered(self, driver, frameworkId, masterInfo):
         if configuration.conf.getboolean('mesos', 'CHECKPOINT') and \
                 configuration.conf.get('mesos', 'FAILOVER_TIMEOUT'):
             # Import here to work around a circular import error
-            from airflow.models import Connection
+            from airflow.models.connection import Connection
 
             # Update the Framework ID in the database.
             session = Session()
@@ -162,7 +158,7 @@ def resourceOffers(self, driver, offers):
 
                 command = mesos_pb2.CommandInfo()
                 command.shell = True
-                command.value = cmd
+                command.value = " ".join(cmd)
                 task.command.MergeFrom(command)
 
                 # If docker image for airflow is specified in config then pull that
@@ -213,7 +209,7 @@ def statusUpdate(self, driver, update):
             self.task_queue.task_done()
 
 
-class MesosExecutor(BaseExecutor, LoginMixin):
+class MesosExecutor(BaseExecutor):
     """
     MesosExecutor allows distributing the execution of task
     instances to multiple mesos workers.
@@ -253,7 +249,7 @@ def start(self):
 
             if configuration.conf.get('mesos', 'FAILOVER_TIMEOUT'):
                 # Import here to work around a circular import error
-                from airflow.models import Connection
+                from airflow.models.connection import Connection
 
                 # Query the database to get the ID of the Mesos Framework, if available.
                 conn_id = FRAMEWORK_CONNID_PREFIX + framework.name
diff --git a/airflow/contrib/hooks/aws_athena_hook.py b/airflow/contrib/hooks/aws_athena_hook.py
new file mode 100644
index 0000000000000..f11ff23c515f4
--- /dev/null
+++ b/airflow/contrib/hooks/aws_athena_hook.py
@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from time import sleep
+from airflow.contrib.hooks.aws_hook import AwsHook
+
+
+class AWSAthenaHook(AwsHook):
+    """
+    Interact with AWS Athena to run, poll queries and return query results
+
+    :param aws_conn_id: aws connection to use.
+    :type aws_conn_id: str
+    :param sleep_time: Time to wait between two consecutive call to check query status on athena
+    :type sleep_time: int
+    """
+
+    INTERMEDIATE_STATES = ('QUEUED', 'RUNNING',)
+    FAILURE_STATES = ('FAILED', 'CANCELLED',)
+    SUCCESS_STATES = ('SUCCEEDED',)
+
+    def __init__(self, aws_conn_id='aws_default', sleep_time=30, *args, **kwargs):
+        super(AWSAthenaHook, self).__init__(aws_conn_id, **kwargs)
+        self.sleep_time = sleep_time
+        self.conn = None
+
+    def get_conn(self):
+        """
+        check if aws conn exists already or create one and return it
+
+        :return: boto3 session
+        """
+        if not self.conn:
+            self.conn = self.get_client_type('athena')
+        return self.conn
+
+    def run_query(self, query, query_context, result_configuration, client_request_token=None):
+        """
+        Run Presto query on athena with provided config and return submitted query_execution_id
+
+        :param query: Presto query to run
+        :type query: str
+        :param query_context: Context in which query need to be run
+        :type query_context: dict
+        :param result_configuration: Dict with path to store results in and config related to encryption
+        :type result_configuration: dict
+        :param client_request_token: Unique token created by user to avoid multiple executions of same query
+        :type client_request_token: str
+        :return: str
+        """
+        response = self.conn.start_query_execution(QueryString=query,
+                                                   ClientRequestToken=client_request_token,
+                                                   QueryExecutionContext=query_context,
+                                                   ResultConfiguration=result_configuration)
+        query_execution_id = response['QueryExecutionId']
+        return query_execution_id
+
+    def check_query_status(self, query_execution_id):
+        """
+        Fetch the status of submitted athena query. Returns None or one of valid query states.
+
+        :param query_execution_id: Id of submitted athena query
+        :type query_execution_id: str
+        :return: str
+        """
+        response = self.conn.get_query_execution(QueryExecutionId=query_execution_id)
+        state = None
+        try:
+            state = response['QueryExecution']['Status']['State']
+        except Exception as ex:
+            self.log.error('Exception while getting query state', ex)
+        finally:
+            return state
+
+    def get_query_results(self, query_execution_id):
+        """
+        Fetch submitted athena query results. returns none if query is in intermediate state or
+        failed/cancelled state else dict of query output
+
+        :param query_execution_id: Id of submitted athena query
+        :type query_execution_id: str
+        :return: dict
+        """
+        query_state = self.check_query_status(query_execution_id)
+        if query_state is None:
+            self.log.error('Invalid Query state')
+            return None
+        elif query_state in self.INTERMEDIATE_STATES or query_state in self.FAILURE_STATES:
+            self.log.error('Query is in {state} state. Cannot fetch results'.format(state=query_state))
+            return None
+        return self.conn.get_query_results(QueryExecutionId=query_execution_id)
+
+    def poll_query_status(self, query_execution_id, max_tries=None):
+        """
+        Poll the status of submitted athena query until query state reaches final state.
+        Returns one of the final states
+
+        :param query_execution_id: Id of submitted athena query
+        :type query_execution_id: str
+        :param max_tries: Number of times to poll for query state before function exits
+        :type max_tries: int
+        :return: str
+        """
+        try_number = 1
+        final_query_state = None  # Query state when query reaches final state or max_tries reached
+        while True:
+            query_state = self.check_query_status(query_execution_id)
+            if query_state is None:
+                self.log.info('Trial {try_number}: Invalid query state. Retrying again'.format(
+                    try_number=try_number))
+            elif query_state in self.INTERMEDIATE_STATES:
+                self.log.info('Trial {try_number}: Query is still in an intermediate state - {state}'
+                              .format(try_number=try_number, state=query_state))
+            else:
+                self.log.info('Trial {try_number}: Query execution completed. Final state is {state}'
+                              .format(try_number=try_number, state=query_state))
+                final_query_state = query_state
+                break
+            if max_tries and try_number >= max_tries:  # Break loop if max_tries reached
+                final_query_state = query_state
+                break
+            try_number += 1
+            sleep(self.sleep_time)
+        return final_query_state
+
+    def stop_query(self, query_execution_id):
+        """
+        Cancel the submitted athena query
+
+        :param query_execution_id: Id of submitted athena query
+        :type query_execution_id: str
+        :return: dict
+        """
+        return self.conn.stop_query_execution(QueryExecutionId=query_execution_id)
diff --git a/airflow/contrib/hooks/aws_glue_catalog_hook.py b/airflow/contrib/hooks/aws_glue_catalog_hook.py
new file mode 100644
index 0000000000000..4b7fba533f920
--- /dev/null
+++ b/airflow/contrib/hooks/aws_glue_catalog_hook.py
@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+
+
+class AwsGlueCatalogHook(AwsHook):
+    """
+    Interact with AWS Glue Catalog
+
+    :param aws_conn_id: ID of the Airflow connection where
+        credentials and extra configuration are stored
+    :type aws_conn_id: str
+    :param region_name: aws region name (example: us-east-1)
+    :type region_name: str
+    """
+
+    def __init__(self,
+                 aws_conn_id='aws_default',
+                 region_name=None,
+                 *args,
+                 **kwargs):
+        self.region_name = region_name
+        super(AwsGlueCatalogHook, self).__init__(aws_conn_id=aws_conn_id, *args, **kwargs)
+
+    def get_conn(self):
+        """
+        Returns glue connection object.
+        """
+        self.conn = self.get_client_type('glue', self.region_name)
+        return self.conn
+
+    def get_partitions(self,
+                       database_name,
+                       table_name,
+                       expression='',
+                       page_size=None,
+                       max_items=None):
+        """
+        Retrieves the partition values for a table.
+
+        :param database_name: The name of the catalog database where the partitions reside.
+        :type database_name: str
+        :param table_name: The name of the partitions' table.
+        :type table_name: str
+        :param expression: An expression filtering the partitions to be returned.
+            Please see official AWS documentation for further information.
+            https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-partitions.html#aws-glue-api-catalog-partitions-GetPartitions
+        :type expression: str
+        :param page_size: pagination size
+        :type page_size: int
+        :param max_items: maximum items to return
+        :type max_items: int
+        :return: set of partition values where each value is a tuple since
+            a partition may be composed of multiple columns. For example:
+            ``{('2018-01-01','1'), ('2018-01-01','2')}``
+        """
+        config = {
+            'PageSize': page_size,
+            'MaxItems': max_items,
+        }
+
+        paginator = self.get_conn().get_paginator('get_partitions')
+        response = paginator.paginate(
+            DatabaseName=database_name,
+            TableName=table_name,
+            Expression=expression,
+            PaginationConfig=config
+        )
+
+        partitions = set()
+        for page in response:
+            for p in page['Partitions']:
+                partitions.add(tuple(p['Values']))
+
+        return partitions
+
+    def check_for_partition(self, database_name, table_name, expression):
+        """
+        Checks whether a partition exists
+
+        :param database_name: Name of hive database (schema) @table belongs to
+        :type database_name: str
+        :param table_name: Name of hive table @partition belongs to
+        :type table_name: str
+        :expression: Expression that matches the partitions to check for
+            (eg `a = 'b' AND c = 'd'`)
+        :type expression: str
+        :rtype: bool
+
+        >>> hook = AwsGlueCatalogHook()
+        >>> t = 'static_babynames_partitioned'
+        >>> hook.check_for_partition('airflow', t, "ds='2015-01-01'")
+        True
+        """
+        partitions = self.get_partitions(database_name, table_name, expression, max_items=1)
+
+        if partitions:
+            return True
+        else:
+            return False
+
+    def get_table(self, database_name, table_name):
+        """
+        Get the information of the table
+
+        :param database_name: Name of hive database (schema) @table belongs to
+        :type database_name: str
+        :param table_name: Name of hive table
+        :type table_name: str
+        :rtype: dict
+
+        >>> hook = AwsGlueCatalogHook()
+        >>> r = hook.get_table('db', 'table_foo')
+        >>> r['Name'] = 'table_foo'
+        """
+
+        result = self.get_conn().get_table(DatabaseName=database_name, Name=table_name)
+
+        return result['Table']
+
+    def get_table_location(self, database_name, table_name):
+        """
+        Get the physical location of the table
+
+        :param database_name: Name of hive database (schema) @table belongs to
+        :type database_name: str
+        :param table_name: Name of hive table
+        :type table_name: str
+        :return: str
+        """
+
+        table = self.get_table(database_name, table_name)
+
+        return table['StorageDescriptor']['Location']
diff --git a/airflow/contrib/hooks/aws_hook.py b/airflow/contrib/hooks/aws_hook.py
index c712d2de7910b..9d4a73e1c0c6b 100644
--- a/airflow/contrib/hooks/aws_hook.py
+++ b/airflow/contrib/hooks/aws_hook.py
@@ -17,7 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-
 import boto3
 import configparser
 import logging
@@ -72,7 +71,7 @@ def _parse_s3_config(config_file_name, config_format='boto', profile=None):
         try:
             access_key = config.get(cred_section, key_id_option)
             secret_key = config.get(cred_section, secret_key_option)
-        except:
+        except Exception:
             logging.warning("Option Error in parsing s3 config file")
             raise
         return access_key, secret_key
@@ -84,8 +83,9 @@ class AwsHook(BaseHook):
     This class is a thin wrapper around the boto3 python library.
     """
 
-    def __init__(self, aws_conn_id='aws_default'):
+    def __init__(self, aws_conn_id='aws_default', verify=None):
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
 
     def _get_credentials(self, region_name):
         aws_access_key_id = None
@@ -96,32 +96,36 @@ def _get_credentials(self, region_name):
         if self.aws_conn_id:
             try:
                 connection_object = self.get_connection(self.aws_conn_id)
+                extra_config = connection_object.extra_dejson
                 if connection_object.login:
                     aws_access_key_id = connection_object.login
                     aws_secret_access_key = connection_object.password
 
-                elif 'aws_secret_access_key' in connection_object.extra_dejson:
-                    aws_access_key_id = connection_object.extra_dejson[
+                elif 'aws_secret_access_key' in extra_config:
+                    aws_access_key_id = extra_config[
                         'aws_access_key_id']
-                    aws_secret_access_key = connection_object.extra_dejson[
+                    aws_secret_access_key = extra_config[
                         'aws_secret_access_key']
 
-                elif 's3_config_file' in connection_object.extra_dejson:
+                elif 's3_config_file' in extra_config:
                     aws_access_key_id, aws_secret_access_key = \
                         _parse_s3_config(
-                            connection_object.extra_dejson['s3_config_file'],
-                            connection_object.extra_dejson.get('s3_config_format'))
+                            extra_config['s3_config_file'],
+                            extra_config.get('s3_config_format'),
+                            extra_config.get('profile'))
 
                 if region_name is None:
-                    region_name = connection_object.extra_dejson.get('region_name')
+                    region_name = extra_config.get('region_name')
 
-                role_arn = connection_object.extra_dejson.get('role_arn')
-                aws_account_id = connection_object.extra_dejson.get('aws_account_id')
-                aws_iam_role = connection_object.extra_dejson.get('aws_iam_role')
+                role_arn = extra_config.get('role_arn')
+                external_id = extra_config.get('external_id')
+                aws_account_id = extra_config.get('aws_account_id')
+                aws_iam_role = extra_config.get('aws_iam_role')
 
                 if role_arn is None and aws_account_id is not None and \
                         aws_iam_role is not None:
-                    role_arn = "arn:aws:iam::" + aws_account_id + ":role/" + aws_iam_role
+                    role_arn = "arn:aws:iam::{}:role/{}" \
+                        .format(aws_account_id, aws_iam_role)
 
                 if role_arn is not None:
                     sts_session = boto3.session.Session(
@@ -130,14 +134,23 @@ def _get_credentials(self, region_name):
                         region_name=region_name)
 
                     sts_client = sts_session.client('sts')
-                    sts_response = sts_client.assume_role(
-                        RoleArn=role_arn,
-                        RoleSessionName='Airflow_' + self.aws_conn_id)
-                    aws_access_key_id = sts_response['Credentials']['AccessKeyId']
-                    aws_secret_access_key = sts_response['Credentials']['SecretAccessKey']
-                    aws_session_token = sts_response['Credentials']['SessionToken']
 
-                endpoint_url = connection_object.extra_dejson.get('host')
+                    if external_id is None:
+                        sts_response = sts_client.assume_role(
+                            RoleArn=role_arn,
+                            RoleSessionName='Airflow_' + self.aws_conn_id)
+                    else:
+                        sts_response = sts_client.assume_role(
+                            RoleArn=role_arn,
+                            RoleSessionName='Airflow_' + self.aws_conn_id,
+                            ExternalId=external_id)
+
+                    credentials = sts_response['Credentials']
+                    aws_access_key_id = credentials['AccessKeyId']
+                    aws_secret_access_key = credentials['SecretAccessKey']
+                    aws_session_token = credentials['SessionToken']
+
+                endpoint_url = extra_config.get('host')
 
             except AirflowException:
                 # No connection found: fallback on boto3 credential strategy
@@ -150,15 +163,17 @@ def _get_credentials(self, region_name):
             aws_session_token=aws_session_token,
             region_name=region_name), endpoint_url
 
-    def get_client_type(self, client_type, region_name=None):
+    def get_client_type(self, client_type, region_name=None, config=None):
         session, endpoint_url = self._get_credentials(region_name)
 
-        return session.client(client_type, endpoint_url=endpoint_url)
+        return session.client(client_type, endpoint_url=endpoint_url,
+                              config=config, verify=self.verify)
 
-    def get_resource_type(self, resource_type, region_name=None):
+    def get_resource_type(self, resource_type, region_name=None, config=None):
         session, endpoint_url = self._get_credentials(region_name)
 
-        return session.resource(resource_type, endpoint_url=endpoint_url)
+        return session.resource(resource_type, endpoint_url=endpoint_url,
+                                config=config, verify=self.verify)
 
     def get_session(self, region_name=None):
         """Get the underlying boto3.session."""
@@ -168,10 +183,23 @@ def get_session(self, region_name=None):
     def get_credentials(self, region_name=None):
         """Get the underlying `botocore.Credentials` object.
 
-        This contains the attributes: access_key, secret_key and token.
+        This contains the following authentication attributes: access_key, secret_key and token.
         """
         session, _ = self._get_credentials(region_name)
-        # Credentials are refreshable, so accessing your access key / secret key
-        # separately can lead to a race condition.
+        # Credentials are refreshable, so accessing your access key and
+        # secret key separately can lead to a race condition.
         # See https://stackoverflow.com/a/36291428/8283373
         return session.get_credentials().get_frozen_credentials()
+
+    def expand_role(self, role):
+        """
+        If the IAM role is a role name, get the Amazon Resource Name (ARN) for the role.
+        If IAM role is already an IAM role ARN, no change is made.
+
+        :param role: IAM role name or ARN
+        :return: IAM role ARN
+        """
+        if '/' in role:
+            return role
+        else:
+            return self.get_client_type('iam').get_role(RoleName=role)['Role']['Arn']
diff --git a/airflow/contrib/hooks/aws_sns_hook.py b/airflow/contrib/hooks/aws_sns_hook.py
new file mode 100644
index 0000000000000..4308b493ce3e3
--- /dev/null
+++ b/airflow/contrib/hooks/aws_sns_hook.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+
+
+class AwsSnsHook(AwsHook):
+    """
+    Interact with Amazon Simple Notification Service.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(AwsSnsHook, self).__init__(*args, **kwargs)
+
+    def get_conn(self):
+        """
+        Get an SNS connection
+        """
+        self.conn = self.get_client_type('sns')
+        return self.conn
+
+    def publish_to_target(self, target_arn, message):
+        """
+        Publish a message to a topic or an endpoint.
+
+        :param target_arn: either a TopicArn or an EndpointArn
+        :type target_arn: str
+        :param message: the default message you want to send
+        :param message: str
+        """
+
+        conn = self.get_conn()
+
+        messages = {
+            'default': message
+        }
+
+        return conn.publish(
+            TargetArn=target_arn,
+            Message=json.dumps(messages),
+            MessageStructure='json'
+        )
diff --git a/airflow/contrib/hooks/azure_container_instance_hook.py b/airflow/contrib/hooks/azure_container_instance_hook.py
new file mode 100644
index 0000000000000..3f29c6de4af5a
--- /dev/null
+++ b/airflow/contrib/hooks/azure_container_instance_hook.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+from airflow.hooks.base_hook import BaseHook
+from airflow.exceptions import AirflowException
+
+from azure.common.client_factory import get_client_from_auth_file
+from azure.common.credentials import ServicePrincipalCredentials
+
+from azure.mgmt.containerinstance import ContainerInstanceManagementClient
+
+
+class AzureContainerInstanceHook(BaseHook):
+    """
+    A hook to communicate with Azure Container Instances.
+
+    This hook requires a service principal in order to work.
+    After creating this service principal
+    (Azure Active Directory/App Registrations), you need to fill in the
+    client_id (Application ID) as login, the generated password as password,
+    and tenantId and subscriptionId in the extra's field as a json.
+
+    :param conn_id: connection id of a service principal which will be used
+        to start the container instance
+    :type conn_id: str
+    """
+
+    def __init__(self, conn_id='azure_default'):
+        self.conn_id = conn_id
+        self.connection = self.get_conn()
+
+    def get_conn(self):
+        conn = self.get_connection(self.conn_id)
+        key_path = conn.extra_dejson.get('key_path', False)
+        if key_path:
+            if key_path.endswith('.json'):
+                self.log.info('Getting connection using a JSON key file.')
+                return get_client_from_auth_file(ContainerInstanceManagementClient,
+                                                 key_path)
+            else:
+                raise AirflowException('Unrecognised extension for key file.')
+
+        if os.environ.get('AZURE_AUTH_LOCATION'):
+            key_path = os.environ.get('AZURE_AUTH_LOCATION')
+            if key_path.endswith('.json'):
+                self.log.info('Getting connection using a JSON key file.')
+                return get_client_from_auth_file(ContainerInstanceManagementClient,
+                                                 key_path)
+            else:
+                raise AirflowException('Unrecognised extension for key file.')
+
+        credentials = ServicePrincipalCredentials(
+            client_id=conn.login,
+            secret=conn.password,
+            tenant=conn.extra_dejson['tenantId']
+        )
+
+        subscription_id = conn.extra_dejson['subscriptionId']
+        return ContainerInstanceManagementClient(credentials, str(subscription_id))
+
+    def create_or_update(self, resource_group, name, container_group):
+        """
+        Create a new container group
+
+        :param resource_group: the name of the resource group
+        :type resource_group: str
+        :param name: the name of the container group
+        :type name: str
+        :param container_group: the properties of the container group
+        :type container_group: azure.mgmt.containerinstance.models.ContainerGroup
+        """
+        self.connection.container_groups.create_or_update(resource_group,
+                                                          name,
+                                                          container_group)
+
+    def get_state_exitcode_details(self, resource_group, name):
+        """
+        Get the state and exitcode of a container group
+
+        :param resource_group: the name of the resource group
+        :type resource_group: str
+        :param name: the name of the container group
+        :type name: str
+        :return: A tuple with the state, exitcode, and details.
+            If the exitcode is unknown 0 is returned.
+        :rtype: tuple(state,exitcode,details)
+        """
+        current_state = self._get_instance_view(resource_group, name).current_state
+        return (current_state.state,
+                current_state.exit_code,
+                current_state.detail_status)
+
+    def _get_instance_view(self, resource_group, name):
+        response = self.connection.container_groups.get(resource_group,
+                                                        name,
+                                                        raw=False)
+        return response.containers[0].instance_view
+
+    def get_messages(self, resource_group, name):
+        """
+        Get the messages of a container group
+
+        :param resource_group: the name of the resource group
+        :type resource_group: str
+        :param name: the name of the container group
+        :type name: str
+        :return: A list of the event messages
+        :rtype: list[str]
+        """
+        instance_view = self._get_instance_view(resource_group, name)
+
+        return [event.message for event in instance_view.events]
+
+    def get_logs(self, resource_group, name, tail=1000):
+        """
+        Get the tail from logs of a container group
+
+        :param resource_group: the name of the resource group
+        :type resource_group: str
+        :param name: the name of the container group
+        :type name: str
+        :param tail: the size of the tail
+        :type tail: int
+        :return: A list of log messages
+        :rtype: list[str]
+        """
+        logs = self.connection.container.list_logs(resource_group, name, name, tail=tail)
+        return logs.content.splitlines(True)
+
+    def delete(self, resource_group, name):
+        """
+        Delete a container group
+
+        :param resource_group: the name of the resource group
+        :type resource_group: str
+        :param name: the name of the container group
+        :type name: str
+        """
+        self.connection.container_groups.delete(resource_group, name)
+
+    def exists(self, resource_group, name):
+        """
+        Test if a container group exists
+
+        :param resource_group: the name of the resource group
+        :type resource_group: str
+        :param name: the name of the container group
+        :type name: str
+        """
+        for container in self.connection.container_groups.list_by_resource_group(resource_group):
+            if container.name == name:
+                return True
+        return False
diff --git a/airflow/contrib/hooks/azure_container_registry_hook.py b/airflow/contrib/hooks/azure_container_registry_hook.py
new file mode 100644
index 0000000000000..af38c1a94380e
--- /dev/null
+++ b/airflow/contrib/hooks/azure_container_registry_hook.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.hooks.base_hook import BaseHook
+from azure.mgmt.containerinstance.models import ImageRegistryCredential
+
+
+class AzureContainerRegistryHook(BaseHook):
+    """
+    A hook to communicate with a Azure Container Registry.
+
+    :param conn_id: connection id of a service principal which will be used
+        to start the container instance
+    :type conn_id: str
+    """
+
+    def __init__(self, conn_id='azure_registry'):
+        self.conn_id = conn_id
+        self.connection = self.get_conn()
+
+    def get_conn(self):
+        conn = self.get_connection(self.conn_id)
+        return ImageRegistryCredential(server=conn.host, username=conn.login, password=conn.password)
diff --git a/airflow/contrib/hooks/azure_container_volume_hook.py b/airflow/contrib/hooks/azure_container_volume_hook.py
new file mode 100644
index 0000000000000..c81c5190776c6
--- /dev/null
+++ b/airflow/contrib/hooks/azure_container_volume_hook.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.hooks.base_hook import BaseHook
+from azure.mgmt.containerinstance.models import (Volume,
+                                                 AzureFileVolume)
+
+
+class AzureContainerVolumeHook(BaseHook):
+    """
+    A hook which wraps an Azure Volume.
+
+    :param wasb_conn_id: connection id of a Azure storage account of
+        which file shares should be mounted
+    :type wasb_conn_id: str
+    """
+
+    def __init__(self, wasb_conn_id='wasb_default'):
+        self.conn_id = wasb_conn_id
+
+    def get_storagekey(self):
+        conn = self.get_connection(self.conn_id)
+        service_options = conn.extra_dejson
+
+        if 'connection_string' in service_options:
+            for keyvalue in service_options['connection_string'].split(";"):
+                key, value = keyvalue.split("=", 1)
+                if key == "AccountKey":
+                    return value
+        return conn.password
+
+    def get_file_volume(self, mount_name, share_name,
+                        storage_account_name, read_only=False):
+        return Volume(name=mount_name,
+                      azure_file=AzureFileVolume(share_name=share_name,
+                                                 storage_account_name=storage_account_name,
+                                                 read_only=read_only,
+                                                 storage_account_key=self.get_storagekey()))
diff --git a/airflow/contrib/hooks/azure_cosmos_hook.py b/airflow/contrib/hooks/azure_cosmos_hook.py
new file mode 100644
index 0000000000000..01b4007b0308f
--- /dev/null
+++ b/airflow/contrib/hooks/azure_cosmos_hook.py
@@ -0,0 +1,287 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import azure.cosmos.cosmos_client as cosmos_client
+from azure.cosmos.errors import HTTPFailure
+import uuid
+
+from airflow.exceptions import AirflowBadRequest
+from airflow.hooks.base_hook import BaseHook
+
+
+class AzureCosmosDBHook(BaseHook):
+    """
+    Interacts with Azure CosmosDB.
+
+    login should be the endpoint uri, password should be the master key
+    optionally, you can use the following extras to default these values
+    {"database_name": "<DATABASE_NAME>", "collection_name": "COLLECTION_NAME"}.
+
+    :param azure_cosmos_conn_id: Reference to the Azure CosmosDB connection.
+    :type azure_cosmos_conn_id: str
+    """
+
+    def __init__(self, azure_cosmos_conn_id='azure_cosmos_default'):
+        self.conn_id = azure_cosmos_conn_id
+        self.connection = self.get_connection(self.conn_id)
+        self.extras = self.connection.extra_dejson
+
+        self.endpoint_uri = self.connection.login
+        self.master_key = self.connection.password
+        self.default_database_name = self.extras.get('database_name')
+        self.default_collection_name = self.extras.get('collection_name')
+        self.cosmos_client = None
+
+    def get_conn(self):
+        """
+        Return a cosmos db client.
+        """
+        if self.cosmos_client is not None:
+            return self.cosmos_client
+
+        # Initialize the Python Azure Cosmos DB client
+        self.cosmos_client = cosmos_client.CosmosClient(self.endpoint_uri, {'masterKey': self.master_key})
+
+        return self.cosmos_client
+
+    def __get_database_name(self, database_name=None):
+        db_name = database_name
+        if db_name is None:
+            db_name = self.default_database_name
+
+        if db_name is None:
+            raise AirflowBadRequest("Database name must be specified")
+
+        return db_name
+
+    def __get_collection_name(self, collection_name=None):
+        coll_name = collection_name
+        if coll_name is None:
+            coll_name = self.default_collection_name
+
+        if coll_name is None:
+            raise AirflowBadRequest("Collection name must be specified")
+
+        return coll_name
+
+    def does_collection_exist(self, collection_name, database_name=None):
+        """
+        Checks if a collection exists in CosmosDB.
+        """
+        if collection_name is None:
+            raise AirflowBadRequest("Collection name cannot be None.")
+
+        existing_container = list(self.get_conn().QueryContainers(
+            get_database_link(self.__get_database_name(database_name)), {
+                "query": "SELECT * FROM r WHERE r.id=@id",
+                "parameters": [
+                    {"name": "@id", "value": collection_name}
+                ]
+            }))
+        if len(existing_container) == 0:
+            return False
+
+        return True
+
+    def create_collection(self, collection_name, database_name=None):
+        """
+        Creates a new collection in the CosmosDB database.
+        """
+        if collection_name is None:
+            raise AirflowBadRequest("Collection name cannot be None.")
+
+        # We need to check to see if this container already exists so we don't try
+        # to create it twice
+        existing_container = list(self.get_conn().QueryContainers(
+            get_database_link(self.__get_database_name(database_name)), {
+                "query": "SELECT * FROM r WHERE r.id=@id",
+                "parameters": [
+                    {"name": "@id", "value": collection_name}
+                ]
+            }))
+
+        # Only create if we did not find it already existing
+        if len(existing_container) == 0:
+            self.get_conn().CreateContainer(
+                get_database_link(self.__get_database_name(database_name)),
+                {"id": collection_name})
+
+    def does_database_exist(self, database_name):
+        """
+        Checks if a database exists in CosmosDB.
+        """
+        if database_name is None:
+            raise AirflowBadRequest("Database name cannot be None.")
+
+        existing_database = list(self.get_conn().QueryDatabases({
+            "query": "SELECT * FROM r WHERE r.id=@id",
+            "parameters": [
+                {"name": "@id", "value": database_name}
+            ]
+        }))
+        if len(existing_database) == 0:
+            return False
+
+        return True
+
+    def create_database(self, database_name):
+        """
+        Creates a new database in CosmosDB.
+        """
+        if database_name is None:
+            raise AirflowBadRequest("Database name cannot be None.")
+
+        # We need to check to see if this database already exists so we don't try
+        # to create it twice
+        existing_database = list(self.get_conn().QueryDatabases({
+            "query": "SELECT * FROM r WHERE r.id=@id",
+            "parameters": [
+                {"name": "@id", "value": database_name}
+            ]
+        }))
+
+        # Only create if we did not find it already existing
+        if len(existing_database) == 0:
+            self.get_conn().CreateDatabase({"id": database_name})
+
+    def delete_database(self, database_name):
+        """
+        Deletes an existing database in CosmosDB.
+        """
+        if database_name is None:
+            raise AirflowBadRequest("Database name cannot be None.")
+
+        self.get_conn().DeleteDatabase(get_database_link(database_name))
+
+    def delete_collection(self, collection_name, database_name=None):
+        """
+        Deletes an existing collection in the CosmosDB database.
+        """
+        if collection_name is None:
+            raise AirflowBadRequest("Collection name cannot be None.")
+
+        self.get_conn().DeleteContainer(
+            get_collection_link(self.__get_database_name(database_name), collection_name))
+
+    def upsert_document(self, document, database_name=None, collection_name=None, document_id=None):
+        """
+        Inserts a new document (or updates an existing one) into an existing
+        collection in the CosmosDB database.
+        """
+        # Assign unique ID if one isn't provided
+        if document_id is None:
+            document_id = str(uuid.uuid4())
+
+        if document is None:
+            raise AirflowBadRequest("You cannot insert a None document")
+
+        # Add document id if isn't found
+        if 'id' in document:
+            if document['id'] is None:
+                document['id'] = document_id
+        else:
+            document['id'] = document_id
+
+        created_document = self.get_conn().CreateItem(
+            get_collection_link(
+                self.__get_database_name(database_name),
+                self.__get_collection_name(collection_name)),
+            document)
+
+        return created_document
+
+    def insert_documents(self, documents, database_name=None, collection_name=None):
+        """
+        Insert a list of new documents into an existing collection in the CosmosDB database.
+        """
+        if documents is None:
+            raise AirflowBadRequest("You cannot insert empty documents")
+
+        created_documents = []
+        for single_document in documents:
+            created_documents.append(
+                self.get_conn().CreateItem(
+                    get_collection_link(
+                        self.__get_database_name(database_name),
+                        self.__get_collection_name(collection_name)),
+                    single_document))
+
+        return created_documents
+
+    def delete_document(self, document_id, database_name=None, collection_name=None):
+        """
+        Delete an existing document out of a collection in the CosmosDB database.
+        """
+        if document_id is None:
+            raise AirflowBadRequest("Cannot delete a document without an id")
+
+        self.get_conn().DeleteItem(
+            get_document_link(
+                self.__get_database_name(database_name),
+                self.__get_collection_name(collection_name),
+                document_id))
+
+    def get_document(self, document_id, database_name=None, collection_name=None):
+        """
+        Get a document from an existing collection in the CosmosDB database.
+        """
+        if document_id is None:
+            raise AirflowBadRequest("Cannot get a document without an id")
+
+        try:
+            return self.get_conn().ReadItem(
+                get_document_link(
+                    self.__get_database_name(database_name),
+                    self.__get_collection_name(collection_name),
+                    document_id))
+        except HTTPFailure:
+            return None
+
+    def get_documents(self, sql_string, database_name=None, collection_name=None, partition_key=None):
+        """
+        Get a list of documents from an existing collection in the CosmosDB database via SQL query.
+        """
+        if sql_string is None:
+            raise AirflowBadRequest("SQL query string cannot be None")
+
+        # Query them in SQL
+        query = {'query': sql_string}
+
+        try:
+            result_iterable = self.get_conn().QueryItems(
+                get_collection_link(
+                    self.__get_database_name(database_name),
+                    self.__get_collection_name(collection_name)),
+                query,
+                partition_key)
+
+            return list(result_iterable)
+        except HTTPFailure:
+            return None
+
+
+def get_database_link(database_id):
+    return "dbs/" + database_id
+
+
+def get_collection_link(database_id, collection_id):
+    return get_database_link(database_id) + "/colls/" + collection_id
+
+
+def get_document_link(database_id, collection_id, document_id):
+    return get_collection_link(database_id, collection_id) + "/docs/" + document_id
diff --git a/airflow/contrib/hooks/azure_data_lake_hook.py b/airflow/contrib/hooks/azure_data_lake_hook.py
index 1a02d78f27b72..049fadbb78274 100644
--- a/airflow/contrib/hooks/azure_data_lake_hook.py
+++ b/airflow/contrib/hooks/azure_data_lake_hook.py
@@ -59,7 +59,7 @@ def check_for_file(self, file_path):
         :param file_path: Path and name of the file.
         :type file_path: str
         :return: True if the file exists, False otherwise.
-        :rtype bool
+        :rtype: bool
         """
         try:
             files = self.connection.glob(file_path, details=False, invalidate_cache=True)
@@ -77,7 +77,7 @@ def upload_file(self, local_path, remote_path, nthreads=64, overwrite=True,
             are not supported.
         :type local_path: str
         :param remote_path: Remote path to upload to; if multiple files, this is the
-            dircetory root to write within.
+            directory root to write within.
         :type remote_path: str
         :param nthreads: Number of threads to use. If None, uses the number of cores.
         :type nthreads: int
diff --git a/airflow/contrib/hooks/azure_fileshare_hook.py b/airflow/contrib/hooks/azure_fileshare_hook.py
index edabc17293606..8afa1540d78b4 100644
--- a/airflow/contrib/hooks/azure_fileshare_hook.py
+++ b/airflow/contrib/hooks/azure_fileshare_hook.py
@@ -56,7 +56,7 @@ def check_for_directory(self, share_name, directory_name, **kwargs):
             `FileService.exists()` takes.
         :type kwargs: object
         :return: True if the file exists, False otherwise.
-        :rtype bool
+        :rtype: bool
         """
         return self.connection.exists(share_name, directory_name,
                                       **kwargs)
@@ -75,7 +75,7 @@ def check_for_file(self, share_name, directory_name, file_name, **kwargs):
             `FileService.exists()` takes.
         :type kwargs: object
         :return: True if the file exists, False otherwise.
-        :rtype bool
+        :rtype: bool
         """
         return self.connection.exists(share_name, directory_name,
                                       file_name, **kwargs)
@@ -92,7 +92,7 @@ def list_directories_and_files(self, share_name, directory_name=None, **kwargs):
             `FileService.list_directories_and_files()` takes.
         :type kwargs: object
         :return: A list of files and directories
-        :rtype list
+        :rtype: list
         """
         return self.connection.list_directories_and_files(share_name,
                                                           directory_name,
@@ -100,7 +100,7 @@ def list_directories_and_files(self, share_name, directory_name=None, **kwargs):
 
     def create_directory(self, share_name, directory_name, **kwargs):
         """
-        Create a new direcotry on a Azure File Share.
+        Create a new directory on a Azure File Share.
 
         :param share_name: Name of the share.
         :type share_name: str
@@ -110,7 +110,7 @@ def create_directory(self, share_name, directory_name, **kwargs):
             `FileService.create_directory()` takes.
         :type kwargs: object
         :return: A list of files and directories
-        :rtype list
+        :rtype: list
         """
         return self.connection.create_directory(share_name, directory_name, **kwargs)
 
diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py
index b452f51c22c69..5744e3b1756a4 100644
--- a/airflow/contrib/hooks/bigquery_hook.py
+++ b/airflow/contrib/hooks/bigquery_hook.py
@@ -23,7 +23,10 @@
 """
 
 import time
+import six
 from builtins import range
+from copy import deepcopy
+from six import iteritems
 
 from past.builtins import basestring
 
@@ -31,8 +34,8 @@
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 from airflow.hooks.dbapi_hook import DbApiHook
 from airflow.utils.log.logging_mixin import LoggingMixin
-from apiclient.discovery import HttpError, build
-from googleapiclient import errors
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 from pandas_gbq.gbq import \
     _check_google_client_version as gbq_check_google_client_version
 from pandas_gbq import read_gbq
@@ -41,7 +44,7 @@
 from pandas_gbq.gbq import GbqConnector
 
 
-class BigQueryHook(GoogleCloudBaseHook, DbApiHook, LoggingMixin):
+class BigQueryHook(GoogleCloudBaseHook, DbApiHook):
     """
     Interact with BigQuery. This hook uses the Google Cloud Platform
     connection.
@@ -51,10 +54,12 @@ class BigQueryHook(GoogleCloudBaseHook, DbApiHook, LoggingMixin):
     def __init__(self,
                  bigquery_conn_id='bigquery_default',
                  delegate_to=None,
-                 use_legacy_sql=True):
+                 use_legacy_sql=True,
+                 location=None):
         super(BigQueryHook, self).__init__(
             gcp_conn_id=bigquery_conn_id, delegate_to=delegate_to)
         self.use_legacy_sql = use_legacy_sql
+        self.location = location
 
     def get_conn(self):
         """
@@ -65,7 +70,9 @@ def get_conn(self):
         return BigQueryConnection(
             service=service,
             project_id=project,
-            use_legacy_sql=self.use_legacy_sql)
+            use_legacy_sql=self.use_legacy_sql,
+            location=self.location,
+        )
 
     def get_service(self):
         """
@@ -93,13 +100,13 @@ def get_pandas_df(self, sql, parameters=None, dialect=None):
         https://github.com/pydata/pandas/issues/6900
 
         :param sql: The BigQuery SQL to execute.
-        :type sql: string
+        :type sql: str
         :param parameters: The parameters to render the SQL query with (not
             used, leave to override superclass method)
         :type parameters: mapping or iterable
         :param dialect: Dialect of BigQuery SQL – legacy SQL or standard SQL
             defaults to use `self.use_legacy_sql` if not specified
-        :type dialect: string in {'legacy', 'standard'}
+        :type dialect: str in {'legacy', 'standard'}
         """
         if dialect is None:
             dialect = 'legacy' if self.use_legacy_sql else 'standard'
@@ -116,12 +123,12 @@ def table_exists(self, project_id, dataset_id, table_id):
         :param project_id: The Google cloud project in which to look for the
             table. The connection supplied to the hook must provide access to
             the specified project.
-        :type project_id: string
+        :type project_id: str
         :param dataset_id: The name of the dataset in which to look for the
             table.
-        :type dataset_id: string
+        :type dataset_id: str
         :param table_id: The name of the table to check the existence of.
-        :type table_id: string
+        :type table_id: str
         """
         service = self.get_service()
         try:
@@ -129,7 +136,7 @@ def table_exists(self, project_id, dataset_id, table_id):
                 projectId=project_id, datasetId=dataset_id,
                 tableId=table_id).execute()
             return True
-        except errors.HttpError as e:
+        except HttpError as e:
             if e.resp['status'] == '404':
                 return False
             raise
@@ -195,21 +202,34 @@ class BigQueryBaseCursor(LoggingMixin):
     PEP 249 cursor isn't needed.
     """
 
-    def __init__(self, service, project_id, use_legacy_sql=True):
+    def __init__(self,
+                 service,
+                 project_id,
+                 use_legacy_sql=True,
+                 api_resource_configs=None,
+                 location=None):
+
         self.service = service
         self.project_id = project_id
         self.use_legacy_sql = use_legacy_sql
+        if api_resource_configs:
+            _validate_value("api_resource_configs", api_resource_configs, dict)
+        self.api_resource_configs = api_resource_configs \
+            if api_resource_configs else {}
         self.running_job_id = None
+        self.location = location
 
     def create_empty_table(self,
                            project_id,
                            dataset_id,
                            table_id,
                            schema_fields=None,
-                           time_partitioning={}
-                           ):
+                           time_partitioning=None,
+                           labels=None,
+                           view=None):
         """
         Creates a new, empty table in the dataset.
+        To create a view, which is defined by a SQL query, parse a dictionary to 'view' kwarg
 
         :param project_id: The project to create the table into.
         :type project_id: str
@@ -218,23 +238,37 @@ def create_empty_table(self,
         :param table_id: The Name of the table to be created.
         :type table_id: str
         :param schema_fields: If set, the schema field list as defined here:
-        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema
+        :type schema_fields: list
+        :param labels: a dictionary containing labels for the table, passed to BigQuery
+        :type labels: dict
 
         **Example**: ::
 
             schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
                            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]
 
-        :type schema_fields: list
         :param time_partitioning: configure optional time partitioning fields i.e.
             partition by field, type and expiration as per API specifications.
 
             .. seealso::
-            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning
+                https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning
         :type time_partitioning: dict
+        :param view: [Optional] A dictionary containing definition for the view.
+            If set, it will create a view instead of a table:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view
+        :type view: dict
 
-        :return:
+        **Example**: ::
+
+            view = {
+                "query": "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 1000",
+                "useLegacySql": False
+            }
+
+        :return: None
         """
+
         project_id = project_id if project_id is not None else self.project_id
 
         table_resource = {
@@ -249,6 +283,12 @@ def create_empty_table(self,
         if time_partitioning:
             table_resource['timePartitioning'] = time_partitioning
 
+        if labels:
+            table_resource['labels'] = labels
+
+        if view:
+            table_resource['view'] = view
+
         self.log.info('Creating Table %s:%s.%s',
                       project_id, dataset_id, table_id)
 
@@ -280,7 +320,8 @@ def create_external_table(self,
                               quote_character=None,
                               allow_quoted_newlines=False,
                               allow_jagged_rows=False,
-                              src_fmt_configs={}
+                              src_fmt_configs=None,
+                              labels=None
                               ):
         """
         Creates a new external table in the dataset with the data in Google
@@ -291,11 +332,11 @@ def create_external_table(self,
         for more details about these parameters.
 
         :param external_project_dataset_table:
-            The dotted (<project>.|<project>:)<dataset>.<table>($<partition>) BigQuery
+            The dotted ``(<project>.|<project>:)<dataset>.<table>($<partition>)`` BigQuery
             table name to create external table.
-            If <project> is not included, project will be the
+            If ``<project>`` is not included, project will be the
             project defined in the connection json.
-        :type external_project_dataset_table: string
+        :type external_project_dataset_table: str
         :param schema_fields: The schema field list as defined here:
             https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#resource
         :type schema_fields: list
@@ -304,7 +345,7 @@ def create_external_table(self,
             per-object name can be used.
         :type source_uris: list
         :param source_format: File format to export.
-        :type source_format: string
+        :type source_format: str
         :param autodetect: Try to detect schema and format options automatically.
             Any option specified explicitly will be honored.
         :type autodetect: bool
@@ -312,8 +353,8 @@ def create_external_table(self,
             Possible values include GZIP and NONE.
             The default value is NONE.
             This setting is ignored for Google Cloud Bigtable,
-                Google Cloud Datastore backups and Avro formats.
-        :type compression: string
+            Google Cloud Datastore backups and Avro formats.
+        :type compression: str
         :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow
             extra values that are not represented in the table schema.
             If true, the extra values are ignored. If false, records with extra columns
@@ -326,13 +367,13 @@ def create_external_table(self,
         :param skip_leading_rows: Number of rows to skip when loading from a CSV.
         :type skip_leading_rows: int
         :param field_delimiter: The delimiter to use when loading from a CSV.
-        :type field_delimiter: string
+        :type field_delimiter: str
         :param quote_character: The value that is used to quote data sections in a CSV
             file.
-        :type quote_character: string
+        :type quote_character: str
         :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not
             (false).
-        :type allow_quoted_newlines: boolean
+        :type allow_quoted_newlines: bool
         :param allow_jagged_rows: Accept rows that are missing trailing optional columns.
             The missing values are treated as nulls. If false, records with missing
             trailing columns are treated as bad records, and if there are too many bad
@@ -341,8 +382,12 @@ def create_external_table(self,
         :type allow_jagged_rows: bool
         :param src_fmt_configs: configure optional fields specific to the source format
         :type src_fmt_configs: dict
+        :param labels: a dictionary containing labels for the table, passed to BigQuery
+        :type labels: dict
         """
 
+        if src_fmt_configs is None:
+            src_fmt_configs = {}
         project_id, dataset_id, external_table_id = \
             _split_tablename(table_input=external_project_dataset_table,
                              default_project_id=self.project_id,
@@ -439,6 +484,9 @@ def create_external_table(self,
             table_resource['externalDataConfiguration'][src_fmt_to_param_mapping[
                 source_format]] = src_fmt_configs
 
+        if labels:
+            table_resource['labels'] = labels
+
         try:
             self.service.tables().insert(
                 projectId=project_id,
@@ -454,22 +502,134 @@ def create_external_table(self,
                 'BigQuery job failed. Error was: {}'.format(err.content)
             )
 
+    def patch_table(self,
+                    dataset_id,
+                    table_id,
+                    project_id=None,
+                    description=None,
+                    expiration_time=None,
+                    external_data_configuration=None,
+                    friendly_name=None,
+                    labels=None,
+                    schema=None,
+                    time_partitioning=None,
+                    view=None,
+                    require_partition_filter=None):
+        """
+        Patch information in an existing table.
+        It only updates fileds that are provided in the request object.
+
+        Reference: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/patch
+
+        :param dataset_id: The dataset containing the table to be patched.
+        :type dataset_id: str
+        :param table_id: The Name of the table to be patched.
+        :type table_id: str
+        :param project_id: The project containing the table to be patched.
+        :type project_id: str
+        :param description: [Optional] A user-friendly description of this table.
+        :type description: str
+        :param expiration_time: [Optional] The time when this table expires,
+            in milliseconds since the epoch.
+        :type expiration_time: int
+        :param external_data_configuration: [Optional] A dictionary containing
+            properties of a table stored outside of BigQuery.
+        :type external_data_configuration: dict
+        :param friendly_name: [Optional] A descriptive name for this table.
+        :type friendly_name: str
+        :param labels: [Optional] A dictionary containing labels associated with this table.
+        :type labels: dict
+        :param schema: [Optional] If set, the schema field list as defined here:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema
+            The supported schema modifications and unsupported schema modification are listed here:
+            https://cloud.google.com/bigquery/docs/managing-table-schemas
+            **Example**: ::
+
+                schema=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+                               {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}]
+
+        :type schema: list
+        :param time_partitioning: [Optional] A dictionary containing time-based partitioning
+             definition for the table.
+        :type time_partitioning: dict
+        :param view: [Optional] A dictionary containing definition for the view.
+            If set, it will patch a view instead of a table:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view
+            **Example**: ::
+
+                view = {
+                    "query": "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 500",
+                    "useLegacySql": False
+                }
+
+        :type view: dict
+        :param require_partition_filter: [Optional] If true, queries over the this table require a
+            partition filter. If false, queries over the table
+        :type require_partition_filter: bool
+
+        """
+
+        project_id = project_id if project_id is not None else self.project_id
+
+        table_resource = {}
+
+        if description is not None:
+            table_resource['description'] = description
+        if expiration_time is not None:
+            table_resource['expirationTime'] = expiration_time
+        if external_data_configuration:
+            table_resource['externalDataConfiguration'] = external_data_configuration
+        if friendly_name is not None:
+            table_resource['friendlyName'] = friendly_name
+        if labels:
+            table_resource['labels'] = labels
+        if schema:
+            table_resource['schema'] = {'fields': schema}
+        if time_partitioning:
+            table_resource['timePartitioning'] = time_partitioning
+        if view:
+            table_resource['view'] = view
+        if require_partition_filter is not None:
+            table_resource['requirePartitionFilter'] = require_partition_filter
+
+        self.log.info('Patching Table %s:%s.%s',
+                      project_id, dataset_id, table_id)
+
+        try:
+            self.service.tables().patch(
+                projectId=project_id,
+                datasetId=dataset_id,
+                tableId=table_id,
+                body=table_resource).execute()
+
+            self.log.info('Table patched successfully: %s:%s.%s',
+                          project_id, dataset_id, table_id)
+
+        except HttpError as err:
+            raise AirflowException(
+                'BigQuery job failed. Error was: {}'.format(err.content)
+            )
+
     def run_query(self,
                   bql=None,
                   sql=None,
-                  destination_dataset_table=False,
+                  destination_dataset_table=None,
                   write_disposition='WRITE_EMPTY',
                   allow_large_results=False,
-                  flatten_results=False,
-                  udf_config=False,
+                  flatten_results=None,
+                  udf_config=None,
                   use_legacy_sql=None,
                   maximum_billing_tier=None,
                   maximum_bytes_billed=None,
                   create_disposition='CREATE_IF_NEEDED',
                   query_params=None,
+                  labels=None,
                   schema_update_options=(),
                   priority='INTERACTIVE',
-                  time_partitioning={}):
+                  time_partitioning=None,
+                  api_resource_configs=None,
+                  cluster_fields=None,
+                  location=None):
         """
         Executes a BigQuery SQL query. Optionally persists results in a BigQuery
         table. See here:
@@ -480,31 +640,38 @@ def run_query(self,
 
         :param bql: (Deprecated. Use `sql` parameter instead) The BigQuery SQL
             to execute.
-        :type bql: string
+        :type bql: str
         :param sql: The BigQuery SQL to execute.
-        :type sql: string
-        :param destination_dataset_table: The dotted <dataset>.<table>
+        :type sql: str
+        :param destination_dataset_table: The dotted ``<dataset>.<table>``
             BigQuery table to save the query results.
-        :type destination_dataset_table: string
+        :type destination_dataset_table: str
         :param write_disposition: What to do if the table already exists in
             BigQuery.
-        :type write_disposition: string
+        :type write_disposition: str
         :param allow_large_results: Whether to allow large results.
-        :type allow_large_results: boolean
+        :type allow_large_results: bool
         :param flatten_results: If true and query uses legacy SQL dialect, flattens
             all nested and repeated fields in the query results. ``allowLargeResults``
             must be true if this is set to false. For standard SQL queries, this
             flag is ignored and results are never flattened.
-        :type flatten_results: boolean
+        :type flatten_results: bool
         :param udf_config: The User Defined Function configuration for the query.
             See https://cloud.google.com/bigquery/user-defined-functions for details.
+        :type udf_config: list
         :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).
             If `None`, defaults to `self.use_legacy_sql`.
-        :type use_legacy_sql: boolean
-        :type udf_config: list
+        :type use_legacy_sql: bool
+        :param api_resource_configs: a dictionary that contain params
+            'configuration' applied for Google BigQuery Jobs API:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs
+            for example, {'query': {'useQueryCache': False}}. You could use it
+            if you need to provide some params that are not supported by the
+            BigQueryHook like args.
+        :type api_resource_configs: dict
         :param maximum_billing_tier: Positive integer that serves as a
             multiplier of the basic price.
-        :type maximum_billing_tier: integer
+        :type maximum_billing_tier: int
         :param maximum_bytes_billed: Limits the bytes billed for this job.
             Queries that will have bytes billed beyond this limit will fail
             (without incurring a charge). If unspecified, this will be
@@ -512,28 +679,55 @@ def run_query(self,
         :type maximum_bytes_billed: float
         :param create_disposition: Specifies whether the job is allowed to
             create new tables.
-        :type create_disposition: string
-        :param query_params a dictionary containing query parameter types and
+        :type create_disposition: str
+        :param query_params: a list of dictionary containing query parameter types and
             values, passed to BigQuery
-        :type query_params: dict
-        :param schema_update_options: Allows the schema of the desitination
+        :type query_params: list
+        :param labels: a dictionary containing labels for the job/query,
+            passed to BigQuery
+        :type labels: dict
+        :param schema_update_options: Allows the schema of the destination
             table to be updated as a side effect of the query job.
         :type schema_update_options: tuple
         :param priority: Specifies a priority for the query.
             Possible values include INTERACTIVE and BATCH.
             The default value is INTERACTIVE.
-        :type priority: string
+        :type priority: str
         :param time_partitioning: configure optional time partitioning fields i.e.
-            partition by field, type and
-            expiration as per API specifications. Note that 'field' is not available in
-            conjunction with dataset.table$partition.
+            partition by field, type and expiration as per API specifications.
         :type time_partitioning: dict
-
+        :param cluster_fields: Request that the result of this query be stored sorted
+            by one or more columns. This is only available in combination with
+            time_partitioning. The order of columns given determines the sort order.
+        :type cluster_fields: list[str]
+        :param location: The geographic location of the job. Required except for
+            US and EU. See details at
+            https://cloud.google.com/bigquery/docs/locations#specifying_your_location
+        :type location: str
         """
 
-        # TODO remove `bql` in Airflow 2.0 - Jira: [AIRFLOW-2513]
+        if time_partitioning is None:
+            time_partitioning = {}
+
+        if location:
+            self.location = location
+
+        if not api_resource_configs:
+            api_resource_configs = self.api_resource_configs
+        else:
+            _validate_value('api_resource_configs',
+                            api_resource_configs, dict)
+        configuration = deepcopy(api_resource_configs)
+        if 'query' not in configuration:
+            configuration['query'] = {}
+
+        else:
+            _validate_value("api_resource_configs['query']",
+                            configuration['query'], dict)
+
         sql = bql if sql is None else sql
 
+        # TODO remove `bql` in Airflow 2.0 - Jira: [AIRFLOW-2513]
         if bql:
             import warnings
             warnings.warn('Deprecated parameter `bql` used in '
@@ -544,88 +738,113 @@ def run_query(self,
                           'Airflow.',
                           category=DeprecationWarning)
 
-        if sql is None:
-            raise TypeError('`BigQueryBaseCursor.run_query` missing 1 required '
-                            'positional argument: `sql`')
+        if sql is None and not configuration['query'].get('query', None):
+            raise TypeError('`BigQueryBaseCursor.run_query` '
+                            'missing 1 required positional argument: `sql`')
 
         # BigQuery also allows you to define how you want a table's schema to change
         # as a side effect of a query job
         # for more details:
         #   https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.schemaUpdateOptions
+
         allowed_schema_update_options = [
             'ALLOW_FIELD_ADDITION', "ALLOW_FIELD_RELAXATION"
         ]
-        if not set(allowed_schema_update_options).issuperset(
-                set(schema_update_options)):
-            raise ValueError(
-                "{0} contains invalid schema update options. "
-                "Please only use one or more of the following options: {1}"
-                .format(schema_update_options, allowed_schema_update_options))
 
-        if use_legacy_sql is None:
-            use_legacy_sql = self.use_legacy_sql
+        if not set(allowed_schema_update_options
+                   ).issuperset(set(schema_update_options)):
+            raise ValueError("{0} contains invalid schema update options. "
+                             "Please only use one or more of the following "
+                             "options: {1}"
+                             .format(schema_update_options,
+                                     allowed_schema_update_options))
 
-        configuration = {
-            'query': {
-                'query': sql,
-                'useLegacySql': use_legacy_sql,
-                'maximumBillingTier': maximum_billing_tier,
-                'maximumBytesBilled': maximum_bytes_billed,
-                'priority': priority
-            }
-        }
+        if schema_update_options:
+            if write_disposition not in ["WRITE_APPEND", "WRITE_TRUNCATE"]:
+                raise ValueError("schema_update_options is only "
+                                 "allowed if write_disposition is "
+                                 "'WRITE_APPEND' or 'WRITE_TRUNCATE'.")
 
         if destination_dataset_table:
-            assert '.' in destination_dataset_table, (
-                'Expected destination_dataset_table in the format of '
-                '<dataset>.<table>. Got: {}').format(destination_dataset_table)
             destination_project, destination_dataset, destination_table = \
                 _split_tablename(table_input=destination_dataset_table,
                                  default_project_id=self.project_id)
-            configuration['query'].update({
-                'allowLargeResults': allow_large_results,
-                'flattenResults': flatten_results,
-                'writeDisposition': write_disposition,
-                'createDisposition': create_disposition,
-                'destinationTable': {
-                    'projectId': destination_project,
-                    'datasetId': destination_dataset,
-                    'tableId': destination_table,
-                }
-            })
-        if udf_config:
-            assert isinstance(udf_config, list)
-            configuration['query'].update({
-                'userDefinedFunctionResources': udf_config
-            })
 
-        if query_params:
-            if self.use_legacy_sql:
-                raise ValueError("Query paramaters are not allowed when using "
-                                 "legacy SQL")
-            else:
-                configuration['query']['queryParameters'] = query_params
+            destination_dataset_table = {
+                'projectId': destination_project,
+                'datasetId': destination_dataset,
+                'tableId': destination_table,
+            }
 
-        time_partitioning = _cleanse_time_partitioning(
-            destination_dataset_table,
-            time_partitioning
-        )
-        if time_partitioning:
-            configuration['query'].update({
-                'timePartitioning': time_partitioning
-            })
+        if cluster_fields:
+            cluster_fields = {'fields': cluster_fields}
+
+        query_param_list = [
+            (sql, 'query', None, six.string_types),
+            (priority, 'priority', 'INTERACTIVE', six.string_types),
+            (use_legacy_sql, 'useLegacySql', self.use_legacy_sql, bool),
+            (query_params, 'queryParameters', None, list),
+            (udf_config, 'userDefinedFunctionResources', None, list),
+            (maximum_billing_tier, 'maximumBillingTier', None, int),
+            (maximum_bytes_billed, 'maximumBytesBilled', None, float),
+            (time_partitioning, 'timePartitioning', {}, dict),
+            (schema_update_options, 'schemaUpdateOptions', None, tuple),
+            (destination_dataset_table, 'destinationTable', None, dict),
+            (cluster_fields, 'clustering', None, dict),
+        ]
 
-        if schema_update_options:
-            if write_disposition not in ["WRITE_APPEND", "WRITE_TRUNCATE"]:
-                raise ValueError("schema_update_options is only "
-                                 "allowed if write_disposition is "
-                                 "'WRITE_APPEND' or 'WRITE_TRUNCATE'.")
-            else:
-                self.log.info(
-                    "Adding experimental "
-                    "'schemaUpdateOptions': {0}".format(schema_update_options))
-                configuration['query'][
-                    'schemaUpdateOptions'] = schema_update_options
+        for param_tuple in query_param_list:
+
+            param, param_name, param_default, param_type = param_tuple
+
+            if param_name not in configuration['query'] and param in [None, {}, ()]:
+                if param_name == 'timePartitioning':
+                    param_default = _cleanse_time_partitioning(
+                        destination_dataset_table, time_partitioning)
+                param = param_default
+
+            if param not in [None, {}, ()]:
+                _api_resource_configs_duplication_check(
+                    param_name, param, configuration['query'])
+
+                configuration['query'][param_name] = param
+
+                # check valid type of provided param,
+                # it last step because we can get param from 2 sources,
+                # and first of all need to find it
+
+                _validate_value(param_name, configuration['query'][param_name],
+                                param_type)
+
+                if param_name == 'schemaUpdateOptions' and param:
+                    self.log.info("Adding experimental 'schemaUpdateOptions': "
+                                  "%s", schema_update_options)
+
+                if param_name == 'destinationTable':
+                    for key in ['projectId', 'datasetId', 'tableId']:
+                        if key not in configuration['query']['destinationTable']:
+                            raise ValueError(
+                                "Not correct 'destinationTable' in "
+                                "api_resource_configs. 'destinationTable' "
+                                "must be a dict with {'projectId':'', "
+                                "'datasetId':'', 'tableId':''}")
+
+                    configuration['query'].update({
+                        'allowLargeResults': allow_large_results,
+                        'flattenResults': flatten_results,
+                        'writeDisposition': write_disposition,
+                        'createDisposition': create_disposition,
+                    })
+
+        if 'useLegacySql' in configuration['query'] and configuration['query']['useLegacySql'] and\
+                'queryParameters' in configuration['query']:
+            raise ValueError("Query parameters are not allowed "
+                             "when using legacy SQL")
+
+        if labels:
+            _api_resource_configs_duplication_check(
+                'labels', labels, configuration)
+            configuration['labels'] = labels
 
         return self.run_with_configuration(configuration)
 
@@ -636,7 +855,8 @@ def run_extract(  # noqa
             compression='NONE',
             export_format='CSV',
             field_delimiter=',',
-            print_header=True):
+            print_header=True,
+            labels=None):
         """
         Executes a BigQuery extract command to copy data from BigQuery to
         Google Cloud Storage. See here:
@@ -645,22 +865,25 @@ def run_extract(  # noqa
 
         For more details about these parameters.
 
-        :param source_project_dataset_table: The dotted <dataset>.<table>
+        :param source_project_dataset_table: The dotted ``<dataset>.<table>``
             BigQuery table to use as the source data.
-        :type source_project_dataset_table: string
+        :type source_project_dataset_table: str
         :param destination_cloud_storage_uris: The destination Google Cloud
             Storage URI (e.g. gs://some-bucket/some-file.txt). Follows
             convention defined here:
             https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
         :type destination_cloud_storage_uris: list
         :param compression: Type of compression to use.
-        :type compression: string
+        :type compression: str
         :param export_format: File format to export.
-        :type export_format: string
+        :type export_format: str
         :param field_delimiter: The delimiter to use when extracting to a CSV.
-        :type field_delimiter: string
+        :type field_delimiter: str
         :param print_header: Whether to print a header for a CSV file extract.
-        :type print_header: boolean
+        :type print_header: bool
+        :param labels: a dictionary containing labels for the job/query,
+            passed to BigQuery
+        :type labels: dict
         """
 
         source_project, source_dataset, source_table = \
@@ -681,6 +904,9 @@ def run_extract(  # noqa
             }
         }
 
+        if labels:
+            configuration['labels'] = labels
+
         if export_format == 'CSV':
             # Only set fieldDelimiter and printHeader fields if using CSV.
             # Google does not like it if you set these fields for other export
@@ -694,7 +920,8 @@ def run_copy(self,
                  source_project_dataset_tables,
                  destination_project_dataset_table,
                  write_disposition='WRITE_EMPTY',
-                 create_disposition='CREATE_IF_NEEDED'):
+                 create_disposition='CREATE_IF_NEEDED',
+                 labels=None):
         """
         Executes a BigQuery copy command to copy data from one BigQuery table
         to another. See here:
@@ -704,19 +931,22 @@ def run_copy(self,
         For more details about these parameters.
 
         :param source_project_dataset_tables: One or more dotted
-            (project:|project.)<dataset>.<table>
+            ``(project:|project.)<dataset>.<table>``
             BigQuery tables to use as the source data. Use a list if there are
             multiple source tables.
-            If <project> is not included, project will be the project defined
+            If ``<project>`` is not included, project will be the project defined
             in the connection json.
         :type source_project_dataset_tables: list|string
         :param destination_project_dataset_table: The destination BigQuery
-            table. Format is: (project:|project.)<dataset>.<table>
-        :type destination_project_dataset_table: string
+            table. Format is: ``(project:|project.)<dataset>.<table>``
+        :type destination_project_dataset_table: str
         :param write_disposition: The write disposition if the table already exists.
-        :type write_disposition: string
+        :type write_disposition: str
         :param create_disposition: The create disposition if the table doesn't exist.
-        :type create_disposition: string
+        :type create_disposition: str
+        :param labels: a dictionary containing labels for the job/query,
+            passed to BigQuery
+        :type labels: dict
         """
         source_project_dataset_tables = ([
             source_project_dataset_tables
@@ -754,12 +984,15 @@ def run_copy(self,
             }
         }
 
+        if labels:
+            configuration['labels'] = labels
+
         return self.run_with_configuration(configuration)
 
     def run_load(self,
                  destination_project_dataset_table,
-                 schema_fields,
                  source_uris,
+                 schema_fields=None,
                  source_format='CSV',
                  create_disposition='CREATE_IF_NEEDED',
                  skip_leading_rows=0,
@@ -771,8 +1004,10 @@ def run_load(self,
                  allow_quoted_newlines=False,
                  allow_jagged_rows=False,
                  schema_update_options=(),
-                 src_fmt_configs={},
-                 time_partitioning={}):
+                 src_fmt_configs=None,
+                 time_partitioning=None,
+                 cluster_fields=None,
+                 autodetect=False):
         """
         Executes a BigQuery load command to load data from Google Cloud Storage
         to BigQuery. See here:
@@ -782,35 +1017,39 @@ def run_load(self,
         For more details about these parameters.
 
         :param destination_project_dataset_table:
-            The dotted (<project>.|<project>:)<dataset>.<table>($<partition>) BigQuery
-            table to load data into. If <project> is not included, project will be the
+            The dotted ``(<project>.|<project>:)<dataset>.<table>($<partition>)`` BigQuery
+            table to load data into. If ``<project>`` is not included, project will be the
             project defined in the connection json. If a partition is specified the
             operator will automatically append the data, create a new partition or create
             a new DAY partitioned table.
-        :type destination_project_dataset_table: string
+        :type destination_project_dataset_table: str
         :param schema_fields: The schema field list as defined here:
             https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load
+            Required if autodetect=False; optional if autodetect=True.
         :type schema_fields: list
+        :param autodetect: Attempt to autodetect the schema for CSV and JSON
+            source files.
+        :type autodetect: bool
         :param source_uris: The source Google Cloud
             Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild
             per-object name can be used.
         :type source_uris: list
         :param source_format: File format to export.
-        :type source_format: string
+        :type source_format: str
         :param create_disposition: The create disposition if the table doesn't exist.
-        :type create_disposition: string
+        :type create_disposition: str
         :param skip_leading_rows: Number of rows to skip when loading from a CSV.
         :type skip_leading_rows: int
         :param write_disposition: The write disposition if the table already exists.
-        :type write_disposition: string
+        :type write_disposition: str
         :param field_delimiter: The delimiter to use when loading from a CSV.
-        :type field_delimiter: string
+        :type field_delimiter: str
         :param max_bad_records: The maximum number of bad records that BigQuery can
             ignore when running the job.
         :type max_bad_records: int
         :param quote_character: The value that is used to quote data sections in a CSV
             file.
-        :type quote_character: string
+        :type quote_character: str
         :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow
             extra values that are not represented in the table schema.
             If true, the extra values are ignored. If false, records with extra columns
@@ -819,23 +1058,25 @@ def run_load(self,
         :type ignore_unknown_values: bool
         :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not
             (false).
-        :type allow_quoted_newlines: boolean
+        :type allow_quoted_newlines: bool
         :param allow_jagged_rows: Accept rows that are missing trailing optional columns.
             The missing values are treated as nulls. If false, records with missing
             trailing columns are treated as bad records, and if there are too many bad
             records, an invalid error is returned in the job result. Only applicable when
             soure_format is CSV.
         :type allow_jagged_rows: bool
-        :param schema_update_options: Allows the schema of the desitination
+        :param schema_update_options: Allows the schema of the destination
             table to be updated as a side effect of the load job.
         :type schema_update_options: tuple
         :param src_fmt_configs: configure optional fields specific to the source format
         :type src_fmt_configs: dict
         :param time_partitioning: configure optional time partitioning fields i.e.
-            partition by field, type and
-            expiration as per API specifications. Note that 'field' is not available in
-            conjunction with dataset.table$partition.
+            partition by field, type and  expiration as per API specifications.
         :type time_partitioning: dict
+        :param cluster_fields: Request that the result of this load be stored sorted
+            by one or more columns. This is only available in combination with
+            time_partitioning. The order of columns given determines the sort order.
+        :type cluster_fields: list[str]
         """
 
         # bigquery only allows certain source formats
@@ -843,6 +1084,14 @@ def run_load(self,
         # if it's not, we raise a ValueError
         # Refer to this link for more details:
         #   https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.tableDefinitions.(key).sourceFormat
+
+        if schema_fields is None and not autodetect:
+            raise ValueError(
+                'You must either pass a schema or autodetect=True.')
+
+        if src_fmt_configs is None:
+            src_fmt_configs = {}
+
         source_format = source_format.upper()
         allowed_formats = [
             "CSV", "NEWLINE_DELIMITED_JSON", "AVRO", "GOOGLE_SHEETS",
@@ -863,7 +1112,7 @@ def run_load(self,
         if not set(allowed_schema_update_options).issuperset(
                 set(schema_update_options)):
             raise ValueError(
-                "{0} contains invalid schema update options. "
+                "{0} contains invalid schema update options."
                 "Please only use one or more of the following options: {1}"
                 .format(schema_update_options, allowed_schema_update_options))
 
@@ -874,6 +1123,7 @@ def run_load(self,
 
         configuration = {
             'load': {
+                'autodetect': autodetect,
                 'createDisposition': create_disposition,
                 'destinationTable': {
                     'projectId': destination_project,
@@ -896,6 +1146,9 @@ def run_load(self,
                 'timePartitioning': time_partitioning
             })
 
+        if cluster_fields:
+            configuration['load'].update({'clustering': {'fields': cluster_fields}})
+
         if schema_fields:
             configuration['load']['schema'] = {'fields': schema_fields}
 
@@ -906,8 +1159,9 @@ def run_load(self,
                                  "'WRITE_APPEND' or 'WRITE_TRUNCATE'.")
             else:
                 self.log.info(
-                    "Adding experimental "
-                    "'schemaUpdateOptions': {0}".format(schema_update_options))
+                    "Adding experimental 'schemaUpdateOptions': %s",
+                    schema_update_options
+                )
                 configuration['load'][
                     'schemaUpdateOptions'] = schema_update_options
 
@@ -936,7 +1190,7 @@ def run_load(self,
             'DATASTORE_BACKUP': ['projectionFields'],
             'NEWLINE_DELIMITED_JSON': ['autodetect', 'ignoreUnknownValues'],
             'PARQUET': ['autodetect', 'ignoreUnknownValues'],
-            'AVRO': [],
+            'AVRO': ['useAvroLogicalTypes'],
         }
         valid_configs = src_fmt_to_configs_mapping[source_format]
         src_fmt_configs = {
@@ -974,12 +1228,18 @@ def run_with_configuration(self, configuration):
 
         # Wait for query to finish.
         keep_polling_job = True
-        while (keep_polling_job):
+        while keep_polling_job:
             try:
-                job = jobs.get(
-                    projectId=self.project_id,
-                    jobId=self.running_job_id).execute()
-                if (job['status']['state'] == 'DONE'):
+                if self.location:
+                    job = jobs.get(
+                        projectId=self.project_id,
+                        jobId=self.running_job_id,
+                        location=self.location).execute()
+                else:
+                    job = jobs.get(
+                        projectId=self.project_id,
+                        jobId=self.running_job_id).execute()
+                if job['status']['state'] == 'DONE':
                     keep_polling_job = False
                     # Check if job had errors.
                     if 'errorResult' in job['status']:
@@ -999,16 +1259,22 @@ def run_with_configuration(self, configuration):
                     time.sleep(5)
                 else:
                     raise Exception(
-                        'BigQuery job status check failed. Final error was: %s',
-                        err.resp.status)
+                        'BigQuery job status check failed. Final error was: {}'.
+                        format(err.resp.status))
 
         return self.running_job_id
 
     def poll_job_complete(self, job_id):
         jobs = self.service.jobs()
         try:
-            job = jobs.get(projectId=self.project_id, jobId=job_id).execute()
-            if (job['status']['state'] == 'DONE'):
+            if self.location:
+                job = jobs.get(projectId=self.project_id,
+                               jobId=job_id,
+                               location=self.location).execute()
+            else:
+                job = jobs.get(projectId=self.project_id,
+                               jobId=job_id).execute()
+            if job['status']['state'] == 'DONE':
                 return True
         except HttpError as err:
             if err.resp.status in [500, 503]:
@@ -1017,8 +1283,8 @@ def poll_job_complete(self, job_id):
                     err.resp.status, job_id)
             else:
                 raise Exception(
-                    'BigQuery job status check failed. Final error was: %s',
-                    err.resp.status)
+                    'BigQuery job status check failed. Final error was: {}'.
+                    format(err.resp.status))
         return False
 
     def cancel_query(self):
@@ -1030,9 +1296,15 @@ def cancel_query(self):
                 not self.poll_job_complete(self.running_job_id)):
             self.log.info('Attempting to cancel job : %s, %s', self.project_id,
                           self.running_job_id)
-            jobs.cancel(
-                projectId=self.project_id,
-                jobId=self.running_job_id).execute()
+            if self.location:
+                jobs.cancel(
+                    projectId=self.project_id,
+                    jobId=self.running_job_id,
+                    location=self.location).execute()
+            else:
+                jobs.cancel(
+                    projectId=self.project_id,
+                    jobId=self.running_job_id).execute()
         else:
             self.log.info('No running BigQuery jobs to cancel.')
             return
@@ -1042,13 +1314,13 @@ def cancel_query(self):
         polling_attempts = 0
 
         job_complete = False
-        while (polling_attempts < max_polling_attempts and not job_complete):
+        while polling_attempts < max_polling_attempts and not job_complete:
             polling_attempts = polling_attempts + 1
             job_complete = self.poll_job_complete(self.running_job_id)
-            if (job_complete):
+            if job_complete:
                 self.log.info('Job successfully canceled: %s, %s',
                               self.project_id, self.running_job_id)
-            elif (polling_attempts == max_polling_attempts):
+            elif polling_attempts == max_polling_attempts:
                 self.log.info(
                     "Stopping polling due to timeout. Job with id %s "
                     "has not completed cancel and may or may not finish.",
@@ -1112,18 +1384,14 @@ def run_table_delete(self, deletion_dataset_table,
         is set to True.
 
         :param deletion_dataset_table: A dotted
-        (<project>.|<project>:)<dataset>.<table> that indicates which table
-        will be deleted.
+            ``(<project>.|<project>:)<dataset>.<table>`` that indicates which table
+            will be deleted.
         :type deletion_dataset_table: str
         :param ignore_if_missing: if True, then return success even if the
-        requested table does not exist.
-        :type ignore_if_missing: boolean
+            requested table does not exist.
+        :type ignore_if_missing: bool
         :return:
         """
-
-        assert '.' in deletion_dataset_table, (
-            'Expected deletion_dataset_table in the format of '
-            '<dataset>.<table>. Got: {}').format(deletion_dataset_table)
         deletion_project, deletion_dataset, deletion_table = \
             _split_tablename(table_input=deletion_dataset_table,
                              default_project_id=self.project_id)
@@ -1155,7 +1423,7 @@ def run_table_upsert(self, dataset_id, table_resource, project_id=None):
             https://cloud.google.com/bigquery/docs/reference/v2/tables#resource
         :type table_resource: dict
         :param project_id: the project to upsert the table into.  If None,
-        project will be self.project_id.
+            project will be self.project_id.
         :return:
         """
         # check to see if the table exists
@@ -1209,10 +1477,10 @@ def run_grant_dataset_view_access(self,
         :param view_table: the table of the view
         :type view_table: str
         :param source_project: the project of the source dataset. If None,
-        self.project_id will be used.
+            self.project_id will be used.
         :type source_project: str
         :param view_project: the project that the view is in. If None,
-        self.project_id will be used.
+            self.project_id will be used.
         :type view_project: str
         :return: the datasets resource of the source dataset.
         """
@@ -1251,10 +1519,257 @@ def run_grant_dataset_view_access(self,
             # if view is already in access, do nothing.
             self.log.info(
                 'Table %s:%s.%s already has authorized view access to %s:%s dataset.',
-                view_project, view_dataset, view_table, source_project,
-                source_dataset)
+                view_project, view_dataset, view_table, source_project, source_dataset)
             return source_dataset_resource
 
+    def create_empty_dataset(self, dataset_id="", project_id="",
+                             dataset_reference=None):
+        """
+        Create a new empty dataset:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/insert
+
+        :param project_id: The name of the project where we want to create
+            an empty a dataset. Don't need to provide, if projectId in dataset_reference.
+        :type project_id: str
+        :param dataset_id: The id of dataset. Don't need to provide,
+            if datasetId in dataset_reference.
+        :type dataset_id: str
+        :param dataset_reference: Dataset reference that could be provided
+            with request body. More info:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource
+        :type dataset_reference: dict
+        """
+
+        if dataset_reference:
+            _validate_value('dataset_reference', dataset_reference, dict)
+        else:
+            dataset_reference = {}
+
+        if "datasetReference" not in dataset_reference:
+            dataset_reference["datasetReference"] = {}
+
+        if not dataset_reference["datasetReference"].get("datasetId") and not dataset_id:
+            raise ValueError(
+                "{} not provided datasetId. Impossible to create dataset")
+
+        dataset_required_params = [(dataset_id, "datasetId", ""),
+                                   (project_id, "projectId", self.project_id)]
+        for param_tuple in dataset_required_params:
+            param, param_name, param_default = param_tuple
+            if param_name not in dataset_reference['datasetReference']:
+                if param_default and not param:
+                    self.log.info(
+                        "%s was not specified. Will be used default value %s.",
+                        param_name, param_default
+                    )
+                    param = param_default
+                dataset_reference['datasetReference'].update(
+                    {param_name: param})
+            elif param:
+                _api_resource_configs_duplication_check(
+                    param_name, param,
+                    dataset_reference['datasetReference'], 'dataset_reference')
+
+        dataset_id = dataset_reference.get("datasetReference").get("datasetId")
+        dataset_project_id = dataset_reference.get("datasetReference").get(
+            "projectId")
+
+        self.log.info('Creating Dataset: %s in project: %s ', dataset_id,
+                      dataset_project_id)
+
+        try:
+            self.service.datasets().insert(
+                projectId=dataset_project_id,
+                body=dataset_reference).execute()
+            self.log.info('Dataset created successfully: In project %s '
+                          'Dataset %s', dataset_project_id, dataset_id)
+
+        except HttpError as err:
+            raise AirflowException(
+                'BigQuery job failed. Error was: {}'.format(err.content)
+            )
+
+    def delete_dataset(self, project_id, dataset_id):
+        """
+        Delete a dataset of Big query in your project.
+        :param project_id: The name of the project where we have the dataset .
+        :type project_id: str
+        :param dataset_id: The dataset to be delete.
+        :type dataset_id: str
+        :return:
+        """
+        project_id = project_id if project_id is not None else self.project_id
+        self.log.info('Deleting from project: %s  Dataset:%s',
+                      project_id, dataset_id)
+
+        try:
+            self.service.datasets().delete(
+                projectId=project_id,
+                datasetId=dataset_id).execute()
+            self.log.info('Dataset deleted successfully: In project %s '
+                          'Dataset %s', project_id, dataset_id)
+
+        except HttpError as err:
+            raise AirflowException(
+                'BigQuery job failed. Error was: {}'.format(err.content)
+            )
+
+    def get_dataset(self, dataset_id, project_id=None):
+        """
+        Method returns dataset_resource if dataset exist
+        and raised 404 error if dataset does not exist
+
+        :param dataset_id: The BigQuery Dataset ID
+        :type dataset_id: str
+        :param project_id: The GCP Project ID
+        :type project_id: str
+        :return: dataset_resource
+
+            .. seealso::
+                For more information, see Dataset Resource content:
+                https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource
+        """
+
+        if not dataset_id or not isinstance(dataset_id, str):
+            raise ValueError("dataset_id argument must be provided and has "
+                             "a type 'str'. You provided: {}".format(dataset_id))
+
+        dataset_project_id = project_id if project_id else self.project_id
+
+        try:
+            dataset_resource = self.service.datasets().get(
+                datasetId=dataset_id, projectId=dataset_project_id).execute()
+            self.log.info("Dataset Resource: %s", dataset_resource)
+        except HttpError as err:
+            raise AirflowException(
+                'BigQuery job failed. Error was: {}'.format(err.content))
+
+        return dataset_resource
+
+    def get_datasets_list(self, project_id=None):
+        """
+        Method returns full list of BigQuery datasets in the current project
+
+        .. seealso::
+            For more information, see:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/list
+
+        :param project_id: Google Cloud Project for which you
+            try to get all datasets
+        :type project_id: str
+        :return: datasets_list
+
+            Example of returned datasets_list: ::
+
+                   {
+                      "kind":"bigquery#dataset",
+                      "location":"US",
+                      "id":"your-project:dataset_2_test",
+                      "datasetReference":{
+                         "projectId":"your-project",
+                         "datasetId":"dataset_2_test"
+                      }
+                   },
+                   {
+                      "kind":"bigquery#dataset",
+                      "location":"US",
+                      "id":"your-project:dataset_1_test",
+                      "datasetReference":{
+                         "projectId":"your-project",
+                         "datasetId":"dataset_1_test"
+                      }
+                   }
+                ]
+        """
+        dataset_project_id = project_id if project_id else self.project_id
+
+        try:
+            datasets_list = self.service.datasets().list(
+                projectId=dataset_project_id).execute()['datasets']
+            self.log.info("Datasets List: %s", datasets_list)
+
+        except HttpError as err:
+            raise AirflowException(
+                'BigQuery job failed. Error was: {}'.format(err.content))
+
+        return datasets_list
+
+    def insert_all(self, project_id, dataset_id, table_id,
+                   rows, ignore_unknown_values=False,
+                   skip_invalid_rows=False, fail_on_error=False):
+        """
+        Method to stream data into BigQuery one record at a time without needing
+        to run a load job
+
+        .. seealso::
+            For more information, see:
+            https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
+
+        :param project_id: The name of the project where we have the table
+        :type project_id: str
+        :param dataset_id: The name of the dataset where we have the table
+        :type dataset_id: str
+        :param table_id: The name of the table
+        :type table_id: str
+        :param rows: the rows to insert
+        :type rows: list
+
+        **Example or rows**:
+            rows=[{"json": {"a_key": "a_value_0"}}, {"json": {"a_key": "a_value_1"}}]
+
+        :param ignore_unknown_values: [Optional] Accept rows that contain values
+            that do not match the schema. The unknown values are ignored.
+            The default value  is false, which treats unknown values as errors.
+        :type ignore_unknown_values: bool
+        :param skip_invalid_rows: [Optional] Insert all valid rows of a request,
+            even if invalid rows exist. The default value is false, which causes
+            the entire request to fail if any invalid rows exist.
+        :type skip_invalid_rows: bool
+        :param fail_on_error: [Optional] Force the task to fail if any errors occur.
+            The default value is false, which indicates the task should not fail
+            even if any insertion errors occur.
+        :type fail_on_error: bool
+        """
+
+        dataset_project_id = project_id if project_id else self.project_id
+
+        body = {
+            "rows": rows,
+            "ignoreUnknownValues": ignore_unknown_values,
+            "kind": "bigquery#tableDataInsertAllRequest",
+            "skipInvalidRows": skip_invalid_rows,
+        }
+
+        try:
+            self.log.info(
+                'Inserting %s row(s) into Table %s:%s.%s',
+                len(rows), dataset_project_id, dataset_id, table_id
+            )
+
+            resp = self.service.tabledata().insertAll(
+                projectId=dataset_project_id, datasetId=dataset_id,
+                tableId=table_id, body=body
+            ).execute()
+
+            if 'insertErrors' not in resp:
+                self.log.info(
+                    'All row(s) inserted successfully: %s:%s.%s',
+                    dataset_project_id, dataset_id, table_id
+                )
+            else:
+                error_msg = '{} insert error(s) occured: {}:{}.{}. Details: {}'.format(
+                    len(resp['insertErrors']),
+                    dataset_project_id, dataset_id, table_id, resp['insertErrors'])
+                if fail_on_error:
+                    raise AirflowException(
+                        'BigQuery job failed. Error was: {}'.format(error_msg)
+                    )
+                self.log.info(error_msg)
+        except HttpError as err:
+            raise AirflowException(
+                'BigQuery job failed. Error was: {}'.format(err.content)
+            )
+
 
 class BigQueryCursor(BigQueryBaseCursor):
     """
@@ -1265,11 +1780,13 @@ class BigQueryCursor(BigQueryBaseCursor):
     https://github.com/dropbox/PyHive/blob/master/pyhive/common.py
     """
 
-    def __init__(self, service, project_id, use_legacy_sql=True):
+    def __init__(self, service, project_id, use_legacy_sql=True, location=None):
         super(BigQueryCursor, self).__init__(
             service=service,
             project_id=project_id,
-            use_legacy_sql=use_legacy_sql)
+            use_legacy_sql=use_legacy_sql,
+            location=location,
+        )
         self.buffersize = None
         self.page_token = None
         self.job_id = None
@@ -1295,7 +1812,7 @@ def execute(self, operation, parameters=None):
         Executes a BigQuery query, and returns the job ID.
 
         :param operation: The query to execute.
-        :type operation: string
+        :type operation: str
         :param parameters: Parameters to substitute into the query.
         :type parameters: dict
         """
@@ -1308,7 +1825,7 @@ def executemany(self, operation, seq_of_parameters):
         Execute a BigQuery query multiple times with different parameters.
 
         :param operation: The query to execute.
-        :type operation: string
+        :type operation: str
         :param seq_of_parameters: List of dictionary parameters to substitute into the
             query.
         :type seq_of_parameters: list
@@ -1423,7 +1940,7 @@ def _bind_parameters(operation, parameters):
     """ Helper method that binds parameters to a SQL query. """
     # inspired by MySQL Python Connector (conversion.py)
     string_parameters = {}
-    for (name, value) in parameters.iteritems():
+    for (name, value) in iteritems(parameters):
         if value is None:
             string_parameters[name] = 'NULL'
         elif isinstance(value, basestring):
@@ -1456,14 +1973,23 @@ def _bq_cast(string_field, bq_type):
     elif bq_type == 'FLOAT' or bq_type == 'TIMESTAMP':
         return float(string_field)
     elif bq_type == 'BOOLEAN':
-        assert string_field in set(['true', 'false'])
+        if string_field not in ['true', 'false']:
+            raise ValueError("{} must have value 'true' or 'false'".format(
+                string_field))
         return string_field == 'true'
     else:
         return string_field
 
 
 def _split_tablename(table_input, default_project_id, var_name=None):
-    assert default_project_id is not None, "INTERNAL: No default project is specified"
+
+    if '.' not in table_input:
+        raise ValueError(
+            'Expected target table name in the format of '
+            '<dataset>.<table>. Got: {}'.format(table_input))
+
+    if not default_project_id:
+        raise ValueError("INTERNAL: No default project is specified")
 
     def var_print(var_name):
         if var_name is None:
@@ -1475,7 +2001,6 @@ def var_print(var_name):
         raise Exception(('{var}Use either : or . to specify project '
                          'got {input}').format(
                              var=var_print(var_name), input=table_input))
-
     cmpt = table_input.rsplit(':', 1)
     project_id = None
     rest = table_input
@@ -1493,8 +2018,10 @@ def var_print(var_name):
 
     cmpt = rest.split('.')
     if len(cmpt) == 3:
-        assert project_id is None, ("{var}Use either : or . to specify project"
-                                    ).format(var=var_print(var_name))
+        if project_id:
+            raise ValueError(
+                "{var}Use either : or . to specify project".format(
+                    var=var_print(var_name)))
         project_id = cmpt[0]
         dataset_id = cmpt[1]
         table_id = cmpt[2]
@@ -1510,11 +2037,10 @@ def var_print(var_name):
     if project_id is None:
         if var_name is not None:
             log = LoggingMixin().log
-            log.info('Project not included in {var}: {input}; '
-                     'using project "{project}"'.format(
-                         var=var_name,
-                         input=table_input,
-                         project=default_project_id))
+            log.info(
+                'Project not included in %s: %s; using project "%s"',
+                var_name, table_input, default_project_id
+            )
         project_id = default_project_id
 
     return project_id, dataset_id, table_id
@@ -1522,13 +2048,30 @@ def var_print(var_name):
 
 def _cleanse_time_partitioning(destination_dataset_table, time_partitioning_in):
     # if it is a partitioned table ($ is in the table name) add partition load option
+
+    if time_partitioning_in is None:
+        time_partitioning_in = {}
+
     time_partitioning_out = {}
     if destination_dataset_table and '$' in destination_dataset_table:
-        assert not time_partitioning_in.get('field'), (
-            "Cannot specify field partition and partition name "
-            "(dataset.table$partition) at the same time"
-        )
         time_partitioning_out['type'] = 'DAY'
-
     time_partitioning_out.update(time_partitioning_in)
     return time_partitioning_out
+
+
+def _validate_value(key, value, expected_type):
+    """ function to check expected type and raise
+    error if type is not correct """
+    if not isinstance(value, expected_type):
+        raise TypeError("{} argument must have a type {} not {}".format(
+            key, expected_type, type(value)))
+
+
+def _api_resource_configs_duplication_check(key, value, config_dict,
+                                            config_dict_name='api_resource_configs'):
+    if key in config_dict and value != config_dict[key]:
+        raise ValueError("Values of {param_name} param are duplicated. "
+                         "{dict_name} contained {param_name} param "
+                         "in `query` config and {param_name} was also provided "
+                         "with arg to run_query() method. Please remove duplicates."
+                         .format(param_name=key, dict_name=config_dict_name))
diff --git a/airflow/contrib/hooks/cassandra_hook.py b/airflow/contrib/hooks/cassandra_hook.py
index 704ba0d8d03e2..08def3a5b775c 100644
--- a/airflow/contrib/hooks/cassandra_hook.py
+++ b/airflow/contrib/hooks/cassandra_hook.py
@@ -36,14 +36,17 @@ class CassandraHook(BaseHook, LoggingMixin):
     Port can be specified in the port field of the connection.
 
     If SSL is enabled in Cassandra, pass in a dict in the extra field as kwargs for
-    ``ssl.wrap_socket()``. For example:
-            {
-                'ssl_options' : {
-                    'ca_certs' : PATH_TO_CA_CERTS
-                }
+    ``ssl.wrap_socket()``. For example::
+
+        {
+            'ssl_options' : {
+                'ca_certs' : PATH_TO_CA_CERTS
             }
+        }
+
+    Default load balancing policy is RoundRobinPolicy. To specify a different
+    LB policy::
 
-    Default load balancing policy is RoundRobinPolicy. To specify a different LB policy:
         - DCAwareRoundRobinPolicy
             {
                 'load_balancing_policy': 'DCAwareRoundRobinPolicy',
@@ -158,22 +161,37 @@ def get_lb_policy(policy_name, policy_args):
                                                            child_policy_args)
                 return TokenAwarePolicy(child_policy)
 
+    def table_exists(self, table):
+        """
+        Checks if a table exists in Cassandra
+
+        :param table: Target Cassandra table.
+                      Use dot notation to target a specific keyspace.
+        :type table: str
+        """
+        keyspace = self.keyspace
+        if '.' in table:
+            keyspace, table = table.split('.', 1)
+        cluster_metadata = self.get_conn().cluster.metadata
+        return (keyspace in cluster_metadata.keyspaces and
+                table in cluster_metadata.keyspaces[keyspace].tables)
+
     def record_exists(self, table, keys):
         """
         Checks if a record exists in Cassandra
 
         :param table: Target Cassandra table.
                       Use dot notation to target a specific keyspace.
-        :type table: string
+        :type table: str
         :param keys: The keys and their values to check the existence.
         :type keys: dict
         """
-        keyspace = None
+        keyspace = self.keyspace
         if '.' in table:
             keyspace, table = table.split('.', 1)
         ks = " AND ".join("{}=%({})s".format(key, key) for key in keys.keys())
         cql = "SELECT * FROM {keyspace}.{table} WHERE {keys}".format(
-            keyspace=(keyspace or self.keyspace), table=table, keys=ks)
+            keyspace=keyspace, table=table, keys=ks)
 
         try:
             rs = self.get_conn().execute(cql, keys)
diff --git a/airflow/contrib/hooks/databricks_hook.py b/airflow/contrib/hooks/databricks_hook.py
index 1443ff4740b94..a3cd31617a4c0 100644
--- a/airflow/contrib/hooks/databricks_hook.py
+++ b/airflow/contrib/hooks/databricks_hook.py
@@ -24,22 +24,21 @@
 from airflow.hooks.base_hook import BaseHook
 from requests import exceptions as requests_exceptions
 from requests.auth import AuthBase
+from time import sleep
+from six.moves.urllib import parse as urlparse
 
-from airflow.utils.log.logging_mixin import LoggingMixin
-
-try:
-    from urllib import parse as urlparse
-except ImportError:
-    import urlparse
-
+RESTART_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/restart")
+START_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/start")
+TERMINATE_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/delete")
 
+RUN_NOW_ENDPOINT = ('POST', 'api/2.0/jobs/run-now')
 SUBMIT_RUN_ENDPOINT = ('POST', 'api/2.0/jobs/runs/submit')
 GET_RUN_ENDPOINT = ('GET', 'api/2.0/jobs/runs/get')
 CANCEL_RUN_ENDPOINT = ('POST', 'api/2.0/jobs/runs/cancel')
 USER_AGENT_HEADER = {'user-agent': 'airflow-{v}'.format(v=__version__)}
 
 
-class DatabricksHook(BaseHook, LoggingMixin):
+class DatabricksHook(BaseHook):
     """
     Interact with Databricks.
     """
@@ -47,38 +46,47 @@ def __init__(
             self,
             databricks_conn_id='databricks_default',
             timeout_seconds=180,
-            retry_limit=3):
+            retry_limit=3,
+            retry_delay=1.0):
         """
         :param databricks_conn_id: The name of the databricks connection to use.
-        :type databricks_conn_id: string
+        :type databricks_conn_id: str
         :param timeout_seconds: The amount of time in seconds the requests library
             will wait before timing-out.
         :type timeout_seconds: int
         :param retry_limit: The number of times to retry the connection in case of
             service outages.
         :type retry_limit: int
+        :param retry_delay: The number of seconds to wait between retries (it
+            might be a floating point number).
+        :type retry_delay: float
         """
         self.databricks_conn_id = databricks_conn_id
         self.databricks_conn = self.get_connection(databricks_conn_id)
         self.timeout_seconds = timeout_seconds
-        assert retry_limit >= 1, 'Retry limit must be greater than equal to 1'
+        if retry_limit < 1:
+            raise ValueError('Retry limit must be greater than equal to 1')
         self.retry_limit = retry_limit
+        self.retry_delay = retry_delay
 
-    def _parse_host(self, host):
+    @staticmethod
+    def _parse_host(host):
         """
         The purpose of this function is to be robust to improper connections
         settings provided by users, specifically in the host field.
 
-
         For example -- when users supply ``https://xx.cloud.databricks.com`` as the
-        host, we must strip out the protocol to get the host.
-        >>> h = DatabricksHook()
-        >>> assert h._parse_host('https://xx.cloud.databricks.com') == \
-            'xx.cloud.databricks.com'
+        host, we must strip out the protocol to get the host.::
+
+            h = DatabricksHook()
+            assert h._parse_host('https://xx.cloud.databricks.com') == \
+                'xx.cloud.databricks.com'
 
         In the case where users supply the correct ``xx.cloud.databricks.com`` as the
-        host, this function is a no-op.
-        >>> assert h._parse_host('xx.cloud.databricks.com') == 'xx.cloud.databricks.com'
+        host, this function is a no-op.::
+
+            assert h._parse_host('xx.cloud.databricks.com') == 'xx.cloud.databricks.com'
+
         """
         urlparse_host = urlparse.urlparse(host).hostname
         if urlparse_host:
@@ -91,8 +99,9 @@ def _parse_host(self, host):
     def _do_api_call(self, endpoint_info, json):
         """
         Utility function to perform an API call with retries
+
         :param endpoint_info: Tuple of method and endpoint
-        :type endpoint_info: (string, string)
+        :type endpoint_info: tuple[string, string]
         :param json: Parameters for this API call.
         :type json: dict
         :return: If the api call returns a OK status code,
@@ -117,7 +126,8 @@ def _do_api_call(self, endpoint_info, json):
         else:
             raise AirflowException('Unexpected HTTP Method: ' + method)
 
-        for attempt_num in range(1, self.retry_limit + 1):
+        attempt_num = 1
+        while True:
             try:
                 response = request_func(
                     url,
@@ -125,21 +135,41 @@ def _do_api_call(self, endpoint_info, json):
                     auth=auth,
                     headers=USER_AGENT_HEADER,
                     timeout=self.timeout_seconds)
-                if response.status_code == requests.codes.ok:
-                    return response.json()
-                else:
+                response.raise_for_status()
+                return response.json()
+            except requests_exceptions.RequestException as e:
+                if not _retryable_error(e):
                     # In this case, the user probably made a mistake.
                     # Don't retry.
                     raise AirflowException('Response: {0}, Status Code: {1}'.format(
-                        response.content, response.status_code))
-            except (requests_exceptions.ConnectionError,
-                    requests_exceptions.Timeout) as e:
-                self.log.error(
-                    'Attempt %s API Request to Databricks failed with reason: %s',
-                    attempt_num, e
-                )
-        raise AirflowException(('API requests to Databricks failed {} times. ' +
-                               'Giving up.').format(self.retry_limit))
+                        e.response.content, e.response.status_code))
+
+                self._log_request_error(attempt_num, e)
+
+            if attempt_num == self.retry_limit:
+                raise AirflowException(('API requests to Databricks failed {} times. ' +
+                                        'Giving up.').format(self.retry_limit))
+
+            attempt_num += 1
+            sleep(self.retry_delay)
+
+    def _log_request_error(self, attempt_num, error):
+        self.log.error(
+            'Attempt %s API Request to Databricks failed with reason: %s',
+            attempt_num, error
+        )
+
+    def run_now(self, json):
+        """
+        Utility function to call the ``api/2.0/jobs/run-now`` endpoint.
+
+        :param json: The data used in the body of the request to the ``run-now`` endpoint.
+        :type json: dict
+        :return: the run_id as a string
+        :rtype: str
+        """
+        response = self._do_api_call(RUN_NOW_ENDPOINT, json)
+        return response['run_id']
 
     def submit_run(self, json):
         """
@@ -148,7 +178,7 @@ def submit_run(self, json):
         :param json: The data used in the body of the request to the ``submit`` endpoint.
         :type json: dict
         :return: the run_id as a string
-        :rtype: string
+        :rtype: str
         """
         response = self._do_api_call(SUBMIT_RUN_ENDPOINT, json)
         return response['run_id']
@@ -172,6 +202,21 @@ def cancel_run(self, run_id):
         json = {'run_id': run_id}
         self._do_api_call(CANCEL_RUN_ENDPOINT, json)
 
+    def restart_cluster(self, json):
+        self._do_api_call(RESTART_CLUSTER_ENDPOINT, json)
+
+    def start_cluster(self, json):
+        self._do_api_call(START_CLUSTER_ENDPOINT, json)
+
+    def terminate_cluster(self, json):
+        self._do_api_call(TERMINATE_CLUSTER_ENDPOINT, json)
+
+
+def _retryable_error(exception):
+    return isinstance(exception, requests_exceptions.ConnectionError) \
+        or isinstance(exception, requests_exceptions.Timeout) \
+        or exception.response is not None and exception.response.status_code >= 500
+
 
 RUN_LIFE_CYCLE_STATES = [
     'PENDING',
diff --git a/airflow/contrib/hooks/datadog_hook.py b/airflow/contrib/hooks/datadog_hook.py
index 3dfeb781aefe0..50209dd112e06 100644
--- a/airflow/contrib/hooks/datadog_hook.py
+++ b/airflow/contrib/hooks/datadog_hook.py
@@ -35,7 +35,7 @@ class DatadogHook(BaseHook, LoggingMixin):
     Airflow runs.
 
     :param datadog_conn_id: The connection to datadog, containing metadata for api keys.
-    :param datadog_conn_id: string
+    :param datadog_conn_id: str
     """
     def __init__(self, datadog_conn_id='datadog_default'):
         conn = self.get_connection(datadog_conn_id)
@@ -50,38 +50,37 @@ def __init__(self, datadog_conn_id='datadog_default'):
         if self.api_key is None:
             raise AirflowException("api_key must be specified in the "
                                    "Datadog connection details")
-        if self.app_key is None:
-            raise AirflowException("app_key must be specified in the "
-                                   "Datadog connection details")
 
         self.log.info("Setting up api keys for Datadog")
-        options = {
-            'api_key': self.api_key,
-            'app_key': self.app_key
-        }
-        initialize(**options)
+        initialize(api_key=self.api_key, app_key=self.app_key)
 
     def validate_response(self, response):
         if response['status'] != 'ok':
             self.log.error("Datadog returned: %s", response)
             raise AirflowException("Error status received from Datadog")
 
-    def send_metric(self, metric_name, datapoint, tags=None):
+    def send_metric(self, metric_name, datapoint, tags=None, type_=None, interval=None):
         """
         Sends a single datapoint metric to DataDog
 
         :param metric_name: The name of the metric
-        :type metric_name: string
+        :type metric_name: str
         :param datapoint: A single integer or float related to the metric
-        :type datapoint: integer or float
+        :type datapoint: int or float
         :param tags: A list of tags associated with the metric
         :type tags: list
+        :param type_: Type of your metric: gauge, rate, or count
+        :type type_: str
+        :param interval: If the type of the metric is rate or count, define the corresponding interval
+        :type interval: int
         """
         response = api.Metric.send(
             metric=metric_name,
             points=datapoint,
             host=self.host,
-            tags=tags)
+            tags=tags,
+            type=type_,
+            interval=interval)
 
         self.validate_response(response)
         return response
@@ -95,7 +94,7 @@ def query_metric(self,
         function applied to it and returns the results.
 
         :param query: The datadog query to execute (see datadog docs)
-        :type query: string
+        :type query: str
         :param from_seconds_ago: How many seconds ago to start querying for.
         :type from_seconds_ago: int
         :param to_seconds_ago: Up to how many seconds ago to query for.
@@ -111,31 +110,48 @@ def query_metric(self,
         self.validate_response(response)
         return response
 
-    def post_event(self, title, text, tags=None, alert_type=None, aggregation_key=None):
+    def post_event(self, title, text, aggregation_key=None, alert_type=None, date_happened=None,
+                   handle=None, priority=None, related_event_id=None, tags=None, device_name=None):
         """
         Posts an event to datadog (processing finished, potentially alerts, other issues)
         Think about this as a means to maintain persistence of alerts, rather than
         alerting itself.
 
         :param title: The title of the event
-        :type title: string
+        :type title: str
         :param text: The body of the event (more information)
-        :type text: string
-        :param tags: List of string tags to apply to the event
-        :type tags: list
+        :type text: str
+        :param aggregation_key: Key that can be used to aggregate this event in a stream
+        :type aggregation_key: str
         :param alert_type: The alert type for the event, one of
             ["error", "warning", "info", "success"]
-        :type alert_type: string
-        :param aggregation_key: Key that can be used to aggregate this event in a stream
-        :type aggregation_key: string
+        :type alert_type: str
+        :param date_happened: POSIX timestamp of the event; defaults to now
+        :type date_happened: int
+        :handle: User to post the event as; defaults to owner of the application key used
+            to submit.
+        :param handle: str
+        :param priority: Priority to post the event as. ("normal" or "low", defaults to "normal")
+        :type priority: str
+        :param related_event_id: Post event as a child of the given event
+        :type related_event_id: id
+        :param tags: List of tags to apply to the event
+        :type tags: list[str]
+        :param device_name: device_name to post the event with
+        :type device_name: list
         """
         response = api.Event.create(
             title=title,
             text=text,
-            host=self.host,
-            tags=tags,
-            alert_type=alert_type,
             aggregation_key=aggregation_key,
+            alert_type=alert_type,
+            date_happened=date_happened,
+            handle=handle,
+            priority=priority,
+            related_event_id=related_event_id,
+            tags=tags,
+            host=self.host,
+            device_name=device_name,
             source_type_name=self.source_type_name)
 
         self.validate_response(response)
diff --git a/airflow/contrib/hooks/datastore_hook.py b/airflow/contrib/hooks/datastore_hook.py
index 5e54cf2a65384..308809e1413b1 100644
--- a/airflow/contrib/hooks/datastore_hook.py
+++ b/airflow/contrib/hooks/datastore_hook.py
@@ -19,7 +19,7 @@
 #
 
 import time
-from apiclient.discovery import build
+from googleapiclient.discovery import build
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 
 
@@ -41,7 +41,7 @@ def __init__(self,
 
     def get_conn(self, version='v1'):
         """
-        Returns a Google Cloud Storage service object.
+        Returns a Google Cloud Datastore service object.
         """
         http_authorized = self._authorize()
         return build(
@@ -172,7 +172,7 @@ def export_to_storage_bucket(self, bucket, namespace=None,
         """
         Export entities from Cloud Datastore to Cloud Storage for backup
         """
-        output_uri_prefix = 'gs://' + ('/').join(filter(None, [bucket, namespace]))
+        output_uri_prefix = 'gs://' + '/'.join(filter(None, [bucket, namespace]))
         if not entity_filter:
             entity_filter = {}
         if not labels:
@@ -191,7 +191,7 @@ def import_from_storage_bucket(self, bucket, file,
         """
         Import a backup from Cloud Storage to Cloud Datastore
         """
-        input_url = 'gs://' + ('/').join(filter(None, [bucket, namespace, file]))
+        input_url = 'gs://' + '/'.join(filter(None, [bucket, namespace, file]))
         if not entity_filter:
             entity_filter = {}
         if not labels:
diff --git a/airflow/contrib/hooks/dingding_hook.py b/airflow/contrib/hooks/dingding_hook.py
new file mode 100644
index 0000000000000..1da373a8fb940
--- /dev/null
+++ b/airflow/contrib/hooks/dingding_hook.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+
+import requests
+
+from airflow import AirflowException
+from airflow.hooks.http_hook import HttpHook
+
+
+class DingdingHook(HttpHook):
+    """
+    This hook allows you send Dingding message using Dingding custom bot.
+    Get Dingding token from conn_id.password. And prefer set domain to
+    conn_id.host, if not will use default ``https://oapi.dingtalk.com``.
+
+    For more detail message in
+    `Dingding custom bot <https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq>`_
+
+    :param dingding_conn_id: The name of the Dingding connection to use
+    :type dingding_conn_id: str
+    :param message_type: Message type you want to send to Dingding, support five type so far
+        including text, link, markdown, actionCard, feedCard
+    :type message_type: str
+    :param message: The message send to Dingding chat group
+    :type message: str or dict
+    :param at_mobiles: Remind specific users with this message
+    :type at_mobiles: list[str]
+    :param at_all: Remind all people in group or not. If True, will overwrite ``at_mobiles``
+    :type at_all: bool
+    """
+
+    def __init__(self,
+                 dingding_conn_id='dingding_default',
+                 message_type='text',
+                 message=None,
+                 at_mobiles=None,
+                 at_all=False,
+                 *args,
+                 **kwargs
+                 ):
+        super(DingdingHook, self).__init__(http_conn_id=dingding_conn_id, *args, **kwargs)
+        self.message_type = message_type
+        self.message = message
+        self.at_mobiles = at_mobiles
+        self.at_all = at_all
+
+    def _get_endpoint(self):
+        """
+        Get Dingding endpoint for sending message.
+        """
+        conn = self.get_connection(self.http_conn_id)
+        token = conn.password
+        if not token:
+            raise AirflowException('Dingding token is requests but get nothing, '
+                                   'check you conn_id configuration.')
+        return 'robot/send?access_token={}'.format(token)
+
+    def _build_message(self):
+        """
+        Build different type of Dingding message
+        As most commonly used type, text message just need post message content
+        rather than a dict like ``{'content': 'message'}``
+        """
+        if self.message_type in ['text', 'markdown']:
+            data = {
+                'msgtype': self.message_type,
+                self.message_type: {
+                    'content': self.message
+                } if self.message_type == 'text' else self.message,
+                'at': {
+                    'atMobiles': self.at_mobiles,
+                    'isAtAll': self.at_all
+                }
+            }
+        else:
+            data = {
+                'msgtype': self.message_type,
+                self.message_type: self.message
+            }
+        return json.dumps(data)
+
+    def get_conn(self, headers=None):
+        """
+        Overwrite HttpHook get_conn because just need base_url and headers and
+        not don't need generic params
+        :param headers: additional headers to be passed through as a dictionary
+        :type headers: dict
+        """
+        conn = self.get_connection(self.http_conn_id)
+        self.base_url = conn.host if conn.host else 'https://oapi.dingtalk.com'
+        session = requests.Session()
+        if headers:
+            session.headers.update(headers)
+        return session
+
+    def send(self):
+        """
+        Send Dingding message
+        """
+        support_type = ['text', 'link', 'markdown', 'actionCard', 'feedCard']
+        if self.message_type not in support_type:
+            raise ValueError('DingdingWebhookHook only support {} '
+                             'so far, but receive {}'.format(support_type, self.message_type))
+
+        data = self._build_message()
+        self.log.info('Sending Dingding type %s message %s', self.message_type, data)
+        resp = self.run(endpoint=self._get_endpoint(),
+                        data=data,
+                        headers={'Content-Type': 'application/json'})
+
+        # Dingding success send message will with errcode equal to 0
+        if int(resp.json().get('errcode')) != 0:
+            raise AirflowException('Send Dingding message failed, receive error '
+                                   'message %s', resp.text)
+        self.log.info('Success Send Dingding message')
diff --git a/airflow/contrib/hooks/emr_hook.py b/airflow/contrib/hooks/emr_hook.py
index 6cd92c6d85874..7571d0988f697 100644
--- a/airflow/contrib/hooks/emr_hook.py
+++ b/airflow/contrib/hooks/emr_hook.py
@@ -23,16 +23,17 @@
 
 class EmrHook(AwsHook):
     """
-    Interact with AWS EMR. emr_conn_id is only neccessary for using the
+    Interact with AWS EMR. emr_conn_id is only necessary for using the
     create_job_flow method.
     """
 
-    def __init__(self, emr_conn_id=None, *args, **kwargs):
+    def __init__(self, emr_conn_id=None, region_name=None, *args, **kwargs):
         self.emr_conn_id = emr_conn_id
+        self.region_name = region_name
         super(EmrHook, self).__init__(*args, **kwargs)
 
     def get_conn(self):
-        self.conn = self.get_client_type('emr')
+        self.conn = self.get_client_type('emr', self.region_name)
         return self.conn
 
     def create_job_flow(self, job_flow_overrides):
@@ -51,19 +52,6 @@ def create_job_flow(self, job_flow_overrides):
         config = emr_conn.extra_dejson.copy()
         config.update(job_flow_overrides)
 
-        response = self.get_conn().run_job_flow(
-            Name=config.get('Name'),
-            LogUri=config.get('LogUri'),
-            ReleaseLabel=config.get('ReleaseLabel'),
-            Instances=config.get('Instances'),
-            Steps=config.get('Steps', []),
-            BootstrapActions=config.get('BootstrapActions', []),
-            Applications=config.get('Applications'),
-            Configurations=config.get('Configurations', []),
-            VisibleToAllUsers=config.get('VisibleToAllUsers'),
-            JobFlowRole=config.get('JobFlowRole'),
-            ServiceRole=config.get('ServiceRole'),
-            Tags=config.get('Tags')
-        )
+        response = self.get_conn().run_job_flow(**config)
 
         return response
diff --git a/airflow/contrib/hooks/fs_hook.py b/airflow/contrib/hooks/fs_hook.py
index 6832f20c225c1..1aa528b6205dc 100644
--- a/airflow/contrib/hooks/fs_hook.py
+++ b/airflow/contrib/hooks/fs_hook.py
@@ -30,7 +30,7 @@ class FSHook(BaseHook):
     example:
     Conn Id: fs_test
     Conn Type: File (path)
-    Host, Shchema, Login, Password, Port: empty
+    Host, Schema, Login, Password, Port: empty
     Extra: {"path": "/tmp"}
     """
 
diff --git a/airflow/contrib/hooks/ftp_hook.py b/airflow/contrib/hooks/ftp_hook.py
index 8beefb372916c..a12ece9bc9420 100644
--- a/airflow/contrib/hooks/ftp_hook.py
+++ b/airflow/contrib/hooks/ftp_hook.py
@@ -24,8 +24,6 @@
 from airflow.hooks.base_hook import BaseHook
 from past.builtins import basestring
 
-from airflow.utils.log.logging_mixin import LoggingMixin
-
 
 def mlsd(conn, path="", facts=None):
     """
@@ -60,7 +58,7 @@ def mlsd(conn, path="", facts=None):
         yield (name, entry)
 
 
-class FTPHook(BaseHook, LoggingMixin):
+class FTPHook(BaseHook):
     """
     Interact with FTP.
 
@@ -148,7 +146,11 @@ def delete_directory(self, path):
         conn = self.get_conn()
         conn.rmd(path)
 
-    def retrieve_file(self, remote_full_path, local_full_path_or_buffer):
+    def retrieve_file(
+            self,
+            remote_full_path,
+            local_full_path_or_buffer,
+            callback=None):
         """
         Transfers the remote file to a local location.
 
@@ -161,23 +163,60 @@ def retrieve_file(self, remote_full_path, local_full_path_or_buffer):
         :param local_full_path_or_buffer: full path to the local file or a
             file-like buffer
         :type local_full_path_or_buffer: str or file-like buffer
+        :param callback: callback which is called each time a block of data
+            is read. if you do not use a callback, these blocks will be written
+            to the file or buffer passed in. if you do pass in a callback, note
+            that writing to a file or buffer will need to be handled inside the
+            callback.
+            [default: output_handle.write()]
+        :type callback: callable
+
+        :Example::
+
+            hook = FTPHook(ftp_conn_id='my_conn')
+
+            remote_path = '/path/to/remote/file'
+            local_path = '/path/to/local/file'
+
+            # with a custom callback (in this case displaying progress on each read)
+            def print_progress(percent_progress):
+                self.log.info('Percent Downloaded: %s%%' % percent_progress)
+
+            total_downloaded = 0
+            total_file_size = hook.get_size(remote_path)
+            output_handle = open(local_path, 'wb')
+            def write_to_file_with_progress(data):
+                total_downloaded += len(data)
+                output_handle.write(data)
+                percent_progress = (total_downloaded / total_file_size) * 100
+                print_progress(percent_progress)
+            hook.retrieve_file(remote_path, None, callback=write_to_file_with_progress)
+
+            # without a custom callback data is written to the local_path
+            hook.retrieve_file(remote_path, local_path)
         """
         conn = self.get_conn()
 
         is_path = isinstance(local_full_path_or_buffer, basestring)
 
-        if is_path:
-            output_handle = open(local_full_path_or_buffer, 'wb')
+        # without a callback, default to writing to a user-provided file or
+        # file-like buffer
+        if not callback:
+            if is_path:
+                output_handle = open(local_full_path_or_buffer, 'wb')
+            else:
+                output_handle = local_full_path_or_buffer
+            callback = output_handle.write
         else:
-            output_handle = local_full_path_or_buffer
+            output_handle = None
 
         remote_path, remote_file_name = os.path.split(remote_full_path)
         conn.cwd(remote_path)
         self.log.info('Retrieving file from FTP: %s', remote_full_path)
-        conn.retrbinary('RETR %s' % remote_file_name, output_handle.write)
+        conn.retrbinary('RETR %s' % remote_file_name, callback)
         self.log.info('Finished retrieving file from FTP: %s', remote_full_path)
 
-        if is_path:
+        if is_path and output_handle:
             output_handle.close()
 
     def store_file(self, remote_full_path, local_full_path_or_buffer):
@@ -230,6 +269,12 @@ def rename(self, from_name, to_name):
         return conn.rename(from_name, to_name)
 
     def get_mod_time(self, path):
+        """
+        Returns a datetime object representing the last time the file was modified
+
+        :param path: remote file path
+        :type path: string
+        """
         conn = self.get_conn()
         ftp_mdtm = conn.sendcmd('MDTM ' + path)
         time_val = ftp_mdtm[4:]
@@ -239,6 +284,16 @@ def get_mod_time(self, path):
         except ValueError:
             return datetime.datetime.strptime(time_val, '%Y%m%d%H%M%S')
 
+    def get_size(self, path):
+        """
+        Returns the size of a file (in bytes)
+
+        :param path: remote file path
+        :type path: string
+        """
+        conn = self.get_conn()
+        return conn.size(path)
+
 
 class FTPSHook(FTPHook):
 
diff --git a/airflow/contrib/hooks/gcp_api_base_hook.py b/airflow/contrib/hooks/gcp_api_base_hook.py
index 053494743f064..db8f7de05e246 100644
--- a/airflow/contrib/hooks/gcp_api_base_hook.py
+++ b/airflow/contrib/hooks/gcp_api_base_hook.py
@@ -18,51 +18,62 @@
 # under the License.
 #
 import json
+import functools
 
 import httplib2
 import google.auth
 import google_auth_httplib2
 import google.oauth2.service_account
+import os
+import tempfile
+
+from google.api_core.exceptions import GoogleAPICallError, AlreadyExists, RetryError
+from googleapiclient.errors import HttpError
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.log.logging_mixin import LoggingMixin
 
 
 _DEFAULT_SCOPES = ('https://www.googleapis.com/auth/cloud-platform',)
+# The name of the environment variable that Google Authentication library uses
+# to get service account key location. Read more:
+# https://cloud.google.com/docs/authentication/getting-started#setting_the_environment_variable
+_G_APP_CRED_ENV_VAR = "GOOGLE_APPLICATION_CREDENTIALS"
 
 
-class GoogleCloudBaseHook(BaseHook, LoggingMixin):
+class GoogleCloudBaseHook(BaseHook):
     """
     A base hook for Google cloud-related hooks. Google cloud has a shared REST
     API client that is built in the same way no matter which service you use.
     This class helps construct and authorize the credentials needed to then
-    call apiclient.discovery.build() to actually discover and build a client
+    call googleapiclient.discovery.build() to actually discover and build a client
     for a Google cloud service.
 
     The class also contains some miscellaneous helper functions.
 
     All hook derived from this base hook use the 'Google Cloud Platform' connection
-    type. Two ways of authentication are supported:
+    type. Three ways of authentication are supported:
 
     Default credentials: Only the 'Project Id' is required. You'll need to
     have set up default credentials, such as by the
     ``GOOGLE_APPLICATION_DEFAULT`` environment variable or from the metadata
     server on Google Compute Engine.
 
-    JSON key file: Specify 'Project Id', 'Key Path' and 'Scope'.
+    JSON key file: Specify 'Project Id', 'Keyfile Path' and 'Scope'.
 
     Legacy P12 key files are not supported.
+
+    JSON data provided in the UI: Specify 'Keyfile JSON'.
     """
 
     def __init__(self, gcp_conn_id='google_cloud_default', delegate_to=None):
         """
         :param gcp_conn_id: The connection ID to use when fetching connection info.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request must have
             domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         self.gcp_conn_id = gcp_conn_id
         self.delegate_to = delegate_to
@@ -75,7 +86,7 @@ def _get_credentials(self):
         key_path = self._get_field('key_path', False)
         keyfile_dict = self._get_field('keyfile_dict', False)
         scope = self._get_field('scope', None)
-        if scope is not None:
+        if scope:
             scopes = [s.strip() for s in scope.split(',')]
         else:
             scopes = _DEFAULT_SCOPES
@@ -87,7 +98,7 @@ def _get_credentials(self):
         elif key_path:
             # Get credentials from a JSON file.
             if key_path.endswith('.json'):
-                self.log.info('Getting connection using a JSON key file.')
+                self.log.debug('Getting connection using JSON key file %s' % key_path)
                 credentials = (
                     google.oauth2.service_account.Credentials.from_service_account_file(
                         key_path, scopes=scopes)
@@ -142,7 +153,7 @@ def _get_field(self, f, default=None):
         key_path, etc. They get formatted as shown below.
         """
         long_f = 'extra__google_cloud_platform__{}'.format(f)
-        if long_f in self.extras:
+        if hasattr(self, 'extras') and long_f in self.extras:
             return self.extras[long_f]
         else:
             return default
@@ -150,3 +161,99 @@ def _get_field(self, f, default=None):
     @property
     def project_id(self):
         return self._get_field('project')
+
+    @staticmethod
+    def catch_http_exception(func):
+        """
+        Function decorator that intercepts HTTP Errors and raises AirflowException
+        with more informative message.
+        """
+
+        @functools.wraps(func)
+        def wrapper_decorator(self, *args, **kwargs):
+            try:
+                return func(self, *args, **kwargs)
+            except GoogleAPICallError as e:
+                if isinstance(e, AlreadyExists):
+                    raise e
+                else:
+                    self.log.error('The request failed:\n%s', str(e))
+                    raise AirflowException(e)
+            except RetryError as e:
+                self.log.error('The request failed due to a retryable error and retry attempts failed.')
+                raise AirflowException(e)
+            except ValueError as e:
+                self.log.error('The request failed, the parameters are invalid.')
+                raise AirflowException(e)
+            except HttpError as e:
+                self.log.error('The request failed:\n%s', str(e))
+                raise AirflowException(e)
+
+        return wrapper_decorator
+
+    @staticmethod
+    def fallback_to_default_project_id(func):
+        """
+        Decorator that provides fallback for Google Cloud Platform project id. If
+        the project is None it will be replaced with the project_id from the
+        service account the Hook is authenticated with. Project id can be specified
+        either via project_id kwarg or via first parameter in positional args.
+
+        :param func: function to wrap
+        :return: result of the function call
+        """
+        @functools.wraps(func)
+        def inner_wrapper(self, *args, **kwargs):
+            if len(args) > 0:
+                raise AirflowException(
+                    "You must use keyword arguments in this methods rather than"
+                    " positional")
+            if 'project_id' in kwargs:
+                kwargs['project_id'] = self._get_project_id(kwargs['project_id'])
+            else:
+                kwargs['project_id'] = self._get_project_id(None)
+            if not kwargs['project_id']:
+                raise AirflowException("The project id must be passed either as "
+                                       "keyword project_id parameter or as project_id extra "
+                                       "in GCP connection definition. Both are not set!")
+            return func(self, *args, **kwargs)
+        return inner_wrapper
+
+    def _get_project_id(self, project_id):
+        """
+        In case project_id is None, overrides it with default project_id from
+        the service account that is authorized.
+
+        :param project_id: project id to
+        :type project_id: str
+        :return: the project_id specified or default project id if project_id is None
+        """
+        return project_id if project_id else self.project_id
+
+    class _Decorators(object):
+        """A private inner class for keeping all decorator methods."""
+
+        @staticmethod
+        def provide_gcp_credential_file(func):
+            """
+            Function decorator that provides a GOOGLE_APPLICATION_CREDENTIALS
+            environment variable, pointing to file path of a JSON file of service
+            account key.
+            """
+            @functools.wraps(func)
+            def wrapper(self, *args, **kwargs):
+                with tempfile.NamedTemporaryFile(mode='w+t') as conf_file:
+                    key_path = self._get_field('key_path', False)
+                    keyfile_dict = self._get_field('keyfile_dict', False)
+                    if key_path:
+                        if key_path.endswith('.p12'):
+                            raise AirflowException(
+                                'Legacy P12 key file are not supported, '
+                                'use a JSON key file.')
+                        os.environ[_G_APP_CRED_ENV_VAR] = key_path
+                    elif keyfile_dict:
+                        conf_file.write(keyfile_dict)
+                        conf_file.flush()
+                        os.environ[_G_APP_CRED_ENV_VAR] = conf_file.name
+                    return func(self, *args, **kwargs)
+            return wrapper
diff --git a/airflow/contrib/hooks/gcp_bigtable_hook.py b/airflow/contrib/hooks/gcp_bigtable_hook.py
new file mode 100644
index 0000000000000..d50a95deb46c7
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_bigtable_hook.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from google.cloud.bigtable import Client
+from google.cloud.bigtable.cluster import Cluster
+from google.cloud.bigtable.instance import Instance
+from google.cloud.bigtable.table import Table
+from google.cloud.bigtable_admin_v2 import enums
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+
+class BigtableHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Cloud Bigtable APIs.
+
+    All the methods in the hook where project_id is used must be called with
+    keyword arguments rather than positional.
+    """
+
+    _client = None
+
+    def __init__(self,
+                 gcp_conn_id='google_cloud_default',
+                 delegate_to=None):
+        super(BigtableHook, self).__init__(gcp_conn_id, delegate_to)
+
+    def _get_client(self, project_id):
+        if not self._client:
+            self._client = Client(project=project_id, credentials=self._get_credentials(),
+                                  admin=True)
+        return self._client
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_instance(self, instance_id, project_id=None):
+        """
+        Retrieves and returns the specified Cloud Bigtable instance if it exists.
+        Otherwise, returns None.
+
+        :param instance_id: The ID of the Cloud Bigtable instance.
+        :type instance_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            BigTable exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        """
+
+        instance = self._get_client(project_id=project_id).instance(instance_id)
+        if not instance.exists():
+            return None
+        return instance
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_instance(self, instance_id, project_id=None):
+        """
+        Deletes the specified Cloud Bigtable instance.
+        Raises google.api_core.exceptions.NotFound if the Cloud Bigtable instance does
+        not exist.
+
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            BigTable exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :param instance_id: The ID of the Cloud Bigtable instance.
+        :type instance_id: str
+        """
+        instance = self.get_instance(instance_id=instance_id, project_id=project_id)
+        if instance:
+            instance.delete()
+        else:
+            self.log.info("The instance '%s' does not exist in project '%s'. Exiting", instance_id,
+                          project_id)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_instance(self,
+                        instance_id,
+                        main_cluster_id,
+                        main_cluster_zone,
+                        project_id=None,
+                        replica_cluster_id=None,
+                        replica_cluster_zone=None,
+                        instance_display_name=None,
+                        instance_type=enums.Instance.Type.TYPE_UNSPECIFIED,
+                        instance_labels=None,
+                        cluster_nodes=None,
+                        cluster_storage_type=enums.StorageType.STORAGE_TYPE_UNSPECIFIED,
+                        timeout=None):
+        """
+        Creates new instance.
+
+        :type instance_id: str
+        :param instance_id: The ID for the new instance.
+        :type main_cluster_id: str
+        :param main_cluster_id: The ID for main cluster for the new instance.
+        :type main_cluster_zone: str
+        :param main_cluster_zone: The zone for main cluster.
+            See https://cloud.google.com/bigtable/docs/locations for more details.
+        :type project_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            BigTable exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type replica_cluster_id: str
+        :param replica_cluster_id: (optional) The ID for replica cluster for the new
+            instance.
+        :type replica_cluster_zone: str
+        :param replica_cluster_zone: (optional)  The zone for replica cluster.
+        :type instance_type: enums.Instance.Type
+        :param instance_type: (optional) The type of the instance.
+        :type instance_display_name: str
+        :param instance_display_name: (optional) Human-readable name of the instance.
+                Defaults to ``instance_id``.
+        :type instance_labels: dict
+        :param instance_labels: (optional) Dictionary of labels to associate with the
+            instance.
+        :type cluster_nodes: int
+        :param cluster_nodes: (optional) Number of nodes for cluster.
+        :type cluster_storage_type: enums.StorageType
+        :param cluster_storage_type: (optional) The type of storage.
+        :type timeout: int
+        :param timeout: (optional) timeout (in seconds) for instance creation.
+                        If None is not specified, Operator will wait indefinitely.
+        """
+        cluster_storage_type = enums.StorageType(cluster_storage_type)
+        instance_type = enums.Instance.Type(instance_type)
+
+        instance = Instance(
+            instance_id,
+            self._get_client(project_id=project_id),
+            instance_display_name,
+            instance_type,
+            instance_labels,
+        )
+
+        clusters = [
+            instance.cluster(
+                main_cluster_id,
+                main_cluster_zone,
+                cluster_nodes,
+                cluster_storage_type
+            )
+        ]
+        if replica_cluster_id and replica_cluster_zone:
+            clusters.append(instance.cluster(
+                replica_cluster_id,
+                replica_cluster_zone,
+                cluster_nodes,
+                cluster_storage_type
+            ))
+        operation = instance.create(
+            clusters=clusters
+        )
+        operation.result(timeout)
+        return instance
+
+    @staticmethod
+    def create_table(instance,
+                     table_id,
+                     initial_split_keys=None,
+                     column_families=None):
+        """
+        Creates the specified Cloud Bigtable table.
+        Raises ``google.api_core.exceptions.AlreadyExists`` if the table exists.
+
+        :type instance: Instance
+        :param instance: The Cloud Bigtable instance that owns the table.
+        :type table_id: str
+        :param table_id: The ID of the table to create in Cloud Bigtable.
+        :type initial_split_keys: list
+        :param initial_split_keys: (Optional) A list of row keys in bytes to use to
+            initially split the table.
+        :type column_families: dict
+        :param column_families: (Optional) A map of columns to create. The key is the
+            column_id str, and the value is a
+            :class:`google.cloud.bigtable.column_family.GarbageCollectionRule`.
+        """
+        if column_families is None:
+            column_families = {}
+        if initial_split_keys is None:
+            initial_split_keys = []
+        table = Table(table_id, instance)
+        table.create(initial_split_keys, column_families)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_table(self, instance_id, table_id, project_id=None):
+        """
+        Deletes the specified table in Cloud Bigtable.
+        Raises google.api_core.exceptions.NotFound if the table does not exist.
+
+        :type instance_id: str
+        :param instance_id: The ID of the Cloud Bigtable instance.
+        :type table_id: str
+        :param table_id: The ID of the table in Cloud Bigtable.
+        :type project_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            BigTable exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        """
+        table = self.get_instance(instance_id=instance_id, project_id=project_id).table(table_id=table_id)
+        table.delete()
+
+    @staticmethod
+    def update_cluster(instance, cluster_id, nodes):
+        """
+        Updates number of nodes in the specified Cloud Bigtable cluster.
+        Raises google.api_core.exceptions.NotFound if the cluster does not exist.
+
+        :type instance: Instance
+        :param instance: The Cloud Bigtable instance that owns the cluster.
+        :type cluster_id: str
+        :param cluster_id: The ID of the cluster.
+        :type nodes: int
+        :param nodes: The desired number of nodes.
+        """
+        cluster = Cluster(cluster_id, instance)
+        cluster.serve_nodes = nodes
+        cluster.update()
+
+    @staticmethod
+    def get_column_families_for_table(instance, table_id):
+        """
+        Fetches Column Families for the specified table in Cloud Bigtable.
+
+        :type instance: Instance
+        :param instance: The Cloud Bigtable instance that owns the table.
+        :type table_id: str
+        :param table_id: The ID of the table in Cloud Bigtable to fetch Column Families
+            from.
+        """
+
+        table = Table(table_id, instance)
+        return table.list_column_families()
+
+    @staticmethod
+    def get_cluster_states_for_table(instance, table_id):
+        """
+        Fetches Cluster States for the specified table in Cloud Bigtable.
+        Raises google.api_core.exceptions.NotFound if the table does not exist.
+
+        :type instance: Instance
+        :param instance: The Cloud Bigtable instance that owns the table.
+        :type table_id: str
+        :param table_id: The ID of the table in Cloud Bigtable to fetch Cluster States
+            from.
+        """
+
+        table = Table(table_id, instance)
+        return table.get_cluster_states()
diff --git a/airflow/contrib/hooks/gcp_compute_hook.py b/airflow/contrib/hooks/gcp_compute_hook.py
new file mode 100644
index 0000000000000..02e0f56bf7697
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_compute_hook.py
@@ -0,0 +1,335 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import time
+from googleapiclient.discovery import build
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+# Number of retries - used by googleapiclient method calls to perform retries
+# For requests that are "retriable"
+NUM_RETRIES = 5
+
+# Time to sleep between active checks of the operation results
+TIME_TO_SLEEP_IN_SECONDS = 1
+
+
+class GceOperationStatus:
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    DONE = "DONE"
+
+
+# noinspection PyAbstractClass
+class GceHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Compute Engine APIs.
+
+    All the methods in the hook where project_id is used must be called with
+    keyword arguments rather than positional.
+
+    """
+    _conn = None
+
+    def __init__(self,
+                 api_version='v1',
+                 gcp_conn_id='google_cloud_default',
+                 delegate_to=None):
+        super(GceHook, self).__init__(gcp_conn_id, delegate_to)
+        self.api_version = api_version
+
+    def get_conn(self):
+        """
+        Retrieves connection to Google Compute Engine.
+
+        :return: Google Compute Engine services object
+        :rtype: dict
+        """
+        if not self._conn:
+            http_authorized = self._authorize()
+            self._conn = build('compute', self.api_version,
+                               http=http_authorized, cache_discovery=False)
+        return self._conn
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def start_instance(self, zone, resource_id, project_id=None):
+        """
+        Starts an existing instance defined by project_id, zone and resource_id.
+        Must be called with keyword arguments rather than positional.
+
+        :param zone: Google Cloud Platform zone where the instance exists
+        :type zone: str
+        :param resource_id: Name of the Compute Engine instance resource
+        :type resource_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().instances().start(
+            project=project_id,
+            zone=zone,
+            instance=resource_id
+        ).execute(num_retries=NUM_RETRIES)
+        try:
+            operation_name = response["name"]
+        except KeyError:
+            raise AirflowException(
+                "Wrong response '{}' returned - it should contain "
+                "'name' field".format(response))
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name,
+                                             zone=zone)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def stop_instance(self, zone, resource_id, project_id=None):
+        """
+        Stops an instance defined by project_id, zone and resource_id
+        Must be called with keyword arguments rather than positional.
+
+        :param zone: Google Cloud Platform zone where the instance exists
+        :type zone: str
+        :param resource_id: Name of the Compute Engine instance resource
+        :type resource_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().instances().stop(
+            project=project_id,
+            zone=zone,
+            instance=resource_id
+        ).execute(num_retries=NUM_RETRIES)
+        try:
+            operation_name = response["name"]
+        except KeyError:
+            raise AirflowException(
+                "Wrong response '{}' returned - it should contain "
+                "'name' field".format(response))
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name,
+                                             zone=zone)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def set_machine_type(self, zone, resource_id, body, project_id=None):
+        """
+        Sets machine type of an instance defined by project_id, zone and resource_id.
+        Must be called with keyword arguments rather than positional.
+
+        :param zone: Google Cloud Platform zone where the instance exists.
+        :type zone: str
+        :param resource_id: Name of the Compute Engine instance resource
+        :type resource_id: str
+        :param body: Body required by the Compute Engine setMachineType API,
+            as described in
+            https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType
+        :type body: dict
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self._execute_set_machine_type(zone, resource_id, body, project_id)
+        try:
+            operation_name = response["name"]
+        except KeyError:
+            raise AirflowException(
+                "Wrong response '{}' returned - it should contain "
+                "'name' field".format(response))
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name,
+                                             zone=zone)
+
+    def _execute_set_machine_type(self, zone, resource_id, body, project_id):
+        return self.get_conn().instances().setMachineType(
+            project=project_id, zone=zone, instance=resource_id, body=body)\
+            .execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_instance_template(self, resource_id, project_id=None):
+        """
+        Retrieves instance template by project_id and resource_id.
+        Must be called with keyword arguments rather than positional.
+
+        :param resource_id: Name of the instance template
+        :type resource_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: Instance template representation as object according to
+            https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates
+        :rtype: dict
+        """
+        response = self.get_conn().instanceTemplates().get(
+            project=project_id,
+            instanceTemplate=resource_id
+        ).execute(num_retries=NUM_RETRIES)
+        return response
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def insert_instance_template(self, body, request_id=None, project_id=None):
+        """
+        Inserts instance template using body specified
+        Must be called with keyword arguments rather than positional.
+
+        :param body: Instance template representation as object according to
+            https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates
+        :type body: dict
+        :param request_id: Optional, unique request_id that you might add to achieve
+            full idempotence (for example when client call times out repeating the request
+            with the same request id will not create a new instance template again)
+            It should be in UUID format as defined in RFC 4122
+        :type request_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().instanceTemplates().insert(
+            project=project_id,
+            body=body,
+            requestId=request_id
+        ).execute(num_retries=NUM_RETRIES)
+        try:
+            operation_name = response["name"]
+        except KeyError:
+            raise AirflowException(
+                "Wrong response '{}' returned - it should contain "
+                "'name' field".format(response))
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_instance_group_manager(self, zone, resource_id, project_id=None):
+        """
+        Retrieves Instance Group Manager by project_id, zone and resource_id.
+        Must be called with keyword arguments rather than positional.
+
+        :param zone: Google Cloud Platform zone where the Instance Group Manager exists
+        :type zone: str
+        :param resource_id: Name of the Instance Group Manager
+        :type resource_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: Instance group manager representation as object according to
+            https://cloud.google.com/compute/docs/reference/rest/beta/instanceGroupManagers
+        :rtype: dict
+        """
+        response = self.get_conn().instanceGroupManagers().get(
+            project=project_id,
+            zone=zone,
+            instanceGroupManager=resource_id
+        ).execute(num_retries=NUM_RETRIES)
+        return response
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def patch_instance_group_manager(self, zone, resource_id,
+                                     body, request_id=None, project_id=None):
+        """
+        Patches Instance Group Manager with the specified body.
+        Must be called with keyword arguments rather than positional.
+
+        :param zone: Google Cloud Platform zone where the Instance Group Manager exists
+        :type zone: str
+        :param resource_id: Name of the Instance Group Manager
+        :type resource_id: str
+        :param body: Instance Group Manager representation as json-merge-patch object
+            according to
+            https://cloud.google.com/compute/docs/reference/rest/beta/instanceTemplates/patch
+        :type body: dict
+        :param request_id: Optional, unique request_id that you might add to achieve
+            full idempotence (for example when client call times out repeating the request
+            with the same request id will not create a new instance template again).
+            It should be in UUID format as defined in RFC 4122
+        :type request_id: str
+        :param project_id: Optional, Google Cloud Platform project ID where the
+            Compute Engine Instance exists. If set to None or missing,
+            the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().instanceGroupManagers().patch(
+            project=project_id,
+            zone=zone,
+            instanceGroupManager=resource_id,
+            body=body,
+            requestId=request_id
+        ).execute(num_retries=NUM_RETRIES)
+        try:
+            operation_name = response["name"]
+        except KeyError:
+            raise AirflowException(
+                "Wrong response '{}' returned - it should contain "
+                "'name' field".format(response))
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name,
+                                             zone=zone)
+
+    def _wait_for_operation_to_complete(self, project_id, operation_name, zone=None):
+        """
+        Waits for the named operation to complete - checks status of the async call.
+
+        :param operation_name: name of the operation
+        :type operation_name: str
+        :param zone: optional region of the request (might be None for global operations)
+        :type zone: str
+        :return: None
+        """
+        service = self.get_conn()
+        while True:
+            if zone is None:
+                # noinspection PyTypeChecker
+                operation_response = self._check_global_operation_status(
+                    service, operation_name, project_id)
+            else:
+                # noinspection PyTypeChecker
+                operation_response = self._check_zone_operation_status(
+                    service, operation_name, project_id, zone)
+            if operation_response.get("status") == GceOperationStatus.DONE:
+                error = operation_response.get("error")
+                if error:
+                    code = operation_response.get("httpErrorStatusCode")
+                    msg = operation_response.get("httpErrorMessage")
+                    # Extracting the errors list as string and trimming square braces
+                    error_msg = str(error.get("errors"))[1:-1]
+                    raise AirflowException("{} {}: ".format(code, msg) + error_msg)
+                # No meaningful info to return from the response in case of success
+                return
+            time.sleep(TIME_TO_SLEEP_IN_SECONDS)
+
+    @staticmethod
+    def _check_zone_operation_status(service, operation_name, project_id, zone):
+        return service.zoneOperations().get(
+            project=project_id, zone=zone, operation=operation_name).execute(
+            num_retries=NUM_RETRIES)
+
+    @staticmethod
+    def _check_global_operation_status(service, operation_name, project_id):
+        return service.globalOperations().get(
+            project=project_id, operation=operation_name).execute(
+            num_retries=NUM_RETRIES)
diff --git a/airflow/contrib/hooks/gcp_container_hook.py b/airflow/contrib/hooks/gcp_container_hook.py
index d36d796d764a5..909cc6338377b 100644
--- a/airflow/contrib/hooks/gcp_container_hook.py
+++ b/airflow/contrib/hooks/gcp_container_hook.py
@@ -21,9 +21,9 @@
 import time
 
 from airflow import AirflowException, version
-from airflow.hooks.base_hook import BaseHook
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 
-from google.api_core.exceptions import AlreadyExists
+from google.api_core.exceptions import AlreadyExists, NotFound
 from google.api_core.gapic_v1.method import DEFAULT
 from google.cloud import container_v1, exceptions
 from google.cloud.container_v1.gapic.enums import Operation
@@ -34,19 +34,30 @@
 OPERATIONAL_POLL_INTERVAL = 15
 
 
-class GKEClusterHook(BaseHook):
+class GKEClusterHook(GoogleCloudBaseHook):
 
-    def __init__(self, project_id, location):
-        self.project_id = project_id
+    def __init__(self,
+                 gcp_conn_id='google_cloud_default',
+                 delegate_to=None,
+                 location=None):
+        super(GKEClusterHook, self).__init__(
+            gcp_conn_id=gcp_conn_id, delegate_to=delegate_to)
+        self._client = None
         self.location = location
 
-        # Add client library info for better error tracking
-        client_info = ClientInfo(client_library_version='airflow_v' + version.version)
-        self.client = container_v1.ClusterManagerClient(client_info=client_info)
+    def get_client(self):
+        if self._client is None:
+            credentials = self._get_credentials()
+            # Add client library info for better error tracking
+            client_info = ClientInfo(client_library_version='airflow_v' + version.version)
+            self._client = container_v1.ClusterManagerClient(credentials=credentials, client_info=client_info)
+        return self._client
 
-    def _dict_to_proto(self, py_dict, proto):
+    @staticmethod
+    def _dict_to_proto(py_dict, proto):
         """
         Converts a python dictionary to the proto supplied
+
         :param py_dict: The dictionary to convert
         :type py_dict: dict
         :param proto: The proto object to merge with dictionary
@@ -58,15 +69,18 @@ def _dict_to_proto(self, py_dict, proto):
         dict_json_str = json.dumps(py_dict)
         return json_format.Parse(dict_json_str, proto)
 
-    def wait_for_operation(self, operation):
+    def wait_for_operation(self, operation, project_id=None):
         """
         Given an operation, continuously fetches the status from Google Cloud until either
         completion or an error occurring
+
         :param operation: The Operation to wait for
-        :type operation: A google.cloud.container_V1.gapic.enums.Operator
+        :type operation: google.cloud.container_V1.gapic.enums.Operation
+        :param project_id: Google Cloud Platform project ID
+        :type project_id: str
         :return: A new, updated operation fetched from Google Cloud
         """
-        self.log.info("Waiting for OPERATION_NAME %s" % operation.name)
+        self.log.info("Waiting for OPERATION_NAME %s", operation.name)
         time.sleep(OPERATIONAL_POLL_INTERVAL)
         while operation.status != Operation.Status.DONE:
             if operation.status == Operation.Status.RUNNING or operation.status == \
@@ -76,27 +90,33 @@ def wait_for_operation(self, operation):
                 raise exceptions.GoogleCloudError(
                     "Operation has failed with status: %s" % operation.status)
             # To update status of operation
-            operation = self.get_operation(operation.name)
+            operation = self.get_operation(operation.name, project_id=project_id or self.project_id)
         return operation
 
-    def get_operation(self, operation_name):
+    def get_operation(self, operation_name, project_id=None):
         """
         Fetches the operation from Google Cloud
+
         :param operation_name: Name of operation to fetch
         :type operation_name: str
+        :param project_id: Google Cloud Platform project ID
+        :type project_id: str
         :return: The new, updated operation from Google Cloud
         """
-        return self.client.get_operation(project_id=self.project_id,
-                                         zone=self.location,
-                                         operation_id=operation_name)
+        return self.get_client().get_operation(project_id=project_id or self.project_id,
+                                               zone=self.location,
+                                               operation_id=operation_name)
 
-    def _append_label(self, cluster_proto, key, val):
+    @staticmethod
+    def _append_label(cluster_proto, key, val):
         """
         Append labels to provided Cluster Protobuf
 
-        Labels must fit the regex [a-z]([-a-z0-9]*[a-z0-9])? (current airflow version
-        string follows semantic versioning spec: x.y.z).
-        :param cluster_proto: The proto to append resource_label airflow version to
+        Labels must fit the regex ``[a-z]([-a-z0-9]*[a-z0-9])?`` (current
+         airflow version string follows semantic versioning spec: x.y.z).
+
+        :param cluster_proto: The proto to append resource_label airflow
+            version to
         :type cluster_proto: google.cloud.container_v1.types.Cluster
         :param key: The key label
         :type key: str
@@ -108,7 +128,7 @@ def _append_label(self, cluster_proto, key, val):
         cluster_proto.resource_labels.update({key: val})
         return cluster_proto
 
-    def delete_cluster(self, name, retry=DEFAULT, timeout=DEFAULT):
+    def delete_cluster(self, name, project_id=None, retry=DEFAULT, timeout=DEFAULT):
         """
         Deletes the cluster, including the Kubernetes endpoint and all
         worker nodes. Firewalls and routes that were configured during
@@ -119,6 +139,8 @@ def delete_cluster(self, name, retry=DEFAULT, timeout=DEFAULT):
 
         :param name: The name of the cluster to delete
         :type name: str
+        :param project_id: Google Cloud Platform project ID
+        :type project_id: str
         :param retry: Retry object used to determine when/if to retry requests.
             If None is specified, requests will not be retried.
         :type retry: google.api_core.retry.Retry
@@ -129,30 +151,35 @@ def delete_cluster(self, name, retry=DEFAULT, timeout=DEFAULT):
         :return: The full url to the delete operation if successful, else None
         """
 
-        self.log.info("Deleting (project_id={}, zone={}, cluster_id={})".format(
-            self.project_id, self.location, name))
+        self.log.info(
+            "Deleting (project_id=%s, zone=%s, cluster_id=%s)", self.project_id, self.location, name
+        )
 
         try:
-            op = self.client.delete_cluster(project_id=self.project_id,
-                                            zone=self.location,
-                                            cluster_id=name,
-                                            retry=retry,
-                                            timeout=timeout)
+            op = self.get_client().delete_cluster(project_id=project_id or self.project_id,
+                                                  zone=self.location,
+                                                  cluster_id=name,
+                                                  retry=retry,
+                                                  timeout=timeout)
             op = self.wait_for_operation(op)
             # Returns server-defined url for the resource
             return op.self_link
-        except exceptions.NotFound as error:
-            self.log.info('Assuming Success: ' + error.message)
+        except NotFound as error:
+            self.log.info('Assuming Success: %s', error.message)
 
-    def create_cluster(self, cluster, retry=DEFAULT, timeout=DEFAULT):
+    def create_cluster(self, cluster, project_id=None, retry=DEFAULT, timeout=DEFAULT):
         """
         Creates a cluster, consisting of the specified number and type of Google Compute
         Engine instances.
 
-        :param cluster: A Cluster protobuf or dict. If dict is provided, it must be of
-            the same form as the protobuf message google.cloud.container_v1.types.Cluster
+        :param cluster: A Cluster protobuf or dict. If dict is provided, it must
+            be of the same form as the protobuf message
+            :class:`google.cloud.container_v1.types.Cluster`
         :type cluster: dict or google.cloud.container_v1.types.Cluster
-        :param retry: A retry object (google.api_core.retry.Retry) used to retry requests.
+        :param project_id: Google Cloud Platform project ID
+        :type project_id: str
+        :param retry: A retry object (``google.api_core.retry.Retry``) used to
+            retry requests.
             If None is specified, requests will not be retried.
         :type retry: google.api_core.retry.Retry
         :param timeout: The amount of time, in seconds, to wait for the request to
@@ -160,7 +187,7 @@ def create_cluster(self, cluster, retry=DEFAULT, timeout=DEFAULT):
             individual attempt.
         :type timeout: float
         :return: The full url to the new, or existing, cluster
-        :raises
+        :raises:
             ParseError: On JSON parsing problems when trying to convert dict
             AirflowException: cluster is not dict type nor Cluster proto type
         """
@@ -174,28 +201,31 @@ def create_cluster(self, cluster, retry=DEFAULT, timeout=DEFAULT):
 
         self._append_label(cluster, 'airflow-version', 'v' + version.version)
 
-        self.log.info("Creating (project_id={}, zone={}, cluster_name={})".format(
-            self.project_id,
-            self.location,
-            cluster.name))
+        self.log.info(
+            "Creating (project_id=%s, zone=%s, cluster_name=%s)",
+            self.project_id, self.location, cluster.name
+        )
         try:
-            op = self.client.create_cluster(project_id=self.project_id,
-                                            zone=self.location,
-                                            cluster=cluster,
-                                            retry=retry,
-                                            timeout=timeout)
+            op = self.get_client().create_cluster(project_id=project_id or self.project_id,
+                                                  zone=self.location,
+                                                  cluster=cluster,
+                                                  retry=retry,
+                                                  timeout=timeout)
             op = self.wait_for_operation(op)
 
             return op.target_link
         except AlreadyExists as error:
-            self.log.info('Assuming Success: ' + error.message)
+            self.log.info('Assuming Success: %s', error.message)
             return self.get_cluster(name=cluster.name).self_link
 
-    def get_cluster(self, name, retry=DEFAULT, timeout=DEFAULT):
+    def get_cluster(self, name, project_id=None, retry=DEFAULT, timeout=DEFAULT):
         """
         Gets details of specified cluster
+
         :param name: The name of the cluster to retrieve
         :type name: str
+        :param project_id: Google Cloud Platform project ID
+        :type project_id: str
         :param retry: A retry object used to retry requests. If None is specified,
             requests will not be retried.
         :type retry: google.api_core.retry.Retry
@@ -203,15 +233,15 @@ def get_cluster(self, name, retry=DEFAULT, timeout=DEFAULT):
             complete. Note that if retry is specified, the timeout applies to each
             individual attempt.
         :type timeout: float
-        :return: A google.cloud.container_v1.types.Cluster instance
+        :return: google.cloud.container_v1.types.Cluster
         """
-        self.log.info("Fetching cluster (project_id={}, zone={}, cluster_name={})".format(
-            self.project_id,
-            self.location,
-            name))
-
-        return self.client.get_cluster(project_id=self.project_id,
-                                       zone=self.location,
-                                       cluster_id=name,
-                                       retry=retry,
-                                       timeout=timeout).self_link
+        self.log.info(
+            "Fetching cluster (project_id=%s, zone=%s, cluster_name=%s)",
+            project_id or self.project_id, self.location, name
+        )
+
+        return self.get_client().get_cluster(project_id=project_id or self.project_id,
+                                             zone=self.location,
+                                             cluster_id=name,
+                                             retry=retry,
+                                             timeout=timeout).self_link
diff --git a/airflow/contrib/hooks/gcp_dataflow_hook.py b/airflow/contrib/hooks/gcp_dataflow_hook.py
index 7abb413a1afcf..44679b3205362 100644
--- a/airflow/contrib/hooks/gcp_dataflow_hook.py
+++ b/airflow/contrib/hooks/gcp_dataflow_hook.py
@@ -17,12 +17,13 @@
 # specific language governing permissions and limitations
 # under the License.
 import json
+import re
 import select
 import subprocess
 import time
 import uuid
 
-from apiclient.discovery import build
+from googleapiclient.discovery import build
 
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -33,12 +34,13 @@
 
 
 class _DataflowJob(LoggingMixin):
-    def __init__(self, dataflow, project_number, name, location, poll_sleep=10):
+    def __init__(self, dataflow, project_number, name, location, poll_sleep=10,
+                 job_id=None):
         self._dataflow = dataflow
         self._project_number = project_number
         self._job_name = name
         self._job_location = location
-        self._job_id = None
+        self._job_id = job_id
         self._job = self._get_job()
         self._poll_sleep = poll_sleep
 
@@ -46,7 +48,7 @@ def _get_job_id_from_name(self):
         jobs = self._dataflow.projects().locations().jobs().list(
             projectId=self._project_number,
             location=self._job_location
-        ).execute()
+        ).execute(num_retries=5)
         for job in jobs['jobs']:
             if job['name'] == self._job_name:
                 self._job_id = job['id']
@@ -54,13 +56,15 @@ def _get_job_id_from_name(self):
         return None
 
     def _get_job(self):
-        if self._job_name:
+        if self._job_id:
+            job = self._dataflow.projects().locations().jobs().get(
+                projectId=self._project_number,
+                location=self._job_location,
+                jobId=self._job_id).execute(num_retries=5)
+        elif self._job_name:
             job = self._get_job_id_from_name()
         else:
-            job = self._dataflow.projects().jobs().get(
-                projectId=self._project_number,
-                jobId=self._job_id
-            ).execute()
+            raise Exception('Missing both dataflow job ID and name.')
 
         if job and 'currentState' in job:
             self.log.info(
@@ -123,36 +127,50 @@ def __init__(self, cmd):
 
     def _line(self, fd):
         if fd == self._proc.stderr.fileno():
-            lines = self._proc.stderr.readlines()
-            for line in lines:
+            line = b''.join(self._proc.stderr.readlines())
+            if line:
                 self.log.warning(line[:-1])
-            if lines:
-                return lines[-1]
+            return line
         if fd == self._proc.stdout.fileno():
-            line = self._proc.stdout.readline()
+            line = b''.join(self._proc.stdout.readlines())
+            if line:
+                self.log.info(line[:-1])
             return line
 
     @staticmethod
     def _extract_job(line):
-        if line is not None:
-            if line.startswith("Submitted job: "):
-                return line[15:-1]
+        # Job id info: https://goo.gl/SE29y9.
+        job_id_pattern = re.compile(
+            br'.*console.cloud.google.com/dataflow.*/jobs/([a-z|0-9|A-Z|\-|\_]+).*')
+        matched_job = job_id_pattern.search(line or '')
+        if matched_job:
+            return matched_job.group(1).decode()
 
     def wait_for_done(self):
         reads = [self._proc.stderr.fileno(), self._proc.stdout.fileno()]
         self.log.info("Start waiting for DataFlow process to complete.")
-        while self._proc.poll() is None:
+        job_id = None
+        # Make sure logs are processed regardless whether the subprocess is
+        # terminated.
+        process_ends = False
+        while True:
             ret = select.select(reads, [], [], 5)
             if ret is not None:
                 for fd in ret[0]:
                     line = self._line(fd)
                     if line:
-                        self.log.debug(line[:-1])
+                        job_id = job_id or self._extract_job(line)
             else:
                 self.log.info("Waiting for DataFlow process to complete.")
-        if self._proc.returncode is not 0:
+            if process_ends:
+                break
+            if self._proc.poll() is not None:
+                # Mark process completion but allows its outputs to be consumed.
+                process_ends = True
+        if self._proc.returncode != 0:
             raise Exception("DataFlow failed with return code {}".format(
                 self._proc.returncode))
+        return job_id
 
 
 class DataFlowHook(GoogleCloudBaseHook):
@@ -166,20 +184,20 @@ def __init__(self,
 
     def get_conn(self):
         """
-        Returns a Google Cloud Storage service object.
+        Returns a Google Cloud Dataflow service object.
         """
         http_authorized = self._authorize()
         return build(
             'dataflow', 'v1b3', http=http_authorized, cache_discovery=False)
 
-    def _start_dataflow(self, task_id, variables, name,
-                        command_prefix, label_formatter):
+    @GoogleCloudBaseHook._Decorators.provide_gcp_credential_file
+    def _start_dataflow(self, variables, name, command_prefix, label_formatter):
         variables = self._set_variables(variables)
-        cmd = command_prefix + self._build_cmd(task_id, variables,
-                                               label_formatter)
-        _Dataflow(cmd).wait_for_done()
+        cmd = command_prefix + self._build_cmd(variables, label_formatter)
+        job_id = _Dataflow(cmd).wait_for_done()
         _DataflowJob(self.get_conn(), variables['project'], name,
-                     variables['region'], self.poll_sleep).wait_for_done()
+                     variables['region'],
+                     self.poll_sleep, job_id).wait_for_done()
 
     @staticmethod
     def _set_variables(variables):
@@ -189,12 +207,9 @@ def _set_variables(variables):
             variables['region'] = DEFAULT_DATAFLOW_LOCATION
         return variables
 
-    def start_java_dataflow(self, task_id, variables, dataflow, job_class=None,
+    def start_java_dataflow(self, job_name, variables, dataflow, job_class=None,
                             append_job_name=True):
-        if append_job_name:
-            name = task_id + "-" + str(uuid.uuid1())[:8]
-        else:
-            name = task_id
+        name = self._build_dataflow_job_name(job_name, append_job_name)
         variables['jobName'] = name
 
         def label_formatter(labels_dict):
@@ -202,34 +217,45 @@ def label_formatter(labels_dict):
                 json.dumps(labels_dict).replace(' ', ''))]
         command_prefix = (["java", "-cp", dataflow, job_class] if job_class
                           else ["java", "-jar", dataflow])
-        self._start_dataflow(task_id, variables, name,
-                             command_prefix, label_formatter)
+        self._start_dataflow(variables, name, command_prefix, label_formatter)
 
-    def start_template_dataflow(self, task_id, variables, parameters, dataflow_template,
+    def start_template_dataflow(self, job_name, variables, parameters, dataflow_template,
                                 append_job_name=True):
-        if append_job_name:
-            name = task_id + "-" + str(uuid.uuid1())[:8]
-        else:
-            name = task_id
+        variables = self._set_variables(variables)
+        name = self._build_dataflow_job_name(job_name, append_job_name)
         self._start_template_dataflow(
             name, variables, parameters, dataflow_template)
 
-    def start_python_dataflow(self, task_id, variables, dataflow, py_options,
+    def start_python_dataflow(self, job_name, variables, dataflow, py_options,
                               append_job_name=True):
-        if append_job_name:
-            name = task_id + "-" + str(uuid.uuid1())[:8]
-        else:
-            name = task_id
+        name = self._build_dataflow_job_name(job_name, append_job_name)
         variables['job_name'] = name
 
         def label_formatter(labels_dict):
             return ['--labels={}={}'.format(key, value)
                     for key, value in labels_dict.items()]
-        self._start_dataflow(task_id, variables, name,
-                             ["python"] + py_options + [dataflow],
+        self._start_dataflow(variables, name, ["python2"] + py_options + [dataflow],
                              label_formatter)
 
-    def _build_cmd(self, task_id, variables, label_formatter):
+    @staticmethod
+    def _build_dataflow_job_name(job_name, append_job_name=True):
+        base_job_name = str(job_name).replace('_', '-')
+
+        if not re.match(r"^[a-z]([-a-z0-9]*[a-z0-9])?$", base_job_name):
+            raise ValueError(
+                'Invalid job_name ({}); the name must consist of'
+                'only the characters [-a-z0-9], starting with a '
+                'letter and ending with a letter or number '.format(base_job_name))
+
+        if append_job_name:
+            safe_job_name = base_job_name + "-" + str(uuid.uuid4())[:8]
+        else:
+            safe_job_name = base_job_name
+
+        return safe_job_name
+
+    @staticmethod
+    def _build_cmd(variables, label_formatter):
         command = ["--runner=DataflowRunner"]
         if variables is not None:
             for attr, value in variables.items():
@@ -241,21 +267,25 @@ def _build_cmd(self, task_id, variables, label_formatter):
                     command.append("--" + attr + "=" + value)
         return command
 
-    def _start_template_dataflow(self, name, variables, parameters, dataflow_template):
+    def _start_template_dataflow(self, name, variables, parameters,
+                                 dataflow_template):
         # Builds RuntimeEnvironment from variables dictionary
         # https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment
         environment = {}
         for key in ['maxWorkers', 'zone', 'serviceAccountEmail', 'tempLocation',
-                    'bypassTempDirValidation', 'machineType']:
+                    'bypassTempDirValidation', 'machineType', 'network', 'subnetwork']:
             if key in variables:
                 environment.update({key: variables[key]})
         body = {"jobName": name,
                 "parameters": parameters,
                 "environment": environment}
         service = self.get_conn()
-        request = service.projects().templates().launch(projectId=variables['project'],
-                                                        gcsPath=dataflow_template,
-                                                        body=body)
+        request = service.projects().locations().templates().launch(
+            projectId=variables['project'],
+            location=variables['region'],
+            gcsPath=dataflow_template,
+            body=body
+        )
         response = request.execute()
         variables = self._set_variables(variables)
         _DataflowJob(self.get_conn(), variables['project'], name, variables['region'],
diff --git a/airflow/contrib/hooks/gcp_dataproc_hook.py b/airflow/contrib/hooks/gcp_dataproc_hook.py
index fc15137cbfa91..0f5d6e216b018 100644
--- a/airflow/contrib/hooks/gcp_dataproc_hook.py
+++ b/airflow/contrib/hooks/gcp_dataproc_hook.py
@@ -20,14 +20,16 @@
 import time
 import uuid
 
-from apiclient.discovery import build
+from googleapiclient.discovery import build
+from zope.deprecation import deprecation
 
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 
 class _DataProcJob(LoggingMixin):
-    def __init__(self, dataproc_api, project_id, job, region='global'):
+    def __init__(self, dataproc_api, project_id, job, region='global',
+                 job_error_states=None):
         self.dataproc_api = dataproc_api
         self.project_id = project_id
         self.region = region
@@ -36,6 +38,7 @@ def __init__(self, dataproc_api, project_id, job, region='global'):
             region=self.region,
             body=job).execute()
         self.job_id = self.job['reference']['jobId']
+        self.job_error_states = job_error_states
         self.log.info(
             'DataProc job %s is %s',
             self.job_id, str(self.job['status']['state'])
@@ -48,13 +51,11 @@ def wait_for_done(self):
                 region=self.region,
                 jobId=self.job_id).execute(num_retries=5)
             if 'ERROR' == self.job['status']['state']:
-                print(str(self.job))
                 self.log.error('DataProc job %s has errors', self.job_id)
                 self.log.error(self.job['status']['details'])
                 self.log.debug(str(self.job))
                 return False
             if 'CANCELLED' == self.job['status']['state']:
-                print(str(self.job))
                 self.log.warning('DataProc job %s is cancelled', self.job_id)
                 if 'details' in self.job['status']:
                     self.log.warning(self.job['status']['details'])
@@ -69,10 +70,14 @@ def wait_for_done(self):
             time.sleep(5)
 
     def raise_error(self, message=None):
-        if 'ERROR' == self.job['status']['state']:
-            if message is None:
-                message = "Google DataProc job has error"
-            raise Exception(message + ": " + str(self.job['status']['details']))
+        job_state = self.job['status']['state']
+        # We always consider ERROR to be an error state.
+        if (self.job_error_states and job_state in self.job_error_states) or 'ERROR' == job_state:
+            ex_message = message or ("Google DataProc job has state: %s" % job_state)
+            ex_details = (str(self.job['status']['details'])
+                          if 'details' in self.job['status']
+                          else "No details available")
+            raise Exception(ex_message + ": " + ex_details)
 
     def get(self):
         return self.job
@@ -80,7 +85,7 @@ def get(self):
 
 class _DataProcJobBuilder:
     def __init__(self, project_id, task_id, cluster_name, job_type, properties):
-        name = task_id + "_" + str(uuid.uuid1())[:8]
+        name = task_id + "_" + str(uuid.uuid4())[:8]
         self.job_type = job_type
         self.job = {
             "job": {
@@ -140,7 +145,7 @@ def set_python_main(self, main):
         self.job["job"][self.job_type]["mainPythonFileUri"] = main
 
     def set_job_name(self, name):
-        self.job["job"]["reference"]["jobId"] = name + "_" + str(uuid.uuid1())[:8]
+        self.job["job"]["reference"]["jobId"] = name + "_" + str(uuid.uuid4())[:8]
 
     def build(self):
         return self.job
@@ -215,16 +220,26 @@ def get_cluster(self, project_id, region, cluster_name):
             clusterName=cluster_name
         ).execute(num_retries=5)
 
-    def submit(self, project_id, job, region='global'):
-        submitted = _DataProcJob(self.get_conn(), project_id, job, region)
+    def submit(self, project_id, job, region='global', job_error_states=None):
+        submitted = _DataProcJob(self.get_conn(), project_id, job, region,
+                                 job_error_states=job_error_states)
         if not submitted.wait_for_done():
-            submitted.raise_error('DataProcTask has errors')
+            submitted.raise_error()
 
     def create_job_template(self, task_id, cluster_name, job_type, properties):
         return _DataProcJobBuilder(self.project_id, task_id, cluster_name,
                                    job_type, properties)
 
-    def await(self, operation):
+    def wait(self, operation):
         """Awaits for Google Cloud Dataproc Operation to complete."""
         submitted = _DataProcOperation(self.get_conn(), operation)
         submitted.wait_for_done()
+
+
+setattr(
+    DataProcHook,
+    "await",
+    deprecation.deprecated(
+        DataProcHook.wait, "renamed to 'wait' for Python3.7 compatibility"
+    ),
+)
diff --git a/airflow/contrib/hooks/gcp_function_hook.py b/airflow/contrib/hooks/gcp_function_hook.py
new file mode 100644
index 0000000000000..86a82dd7607c7
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_function_hook.py
@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import time
+import requests
+from googleapiclient.discovery import build
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+# Number of retries - used by googleapiclient method calls to perform retries
+# For requests that are "retriable"
+NUM_RETRIES = 5
+
+# Time to sleep between active checks of the operation results
+TIME_TO_SLEEP_IN_SECONDS = 1
+
+
+# noinspection PyAbstractClass
+class GcfHook(GoogleCloudBaseHook):
+    """
+    Hook for the Google Cloud Functions APIs.
+
+    All the methods in the hook where project_id is used must be called with
+    keyword arguments rather than positional.
+    """
+    _conn = None
+
+    def __init__(self,
+                 api_version,
+                 gcp_conn_id='google_cloud_default',
+                 delegate_to=None):
+        super(GcfHook, self).__init__(gcp_conn_id, delegate_to)
+        self.api_version = api_version
+
+    @staticmethod
+    def _full_location(project_id, location):
+        """
+        Retrieve full location of the function in the form of
+        projects/<GCP_PROJECT_ID>/locations/<GCP_LOCATION>
+
+        :param project_id: The Google Cloud Project project_id where the function belongs.
+        :type project_id: str
+        :param location: The location where the function is created.
+        :type location: str
+        :return:
+        """
+        return 'projects/{}/locations/{}'.format(project_id, location)
+
+    def get_conn(self):
+        """
+        Retrieves the connection to Cloud Functions.
+
+        :return: Google Cloud Function services object.
+        :rtype: dict
+        """
+        if not self._conn:
+            http_authorized = self._authorize()
+            self._conn = build('cloudfunctions', self.api_version,
+                               http=http_authorized, cache_discovery=False)
+        return self._conn
+
+    def get_function(self, name):
+        """
+        Returns the Cloud Function with the given name.
+
+        :param name: Name of the function.
+        :type name: str
+        :return: A Cloud Functions object representing the function.
+        :rtype: dict
+        """
+        return self.get_conn().projects().locations().functions().get(
+            name=name).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_new_function(self, location, body, project_id=None):
+        """
+        Creates a new function in Cloud Function in the location specified in the body.
+
+        :param location: The location of the function.
+        :type location: str
+        :param body: The body required by the Cloud Functions insert API.
+        :type body: dict
+        :param project_id: Optional, Google Cloud Project project_id where the function belongs.
+            If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().projects().locations().functions().create(
+            location=self._full_location(project_id, location),
+            body=body
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(operation_name=operation_name)
+
+    def update_function(self, name, body, update_mask):
+        """
+        Updates Cloud Functions according to the specified update mask.
+
+        :param name: The name of the function.
+        :type name: str
+        :param body: The body required by the cloud function patch API.
+        :type body: dict
+        :param update_mask: The update mask - array of fields that should be patched.
+        :type update_mask: [str]
+        :return: None
+        """
+        response = self.get_conn().projects().locations().functions().patch(
+            updateMask=",".join(update_mask),
+            name=name,
+            body=body
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def upload_function_zip(self, location, zip_path, project_id=None):
+        """
+        Uploads zip file with sources.
+
+        :param location: The location where the function is created.
+        :type location: str
+        :param zip_path: The path of the valid .zip file to upload.
+        :type zip_path: str
+        :param project_id: Optional, Google Cloud Project project_id where the function belongs.
+            If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: The upload URL that was returned by generateUploadUrl method.
+        """
+        response = self.get_conn().projects().locations().functions().generateUploadUrl(
+            parent=self._full_location(project_id, location)
+        ).execute(num_retries=NUM_RETRIES)
+        upload_url = response.get('uploadUrl')
+        with open(zip_path, 'rb') as fp:
+            requests.put(
+                url=upload_url,
+                data=fp,
+                # Those two headers needs to be specified according to:
+                # https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/generateUploadUrl
+                # nopep8
+                headers={
+                    'Content-type': 'application/zip',
+                    'x-goog-content-length-range': '0,104857600',
+                }
+            )
+        return upload_url
+
+    def delete_function(self, name):
+        """
+        Deletes the specified Cloud Function.
+
+        :param name: The name of the function.
+        :type name: str
+        :return: None
+        """
+        response = self.get_conn().projects().locations().functions().delete(
+            name=name).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(operation_name=operation_name)
+
+    def _wait_for_operation_to_complete(self, operation_name):
+        """
+        Waits for the named operation to complete - checks status of the
+        asynchronous call.
+
+        :param operation_name: The name of the operation.
+        :type operation_name: str
+        :return: The response returned by the operation.
+        :rtype: dict
+        :exception: AirflowException in case error is returned.
+        """
+        service = self.get_conn()
+        while True:
+            operation_response = service.operations().get(
+                name=operation_name,
+            ).execute(num_retries=NUM_RETRIES)
+            if operation_response.get("done"):
+                response = operation_response.get("response")
+                error = operation_response.get("error")
+                # Note, according to documentation always either response or error is
+                # set when "done" == True
+                if error:
+                    raise AirflowException(str(error))
+                return response
+            time.sleep(TIME_TO_SLEEP_IN_SECONDS)
diff --git a/airflow/contrib/hooks/gcp_kms_hook.py b/airflow/contrib/hooks/gcp_kms_hook.py
new file mode 100644
index 0000000000000..138cc173ed973
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_kms_hook.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import base64
+
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+from googleapiclient.discovery import build
+
+
+def _b64encode(s):
+    """ Base 64 encodes a bytes object to a string """
+    return base64.b64encode(s).decode('ascii')
+
+
+def _b64decode(s):
+    """ Base 64 decodes a string to bytes. """
+    return base64.b64decode(s.encode('utf-8'))
+
+
+class GoogleCloudKMSHook(GoogleCloudBaseHook):
+    """
+    Interact with Google Cloud KMS. This hook uses the Google Cloud Platform
+    connection.
+    """
+
+    def __init__(self, gcp_conn_id='google_cloud_default', delegate_to=None):
+        super(GoogleCloudKMSHook, self).__init__(gcp_conn_id, delegate_to=delegate_to)
+
+    def get_conn(self):
+        """
+        Returns a KMS service object.
+
+        :rtype: googleapiclient.discovery.Resource
+        """
+        http_authorized = self._authorize()
+        return build(
+            'cloudkms', 'v1', http=http_authorized, cache_discovery=False)
+
+    def encrypt(self, key_name, plaintext, authenticated_data=None):
+        """
+        Encrypts a plaintext message using Google Cloud KMS.
+
+        :param key_name: The Resource Name for the key (or key version)
+                         to be used for encyption. Of the form
+                         ``projects/*/locations/*/keyRings/*/cryptoKeys/**``
+        :type key_name: str
+        :param plaintext: The message to be encrypted.
+        :type plaintext: bytes
+        :param authenticated_data: Optional additional authenticated data that
+                                   must also be provided to decrypt the message.
+        :type authenticated_data: bytes
+        :return: The base 64 encoded ciphertext of the original message.
+        :rtype: str
+        """
+        keys = self.get_conn().projects().locations().keyRings().cryptoKeys()
+        body = {'plaintext': _b64encode(plaintext)}
+        if authenticated_data:
+            body['additionalAuthenticatedData'] = _b64encode(authenticated_data)
+
+        request = keys.encrypt(name=key_name, body=body)
+        response = request.execute()
+
+        ciphertext = response['ciphertext']
+        return ciphertext
+
+    def decrypt(self, key_name, ciphertext, authenticated_data=None):
+        """
+        Decrypts a ciphertext message using Google Cloud KMS.
+
+        :param key_name: The Resource Name for the key to be used for decyption.
+                         Of the form ``projects/*/locations/*/keyRings/*/cryptoKeys/**``
+        :type key_name: str
+        :param ciphertext: The message to be decrypted.
+        :type ciphertext: str
+        :param authenticated_data: Any additional authenticated data that was
+                                   provided when encrypting the message.
+        :type authenticated_data: bytes
+        :return: The original message.
+        :rtype: bytes
+        """
+        keys = self.get_conn().projects().locations().keyRings().cryptoKeys()
+        body = {'ciphertext': ciphertext}
+        if authenticated_data:
+            body['additionalAuthenticatedData'] = _b64encode(authenticated_data)
+
+        request = keys.decrypt(name=key_name, body=body)
+        response = request.execute()
+
+        plaintext = _b64decode(response['plaintext'])
+        return plaintext
diff --git a/airflow/contrib/hooks/gcp_mlengine_hook.py b/airflow/contrib/hooks/gcp_mlengine_hook.py
index 66f392b1564c9..715e82d47d5de 100644
--- a/airflow/contrib/hooks/gcp_mlengine_hook.py
+++ b/airflow/contrib/hooks/gcp_mlengine_hook.py
@@ -15,8 +15,8 @@
 # limitations under the License.
 import random
 import time
-from apiclient import errors
-from apiclient.discovery import build
+from googleapiclient.errors import HttpError
+from googleapiclient.discovery import build
 
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -37,7 +37,7 @@ def _poll_with_exponential_delay(request, max_n, is_done_func, is_error_func):
                 return response
             else:
                 time.sleep((2**i) + (random.randint(0, 1000) / 1000))
-        except errors.HttpError as e:
+        except HttpError as e:
             if e.resp.status != 429:
                 log.info('Something went wrong. Not retrying: %s', format(e))
                 raise
@@ -63,10 +63,11 @@ def create_job(self, project_id, job, use_existing_job_fn=None):
 
         :param project_id: The Google Cloud project id within which MLEngine
             job will be launched.
-        :type project_id: string
+        :type project_id: str
 
         :param job: MLEngine Job object that should be provided to the MLEngine
             API, such as: ::
+
                 {
                   'jobId': 'my_job_id',
                   'trainingInput': {
@@ -74,6 +75,7 @@ def create_job(self, project_id, job, use_existing_job_fn=None):
                     ...
                   }
                 }
+
         :type job: dict
 
         :param use_existing_job_fn: In case that a MLEngine job with the same
@@ -96,7 +98,7 @@ def create_job(self, project_id, job, use_existing_job_fn=None):
 
         try:
             request.execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             # 409 means there is an existing job with the same job ID.
             if e.resp.status == 409:
                 if use_existing_job_fn is not None:
@@ -126,14 +128,14 @@ def _get_job(self, project_id, job_id):
         :rtype: dict
 
         Raises:
-            apiclient.errors.HttpError: if HTTP error is returned from server
+            googleapiclient.errors.HttpError: if HTTP error is returned from server
         """
         job_name = 'projects/{}/jobs/{}'.format(project_id, job_id)
         request = self._mlengine.projects().jobs().get(name=job_name)
         while True:
             try:
                 return request.execute()
-            except errors.HttpError as e:
+            except HttpError as e:
                 if e.resp.status == 429:
                     # polling after 30 seconds when quota failure occurs
                     time.sleep(30)
@@ -149,10 +151,11 @@ def _wait_for_job_done(self, project_id, job_id, interval=30):
         a terminal state.
 
         Raises:
-            apiclient.errors.HttpError: if HTTP error is returned when getting
+            googleapiclient.errors.HttpError: if HTTP error is returned when getting
             the job
         """
-        assert interval > 0
+        if interval <= 0:
+            raise ValueError("Interval must be > 0")
         while True:
             job = self._get_job(project_id, job_id)
             if job['state'] in ['SUCCEEDED', 'FAILED', 'CANCELLED']:
@@ -192,7 +195,7 @@ def set_default_version(self, project_id, model_name, version_name):
             response = request.execute()
             self.log.info('Successfully set version: %s to default', response)
             return response
-        except errors.HttpError as e:
+        except HttpError as e:
             self.log.error('Something went wrong: %s', e)
             raise
 
@@ -242,7 +245,9 @@ def create_model(self, project_id, model):
         """
         Create a Model. Blocks until finished.
         """
-        assert model['name'] is not None and model['name'] is not ''
+        if not model['name']:
+            raise ValueError("Model name must be provided and "
+                             "could not be an empty string")
         project = 'projects/{}'.format(project_id)
 
         request = self._mlengine.projects().models().create(
@@ -253,13 +258,15 @@ def get_model(self, project_id, model_name):
         """
         Gets a Model. Blocks until finished.
         """
-        assert model_name is not None and model_name is not ''
+        if not model_name:
+            raise ValueError("Model name must be provided and "
+                             "it could not be an empty string")
         full_model_name = 'projects/{}/models/{}'.format(
             project_id, model_name)
         request = self._mlengine.projects().models().get(name=full_model_name)
         try:
             return request.execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             if e.resp.status == 404:
                 self.log.error('Model was not found: %s', e)
                 return None
diff --git a/airflow/contrib/hooks/gcp_natural_language_hook.py b/airflow/contrib/hooks/gcp_natural_language_hook.py
new file mode 100644
index 0000000000000..c5fc935854256
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_natural_language_hook.py
@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from google.cloud.language_v1 import LanguageServiceClient
+
+
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+
+# noinspection PyAbstractClass
+class CloudNaturalLanguageHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Cloud Natural Language Service.
+
+    :param gcp_conn_id: The connection ID to use when fetching connection info.
+    :type gcp_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide delegation enabled.
+    :type delegate_to: str
+    """
+
+    _conn = None
+
+    def __init__(self, gcp_conn_id="google_cloud_default", delegate_to=None):
+        super(CloudNaturalLanguageHook, self).__init__(gcp_conn_id, delegate_to)
+
+    def get_conn(self):
+        """
+        Retrieves connection to Cloud Natural Language service.
+
+        :return: Cloud Natural Language service object
+        :rtype: google.cloud.language_v1.LanguageServiceClient
+        """
+        if not self._conn:
+            self._conn = LanguageServiceClient(credentials=self._get_credentials())
+        return self._conn
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def analyze_entities(self, document, encoding_type=None, retry=None, timeout=None, metadata=None):
+        """
+        Finds named entities in the text along with entity types,
+        salience, mentions for each entity, and other properties.
+
+        :param document: Input document.
+            If a dict is provided, it must be of the same form as the protobuf message Document
+        :type document: dict or class google.cloud.language_v1.types.Document
+        :param encoding_type: The encoding type used by the API to calculate offsets.
+        :type encoding_type: google.cloud.language_v1.types.EncodingType
+        :param retry: A retry object used to retry requests. If None is specified, requests will not be
+            retried.
+        :type retry: google.api_core.retry.Retry
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            retry is specified, the timeout applies to each individual attempt.
+        :type timeout: float
+        :param metadata: Additional metadata that is provided to the method.
+        :type metadata: sequence[tuple[str, str]]]
+        :rtype: google.cloud.language_v1.types.AnalyzeEntitiesResponse
+        """
+        client = self.get_conn()
+
+        return client.analyze_entities(
+            document=document, encoding_type=encoding_type, retry=retry, timeout=timeout, metadata=metadata
+        )
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def analyze_entity_sentiment(self, document, encoding_type=None, retry=None, timeout=None, metadata=None):
+        """
+        Finds entities, similar to AnalyzeEntities in the text and analyzes sentiment associated with each
+        entity and its mentions.
+
+        :param document: Input document.
+            If a dict is provided, it must be of the same form as the protobuf message Document
+        :type document: dict or class google.cloud.language_v1.types.Document
+        :param encoding_type: The encoding type used by the API to calculate offsets.
+        :type encoding_type: google.cloud.language_v1.types.EncodingType
+        :param retry: A retry object used to retry requests. If None is specified, requests will not be
+            retried.
+        :type retry: google.api_core.retry.Retry
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            retry is specified, the timeout applies to each individual attempt.
+        :type timeout: float
+        :param metadata: Additional metadata that is provided to the method.
+        :type metadata: sequence[tuple[str, str]]]
+        :rtype: google.cloud.language_v1.types.AnalyzeEntitiesResponse
+        """
+        client = self.get_conn()
+
+        return client.analyze_entity_sentiment(
+            document=document, encoding_type=encoding_type, retry=retry, timeout=timeout, metadata=metadata
+        )
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def analyze_sentiment(self, document, encoding_type=None, retry=None, timeout=None, metadata=None):
+        """
+        Analyzes the sentiment of the provided text.
+
+        :param document: Input document.
+            If a dict is provided, it must be of the same form as the protobuf message Document
+        :type document: dict or class google.cloud.language_v1.types.Document
+        :param encoding_type: The encoding type used by the API to calculate offsets.
+        :type encoding_type: google.cloud.language_v1.types.EncodingType
+        :param retry: A retry object used to retry requests. If None is specified, requests will not be
+            retried.
+        :type retry: google.api_core.retry.Retry
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            retry is specified, the timeout applies to each individual attempt.
+        :type timeout: float
+        :param metadata: Additional metadata that is provided to the method.
+        :type metadata: sequence[tuple[str, str]]]
+        :rtype: google.cloud.language_v1.types.AnalyzeEntitiesResponse
+        """
+        client = self.get_conn()
+
+        return client.analyze_sentiment(
+            document=document, encoding_type=encoding_type, retry=retry, timeout=timeout, metadata=metadata
+        )
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def analyze_syntax(self, document, encoding_type=None, retry=None, timeout=None, metadata=None):
+        """
+        Analyzes the syntax of the text and provides sentence boundaries and tokenization along with part
+        of speech tags, dependency trees, and other properties.
+
+        :param document: Input document.
+            If a dict is provided, it must be of the same form as the protobuf message Document
+        :type document: dict or class google.cloud.language_v1.types.Document#
+        :param encoding_type: The encoding type used by the API to calculate offsets.
+        :type encoding_type: google.cloud.language_v1.types.EncodingType
+        :param retry: A retry object used to retry requests. If None is specified, requests will not be
+            retried.
+        :type retry: google.api_core.retry.Retry
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            retry is specified, the timeout applies to each individual attempt.
+        :type timeout: float
+        :param metadata: Additional metadata that is provided to the method.
+        :type metadata: sequence[tuple[str, str]]]
+        :rtype: google.cloud.language_v1.types.AnalyzeSyntaxResponse
+        """
+        client = self.get_conn()
+
+        return client.analyze_syntax(
+            document=document, encoding_type=encoding_type, retry=retry, timeout=timeout, metadata=metadata
+        )
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def annotate_text(self, document, features, encoding_type=None, retry=None, timeout=None, metadata=None):
+        """
+        A convenience method that provides all the features that analyzeSentiment,
+        analyzeEntities, and analyzeSyntax provide in one call.
+
+        :param document: Input document.
+            If a dict is provided, it must be of the same form as the protobuf message Document
+        :type document: dict or google.cloud.language_v1.types.Document
+        :param features: The enabled features.
+            If a dict is provided, it must be of the same form as the protobuf message Features
+        :type features: dict or google.cloud.language_v1.enums.Features
+        :param encoding_type: The encoding type used by the API to calculate offsets.
+        :type encoding_type: google.cloud.language_v1.types.EncodingType
+        :param retry: A retry object used to retry requests. If None is specified, requests will not be
+            retried.
+        :type retry: google.api_core.retry.Retry
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            retry is specified, the timeout applies to each individual attempt.
+        :type timeout: float
+        :param metadata: Additional metadata that is provided to the method.
+        :type metadata: sequence[tuple[str, str]]]
+        :rtype: google.cloud.language_v1.types.AnnotateTextResponse
+        """
+        client = self.get_conn()
+
+        return client.annotate_text(
+            document=document,
+            features=features,
+            encoding_type=encoding_type,
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def classify_text(self, document, retry=None, timeout=None, metadata=None):
+        """
+        Classifies a document into categories.
+
+        :param document: Input document.
+            If a dict is provided, it must be of the same form as the protobuf message Document
+        :type document: dict or class google.cloud.language_v1.types.Document
+        :param retry: A retry object used to retry requests. If None is specified, requests will not be
+            retried.
+        :type retry: google.api_core.retry.Retry
+        :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+            retry is specified, the timeout applies to each individual attempt.
+        :type timeout: float
+        :param metadata: Additional metadata that is provided to the method.
+        :type metadata: sequence[tuple[str, str]]]
+        :rtype: google.cloud.language_v1.types.AnalyzeEntitiesResponse
+        """
+        client = self.get_conn()
+
+        return client.classify_text(document=document, retry=retry, timeout=timeout, metadata=metadata)
diff --git a/airflow/contrib/hooks/gcp_pubsub_hook.py b/airflow/contrib/hooks/gcp_pubsub_hook.py
index 1d55d5d487237..50512d2127463 100644
--- a/airflow/contrib/hooks/gcp_pubsub_hook.py
+++ b/airflow/contrib/hooks/gcp_pubsub_hook.py
@@ -19,8 +19,8 @@
 
 from uuid import uuid4
 
-from apiclient.discovery import build
-from apiclient import errors
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 
@@ -50,7 +50,7 @@ def __init__(self, gcp_conn_id='google_cloud_default', delegate_to=None):
     def get_conn(self):
         """Returns a Pub/Sub service object.
 
-        :rtype: apiclient.discovery.Resource
+        :rtype: googleapiclient.discovery.Resource
         """
         http_authorized = self._authorize()
         return build(
@@ -60,10 +60,10 @@ def publish(self, project, topic, messages):
         """Publishes messages to a Pub/Sub topic.
 
         :param project: the GCP project ID in which to publish
-        :type project: string
+        :type project: str
         :param topic: the Pub/Sub topic to which to publish; do not
             include the ``projects/{project}/topics/`` prefix.
-        :type topic: string
+        :type topic: str
         :param messages: messages to publish; if the data field in a
             message is set, it should already be base64 encoded.
         :type messages: list of PubSub messages; see
@@ -75,7 +75,7 @@ def publish(self, project, topic, messages):
             topic=full_topic, body=body)
         try:
             request.execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             raise PubSubException(
                 'Error publishing to topic {}'.format(full_topic), e)
 
@@ -84,10 +84,10 @@ def create_topic(self, project, topic, fail_if_exists=False):
 
         :param project: the GCP project ID in which to create
             the topic
-        :type project: string
+        :type project: str
         :param topic: the Pub/Sub topic name to create; do not
             include the ``projects/{project}/topics/`` prefix.
-        :type topic: string
+        :type topic: str
         :param fail_if_exists: if set, raise an exception if the topic
             already exists
         :type fail_if_exists: bool
@@ -97,7 +97,7 @@ def create_topic(self, project, topic, fail_if_exists=False):
         try:
             service.projects().topics().create(
                 name=full_topic, body={}).execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             # Status code 409 indicates that the topic already exists.
             if str(e.resp['status']) == '409':
                 message = 'Topic already exists: {}'.format(full_topic)
@@ -112,10 +112,10 @@ def delete_topic(self, project, topic, fail_if_not_exists=False):
         """Deletes a Pub/Sub topic if it exists.
 
         :param project: the GCP project ID in which to delete the topic
-        :type project: string
+        :type project: str
         :param topic: the Pub/Sub topic name to delete; do not
             include the ``projects/{project}/topics/`` prefix.
-        :type topic: string
+        :type topic: str
         :param fail_if_not_exists: if set, raise an exception if the topic
             does not exist
         :type fail_if_not_exists: bool
@@ -124,7 +124,7 @@ def delete_topic(self, project, topic, fail_if_not_exists=False):
         full_topic = _format_topic(project, topic)
         try:
             service.projects().topics().delete(topic=full_topic).execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             # Status code 409 indicates that the topic was not found
             if str(e.resp['status']) == '404':
                 message = 'Topic does not exist: {}'.format(full_topic)
@@ -142,17 +142,17 @@ def create_subscription(self, topic_project, topic, subscription=None,
 
         :param topic_project: the GCP project ID of the topic that the
             subscription will be bound to.
-        :type topic_project: string
+        :type topic_project: str
         :param topic: the Pub/Sub topic name that the subscription will be bound
             to create; do not include the ``projects/{project}/subscriptions/``
             prefix.
-        :type topic: string
+        :type topic: str
         :param subscription: the Pub/Sub subscription name. If empty, a random
             name will be generated using the uuid module
-        :type subscription: string
+        :type subscription: str
         :param subscription_project: the GCP project ID where the subscription
             will be created. If unspecified, ``topic_project`` will be used.
-        :type subscription_project: string
+        :type subscription_project: str
         :param ack_deadline_secs: Number of seconds that a subscriber has to
             acknowledge each message pulled from the subscription
         :type ack_deadline_secs: int
@@ -161,7 +161,7 @@ def create_subscription(self, topic_project, topic, subscription=None,
         :type fail_if_exists: bool
         :return: subscription name which will be the system-generated value if
             the ``subscription`` parameter is not supplied
-        :rtype: string
+        :rtype: str
         """
         service = self.get_conn()
         full_topic = _format_topic(topic_project, topic)
@@ -178,7 +178,7 @@ def create_subscription(self, topic_project, topic, subscription=None,
         try:
             service.projects().subscriptions().create(
                 name=full_subscription, body=body).execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             # Status code 409 indicates that the subscription already exists.
             if str(e.resp['status']) == '409':
                 message = 'Subscription already exists: {}'.format(
@@ -197,10 +197,10 @@ def delete_subscription(self, project, subscription,
         """Deletes a Pub/Sub subscription, if it exists.
 
         :param project: the GCP project ID where the subscription exists
-        :type project: string
+        :type project: str
         :param subscription: the Pub/Sub subscription name to delete; do not
             include the ``projects/{project}/subscriptions/`` prefix.
-        :type subscription: string
+        :type subscription: str
         :param fail_if_not_exists: if set, raise an exception if the topic
             does not exist
         :type fail_if_not_exists: bool
@@ -210,7 +210,7 @@ def delete_subscription(self, project, subscription,
         try:
             service.projects().subscriptions().delete(
                 subscription=full_subscription).execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             # Status code 404 indicates that the subscription was not found
             if str(e.resp['status']) == '404':
                 message = 'Subscription does not exist: {}'.format(
@@ -228,10 +228,10 @@ def pull(self, project, subscription, max_messages,
         """Pulls up to ``max_messages`` messages from Pub/Sub subscription.
 
         :param project: the GCP project ID where the subscription exists
-        :type project: string
+        :type project: str
         :param subscription: the Pub/Sub subscription name to pull from; do not
             include the 'projects/{project}/topics/' prefix.
-        :type subscription: string
+        :type subscription: str
         :param max_messages: The maximum number of messages to return from
             the Pub/Sub API.
         :type max_messages: int
@@ -239,11 +239,10 @@ def pull(self, project, subscription, max_messages,
             return if no messages are available. Otherwise, the request will
             block for an undisclosed, but bounded period of time
         :type return_immediately: bool
-        :return A list of Pub/Sub ReceivedMessage objects each containing
+        :return: A list of Pub/Sub ReceivedMessage objects each containing
             an ``ackId`` property and a ``message`` property, which includes
             the base64-encoded message content. See
-            https://cloud.google.com/pubsub/docs/reference/rest/v1/\
-                projects.subscriptions/pull#ReceivedMessage
+            https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull#ReceivedMessage
         """
         service = self.get_conn()
         full_subscription = _format_subscription(project, subscription)
@@ -255,7 +254,7 @@ def pull(self, project, subscription, max_messages,
             response = service.projects().subscriptions().pull(
                 subscription=full_subscription, body=body).execute()
             return response.get('receivedMessages', [])
-        except errors.HttpError as e:
+        except HttpError as e:
             raise PubSubException(
                 'Error pulling messages from subscription {}'.format(
                     full_subscription), e)
@@ -265,10 +264,10 @@ def acknowledge(self, project, subscription, ack_ids):
 
         :param project: the GCP project name or ID in which to create
             the topic
-        :type project: string
+        :type project: str
         :param subscription: the Pub/Sub subscription name to delete; do not
             include the 'projects/{project}/topics/' prefix.
-        :type subscription: string
+        :type subscription: str
         :param ack_ids: List of ReceivedMessage ackIds from a previous pull
             response
         :type ack_ids: list
@@ -279,7 +278,7 @@ def acknowledge(self, project, subscription, ack_ids):
             service.projects().subscriptions().acknowledge(
                 subscription=full_subscription, body={'ackIds': ack_ids}
             ).execute()
-        except errors.HttpError as e:
+        except HttpError as e:
             raise PubSubException(
                 'Error acknowledging {} messages pulled from subscription {}'
                 .format(len(ack_ids), full_subscription), e)
diff --git a/airflow/contrib/hooks/gcp_spanner_hook.py b/airflow/contrib/hooks/gcp_spanner_hook.py
new file mode 100644
index 0000000000000..6b442c46989f5
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_spanner_hook.py
@@ -0,0 +1,349 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from google.api_core.exceptions import GoogleAPICallError, AlreadyExists
+from google.cloud.spanner_v1.client import Client
+from google.longrunning.operations_grpc_pb2 import Operation  # noqa: F401
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+
+class CloudSpannerHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Cloud Spanner APIs.
+
+    All the methods in the hook where project_id is used must be called with
+    keyword arguments rather than positional.
+    """
+    _client = None
+
+    def __init__(self,
+                 gcp_conn_id='google_cloud_default',
+                 delegate_to=None):
+        super(CloudSpannerHook, self).__init__(gcp_conn_id, delegate_to)
+
+    def _get_client(self, project_id):
+        """
+        Provides a client for interacting with the Cloud Spanner API.
+
+        :param project_id: The ID of the  GCP project.
+        :type project_id: str
+        :return: google.cloud.spanner_v1.client.Client
+        :rtype: object
+        """
+        if not self._client:
+            self._client = Client(project=project_id, credentials=self._get_credentials())
+        return self._client
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_instance(self, instance_id, project_id=None):
+        """
+        Gets information about a particular instance.
+
+        :param project_id: Optional, The ID of the  GCP project that owns the Cloud Spanner
+            database.  If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :return: google.cloud.spanner_v1.instance.Instance
+        :rtype: object
+        """
+        instance = self._get_client(project_id=project_id).instance(instance_id=instance_id)
+        if not instance.exists():
+            return None
+        return instance
+
+    def _apply_to_instance(self, project_id, instance_id, configuration_name, node_count,
+                           display_name, func):
+        """
+        Invokes a method on a given instance by applying a specified Callable.
+
+        :param project_id: The ID of the  GCP project that owns the Cloud Spanner
+            database.
+        :type project_id: str
+        :param instance_id: The ID of the instance.
+        :type instance_id: str
+        :param configuration_name: Name of the instance configuration defining how the
+            instance will be created. Required for instances which do not yet exist.
+        :type configuration_name: str
+        :param node_count: (Optional) Number of nodes allocated to the instance.
+        :type node_count: int
+        :param display_name: (Optional) The display name for the instance in the Cloud
+            Console UI. (Must be between 4 and 30 characters.) If this value is not set
+            in the constructor, will fall back to the instance ID.
+        :type display_name: str
+        :param func: Method of the instance to be called.
+        :type func: Callable
+        """
+        # noinspection PyUnresolvedReferences
+        instance = self._get_client(project_id=project_id).instance(
+            instance_id=instance_id, configuration_name=configuration_name,
+            node_count=node_count, display_name=display_name)
+        try:
+            operation = func(instance)  # type: Operation
+        except GoogleAPICallError as e:
+            self.log.error('An error occurred: %s. Exiting.', e.message)
+            raise e
+
+        if operation:
+            result = operation.result()
+            self.log.info(result)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_instance(self, instance_id, configuration_name, node_count,
+                        display_name, project_id=None):
+        """
+        Creates a new Cloud Spanner instance.
+
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param configuration_name: The name of the instance configuration defining how the
+            instance will be created. Possible configuration values can be retrieved via
+            https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instanceConfigs/list
+        :type configuration_name: str
+        :param node_count: (Optional) The number of nodes allocated to the Cloud Spanner
+            instance.
+        :type node_count: int
+        :param display_name: (Optional) The display name for the instance in the GCP
+            Console. Must be between 4 and 30 characters.  If this value is not set in
+            the constructor, the name falls back to the instance ID.
+        :type display_name: str
+        :param project_id: Optional, the ID of the  GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        self._apply_to_instance(project_id, instance_id, configuration_name,
+                                node_count, display_name, lambda x: x.create())
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def update_instance(self, instance_id, configuration_name, node_count,
+                        display_name, project_id=None):
+        """
+        Updates an existing Cloud Spanner instance.
+
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param configuration_name: The name of the instance configuration defining how the
+            instance will be created. Possible configuration values can be retrieved via
+            https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instanceConfigs/list
+        :type configuration_name: str
+        :param node_count: (Optional) The number of nodes allocated to the Cloud Spanner
+            instance.
+        :type node_count: int
+        :param display_name: (Optional) The display name for the instance in the GCP
+            Console. Must be between 4 and 30 characters. If this value is not set in
+            the constructor, the name falls back to the instance ID.
+        :type display_name: str
+        :param project_id: Optional, the ID of the  GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        return self._apply_to_instance(project_id, instance_id, configuration_name,
+                                       node_count, display_name, lambda x: x.update())
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_instance(self, instance_id, project_id=None):
+        """
+        Deletes an existing Cloud Spanner instance.
+
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param project_id: Optional, the ID of the GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+
+        instance = self._get_client(project_id=project_id).instance(instance_id)
+        try:
+            instance.delete()
+            return
+        except GoogleAPICallError as e:
+            self.log.error('An error occurred: %s. Exiting.', e.message)
+            raise e
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_database(self, instance_id, database_id, project_id=None):
+        """
+        Retrieves a database in Cloud Spanner. If the database does not exist
+        in the specified instance, it returns None.
+
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param database_id: The ID of the database in Cloud Spanner.
+        :type database_id: str
+        :param project_id: Optional, the ID of the  GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: Database object or None if database does not exist
+        :rtype: google.cloud.spanner_v1.database.Database or None
+        """
+
+        instance = self._get_client(project_id=project_id).instance(
+            instance_id=instance_id)
+        if not instance.exists():
+            raise AirflowException("The instance {} does not exist in project {} !".
+                                   format(instance_id, project_id))
+        database = instance.database(database_id=database_id)
+        if not database.exists():
+            return None
+        else:
+            return database
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_database(self, instance_id, database_id, ddl_statements, project_id=None):
+        """
+        Creates a new database in Cloud Spanner.
+
+        :type project_id: str
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param database_id: The ID of the database to create in Cloud Spanner.
+        :type database_id: str
+        :param ddl_statements: The string list containing DDL for the new database.
+        :type ddl_statements: list[str]
+        :param project_id: Optional, the ID of the  GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :return: None
+        """
+
+        instance = self._get_client(project_id=project_id).instance(
+            instance_id=instance_id)
+        if not instance.exists():
+            raise AirflowException("The instance {} does not exist in project {} !".
+                                   format(instance_id, project_id))
+        database = instance.database(database_id=database_id,
+                                     ddl_statements=ddl_statements)
+        try:
+            operation = database.create()  # type: Operation
+        except GoogleAPICallError as e:
+            self.log.error('An error occurred: %s. Exiting.', e.message)
+            raise e
+
+        if operation:
+            result = operation.result()
+            self.log.info(result)
+        return
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def update_database(self, instance_id, database_id, ddl_statements,
+                        project_id=None,
+                        operation_id=None):
+        """
+        Updates DDL of a database in Cloud Spanner.
+
+        :type project_id: str
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param database_id: The ID of the database in Cloud Spanner.
+        :type database_id: str
+        :param ddl_statements: The string list containing DDL for the new database.
+        :type ddl_statements: list[str]
+        :param project_id: Optional, the ID of the GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :param operation_id: (Optional) The unique per database operation ID that can be
+            specified to implement idempotency check.
+        :type operation_id: str
+        :return: None
+        """
+
+        instance = self._get_client(project_id=project_id).instance(
+            instance_id=instance_id)
+        if not instance.exists():
+            raise AirflowException("The instance {} does not exist in project {} !".
+                                   format(instance_id, project_id))
+        database = instance.database(database_id=database_id)
+        try:
+            operation = database.update_ddl(
+                ddl_statements=ddl_statements, operation_id=operation_id)
+            if operation:
+                result = operation.result()
+                self.log.info(result)
+            return
+        except AlreadyExists as e:
+            if e.code == 409 and operation_id in e.message:
+                self.log.info("Replayed update_ddl message - the operation id %s "
+                              "was already done before.", operation_id)
+                return
+        except GoogleAPICallError as e:
+            self.log.error('An error occurred: %s. Exiting.', e.message)
+            raise e
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_database(self, instance_id, database_id, project_id=None):
+        """
+        Drops a database in Cloud Spanner.
+
+        :type project_id: str
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param database_id: The ID of the database in Cloud Spanner.
+        :type database_id: str
+        :param project_id: Optional, the ID of the  GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :return: True if everything succeeded
+        :rtype: bool
+        """
+
+        instance = self._get_client(project_id=project_id).\
+            instance(instance_id=instance_id)
+        if not instance.exists():
+            raise AirflowException("The instance {} does not exist in project {} !".
+                                   format(instance_id, project_id))
+        database = instance.database(database_id=database_id)
+        if not database.exists():
+            self.log.info("The database {} is already deleted from instance {}. "
+                          "Exiting.".format(database_id, instance_id))
+            return
+        try:
+            operation = database.drop()  # type: Operation
+        except GoogleAPICallError as e:
+            self.log.error('An error occurred: %s. Exiting.', e.message)
+            raise e
+
+        if operation:
+            result = operation.result()
+            self.log.info(result)
+        return
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def execute_dml(self, instance_id, database_id, queries, project_id=None):
+        """
+        Executes an arbitrary DML query (INSERT, UPDATE, DELETE).
+
+        :param instance_id: The ID of the Cloud Spanner instance.
+        :type instance_id: str
+        :param database_id: The ID of the database in Cloud Spanner.
+        :type database_id: str
+        :param queries: The queries to execute.
+        :type queries: str
+        :param project_id: Optional, the ID of the  GCP project that owns the Cloud Spanner
+            database. If set to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        """
+        self._get_client(project_id=project_id).instance(instance_id=instance_id).\
+            database(database_id=database_id).run_in_transaction(
+            lambda transaction: self._execute_sql_in_transaction(transaction, queries))
+
+    @staticmethod
+    def _execute_sql_in_transaction(transaction, queries):
+        for sql in queries:
+            transaction.execute_update(sql)
diff --git a/airflow/contrib/hooks/gcp_sql_hook.py b/airflow/contrib/hooks/gcp_sql_hook.py
new file mode 100644
index 0000000000000..b81fc81cb4766
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_sql_hook.py
@@ -0,0 +1,1000 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import errno
+import json
+import os
+import random
+import re
+import shutil
+import string
+
+import socket
+import platform
+import subprocess
+import time
+import uuid
+import os.path
+
+from googleapiclient.errors import HttpError
+from subprocess import Popen, PIPE
+from six.moves.urllib.parse import quote_plus
+
+import requests
+from googleapiclient.discovery import build
+
+from airflow import AirflowException, LoggingMixin
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+# Number of retries - used by googleapiclient method calls to perform retries
+# For requests that are "retriable"
+from airflow.hooks.base_hook import BaseHook
+from airflow.hooks.mysql_hook import MySqlHook
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.models.connection import Connection
+from airflow.utils.db import provide_session
+
+UNIX_PATH_MAX = 108
+
+NUM_RETRIES = 5
+
+# Time to sleep between active checks of the operation results
+TIME_TO_SLEEP_IN_SECONDS = 1
+
+
+class CloudSqlOperationStatus:
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    DONE = "DONE"
+    UNKNOWN = "UNKNOWN"
+
+
+# noinspection PyAbstractClass
+class CloudSqlHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Cloud SQL APIs.
+
+    All the methods in the hook where project_id is used must be called with
+    keyword arguments rather than positional.
+    """
+    _conn = None
+
+    def __init__(self,
+                 api_version,
+                 gcp_conn_id='google_cloud_default',
+                 delegate_to=None):
+        super(CloudSqlHook, self).__init__(gcp_conn_id, delegate_to)
+        self.api_version = api_version
+
+    def get_conn(self):
+        """
+        Retrieves connection to Cloud SQL.
+
+        :return: Google Cloud SQL services object.
+        :rtype: dict
+        """
+        if not self._conn:
+            http_authorized = self._authorize()
+            self._conn = build('sqladmin', self.api_version,
+                               http=http_authorized, cache_discovery=False)
+        return self._conn
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_instance(self, instance, project_id=None):
+        """
+        Retrieves a resource containing information about a Cloud SQL instance.
+
+        :param instance: Database instance ID. This does not include the project ID.
+        :type instance: str
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: A Cloud SQL instance resource.
+        :rtype: dict
+        """
+        return self.get_conn().instances().get(
+            project=project_id,
+            instance=instance
+        ).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_instance(self, body, project_id=None):
+        """
+        Creates a new Cloud SQL instance.
+
+        :param body: Body required by the Cloud SQL insert API, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert#request-body.
+        :type body: dict
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().instances().insert(
+            project=project_id,
+            body=body
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def patch_instance(self, body, instance, project_id=None):
+        """
+        Updates settings of a Cloud SQL instance.
+
+        Caution: This is not a partial update, so you must include values for
+        all the settings that you want to retain.
+
+        :param body: Body required by the Cloud SQL patch API, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch#request-body.
+        :type body: dict
+        :param instance: Cloud SQL instance ID. This does not include the project ID.
+        :type instance: str
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().instances().patch(
+            project=project_id,
+            instance=instance,
+            body=body
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_instance(self, instance, project_id=None):
+        """
+        Deletes a Cloud SQL instance.
+
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :param instance: Cloud SQL instance ID. This does not include the project ID.
+        :type instance: str
+        :return: None
+        """
+        response = self.get_conn().instances().delete(
+            project=project_id,
+            instance=instance,
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_database(self, instance, database, project_id=None):
+        """
+        Retrieves a database resource from a Cloud SQL instance.
+
+        :param instance: Database instance ID. This does not include the project ID.
+        :type instance: str
+        :param database: Name of the database in the instance.
+        :type database: str
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: A Cloud SQL database resource, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases#resource.
+        :rtype: dict
+        """
+        return self.get_conn().databases().get(
+            project=project_id,
+            instance=instance,
+            database=database
+        ).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_database(self, instance, body, project_id=None):
+        """
+        Creates a new database inside a Cloud SQL instance.
+
+        :param instance: Database instance ID. This does not include the project ID.
+        :type instance: str
+        :param body: The request body, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert#request-body.
+        :type body: dict
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().databases().insert(
+            project=project_id,
+            instance=instance,
+            body=body
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def patch_database(self, instance, database, body, project_id=None):
+        """
+        Updates a database resource inside a Cloud SQL instance.
+
+        This method supports patch semantics.
+        See https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch.
+
+        :param instance: Database instance ID. This does not include the project ID.
+        :type instance: str
+        :param database: Name of the database to be updated in the instance.
+        :type database: str
+        :param body: The request body, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert#request-body.
+        :type body: dict
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().databases().patch(
+            project=project_id,
+            instance=instance,
+            database=database,
+            body=body
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_database(self, instance, database, project_id=None):
+        """
+        Deletes a database from a Cloud SQL instance.
+
+        :param instance: Database instance ID. This does not include the project ID.
+        :type instance: str
+        :param database: Name of the database to be deleted in the instance.
+        :type database: str
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        response = self.get_conn().databases().delete(
+            project=project_id,
+            instance=instance,
+            database=database
+        ).execute(num_retries=NUM_RETRIES)
+        operation_name = response["name"]
+        self._wait_for_operation_to_complete(project_id=project_id,
+                                             operation_name=operation_name)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def export_instance(self, instance, body, project_id=None):
+        """
+        Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump
+        or CSV file.
+
+        :param instance: Database instance ID of the Cloud SQL instance. This does not include the
+            project ID.
+        :type instance: str
+        :param body: The request body, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/export#request-body
+        :type body: dict
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        try:
+            response = self.get_conn().instances().export(
+                project=project_id,
+                instance=instance,
+                body=body
+            ).execute(num_retries=NUM_RETRIES)
+            operation_name = response["name"]
+            self._wait_for_operation_to_complete(project_id=project_id,
+                                                 operation_name=operation_name)
+        except HttpError as ex:
+            raise AirflowException(
+                'Exporting instance {} failed: {}'.format(instance, ex.content)
+            )
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def import_instance(self, instance, body, project_id=None):
+        """
+        Imports data into a Cloud SQL instance from a SQL dump or CSV file in
+        Cloud Storage.
+
+        :param instance: Database instance ID. This does not include the
+            project ID.
+        :type instance: str
+        :param body: The request body, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/export#request-body
+        :type body: dict
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the GCP connection is used.
+        :type project_id: str
+        :return: None
+        """
+        try:
+            response = self.get_conn().instances().import_(
+                project=project_id,
+                instance=instance,
+                body=body
+            ).execute(num_retries=NUM_RETRIES)
+            operation_name = response["name"]
+            self._wait_for_operation_to_complete(project_id=project_id,
+                                                 operation_name=operation_name)
+        except HttpError as ex:
+            raise AirflowException(
+                'Importing instance {} failed: {}'.format(instance, ex.content)
+            )
+
+    def _wait_for_operation_to_complete(self, project_id, operation_name):
+        """
+        Waits for the named operation to complete - checks status of the
+        asynchronous call.
+
+        :param project_id: Project ID of the project that contains the instance.
+        :type project_id: str
+        :param operation_name: Name of the operation.
+        :type operation_name: str
+        :return: None
+        """
+        service = self.get_conn()
+        while True:
+            operation_response = service.operations().get(
+                project=project_id,
+                operation=operation_name,
+            ).execute(num_retries=NUM_RETRIES)
+            if operation_response.get("status") == CloudSqlOperationStatus.DONE:
+                error = operation_response.get("error")
+                if error:
+                    # Extracting the errors list as string and trimming square braces
+                    error_msg = str(error.get("errors"))[1:-1]
+                    raise AirflowException(error_msg)
+                # No meaningful info to return from the response in case of success
+                return
+            time.sleep(TIME_TO_SLEEP_IN_SECONDS)
+
+
+CLOUD_SQL_PROXY_DOWNLOAD_URL = "https://dl.google.com/cloudsql/cloud_sql_proxy.{}.{}"
+CLOUD_SQL_PROXY_VERSION_DOWNLOAD_URL = \
+    "https://storage.googleapis.com/cloudsql-proxy/{}/cloud_sql_proxy.{}.{}"
+
+GCP_CREDENTIALS_KEY_PATH = "extra__google_cloud_platform__key_path"
+GCP_CREDENTIALS_KEYFILE_DICT = "extra__google_cloud_platform__keyfile_dict"
+
+
+class CloudSqlProxyRunner(LoggingMixin):
+    """
+    Downloads and runs cloud-sql-proxy as subprocess of the Python process.
+
+    The cloud-sql-proxy needs to be downloaded and started before we can connect
+    to the Google Cloud SQL instance via database connection. It establishes
+    secure tunnel connection to the database. It authorizes using the
+    GCP credentials that are passed by the configuration.
+
+    More details about the proxy can be found here:
+    https://cloud.google.com/sql/docs/mysql/sql-proxy
+
+    """
+
+    def __init__(self,
+                 path_prefix,
+                 instance_specification,
+                 gcp_conn_id='google_cloud_default',
+                 project_id=None,
+                 sql_proxy_version=None,
+                 sql_proxy_binary_path=None):
+        """
+        Creates the proxy runner class.
+
+        :param path_prefix: Unique path prefix where proxy will be downloaded and
+            directories created for unix sockets.
+        :type path_prefix: str
+        :param instance_specification: Specification of the instance to connect the
+            proxy to. It should be specified in the form that is described in
+            https://cloud.google.com/sql/docs/mysql/sql-proxy#multiple-instances in
+            -instances parameter (typically in the form of ``<project>:<region>:<instance>``
+            for UNIX socket connections and in the form of
+            ``<project>:<region>:<instance>=tcp:<port>`` for TCP connections.
+        :type instance_specification: str
+        :param gcp_conn_id: Id of Google Cloud Platform connection to use for
+            authentication
+        :type gcp_conn_id: str
+        :param project_id: Optional id of the GCP project to connect to - it overwrites
+            default project id taken from the GCP connection.
+        :type project_id: str
+        :param sql_proxy_version: Specific version of SQL proxy to download
+            (for example 'v1.13'). By default latest version is downloaded.
+        :type sql_proxy_version: str
+        :param sql_proxy_binary_path: If specified, then proxy will be
+            used from the path specified rather than dynamically generated. This means
+            that if the binary is not present in that path it will also be downloaded.
+        :type sql_proxy_binary_path: str
+        """
+        super(CloudSqlProxyRunner, self).__init__()
+        self.path_prefix = path_prefix
+        if not self.path_prefix:
+            raise AirflowException("The path_prefix must not be empty!")
+        self.sql_proxy_was_downloaded = False
+        self.sql_proxy_version = sql_proxy_version
+        self.download_sql_proxy_dir = None
+        self.sql_proxy_process = None
+        self.instance_specification = instance_specification
+        self.project_id = project_id
+        self.gcp_conn_id = gcp_conn_id
+        self.command_line_parameters = []
+        self.cloud_sql_proxy_socket_directory = self.path_prefix
+        self.sql_proxy_path = sql_proxy_binary_path if sql_proxy_binary_path \
+            else self.path_prefix + "_cloud_sql_proxy"
+        self.credentials_path = self.path_prefix + "_credentials.json"
+        self._build_command_line_parameters()
+
+    def _build_command_line_parameters(self):
+        self.command_line_parameters.extend(
+            ['-dir', self.cloud_sql_proxy_socket_directory])
+        self.command_line_parameters.extend(
+            ['-instances', self.instance_specification])
+
+    @staticmethod
+    def _is_os_64bit():
+        return platform.machine().endswith('64')
+
+    def _download_sql_proxy_if_needed(self):
+        if os.path.isfile(self.sql_proxy_path):
+            self.log.info("cloud-sql-proxy is already present")
+            return
+        system = platform.system().lower()
+        processor = "amd64" if CloudSqlProxyRunner._is_os_64bit() else "386"
+        if not self.sql_proxy_version:
+            download_url = CLOUD_SQL_PROXY_DOWNLOAD_URL.format(system, processor)
+        else:
+            download_url = CLOUD_SQL_PROXY_VERSION_DOWNLOAD_URL.format(
+                self.sql_proxy_version, system, processor)
+        proxy_path_tmp = self.sql_proxy_path + ".tmp"
+        self.log.info("Downloading cloud_sql_proxy from %s to %s",
+                      download_url, proxy_path_tmp)
+        r = requests.get(download_url, allow_redirects=True)
+        # Downloading to .tmp file first to avoid case where partially downloaded
+        # binary is used by parallel operator which uses the same fixed binary path
+        with open(proxy_path_tmp, 'wb') as f:
+            f.write(r.content)
+        if r.status_code != 200:
+            raise AirflowException(
+                "The cloud-sql-proxy could not be downloaded. Status code = {}. "
+                "Reason = {}".format(r.status_code, r.reason))
+        self.log.info("Moving sql_proxy binary from %s to %s",
+                      proxy_path_tmp, self.sql_proxy_path)
+        shutil.move(proxy_path_tmp, self.sql_proxy_path)
+        os.chmod(self.sql_proxy_path, 0o744)  # Set executable bit
+        self.sql_proxy_was_downloaded = True
+
+    @provide_session
+    def _get_credential_parameters(self, session):
+        connection = session.query(Connection). \
+            filter(Connection.conn_id == self.gcp_conn_id).first()
+        session.expunge_all()
+        if GCP_CREDENTIALS_KEY_PATH in connection.extra_dejson:
+            credential_params = [
+                '-credential_file',
+                connection.extra_dejson[GCP_CREDENTIALS_KEY_PATH]
+            ]
+        elif GCP_CREDENTIALS_KEYFILE_DICT in connection.extra_dejson:
+            credential_file_content = json.loads(
+                connection.extra_dejson[GCP_CREDENTIALS_KEYFILE_DICT])
+            self.log.info("Saving credentials to %s", self.credentials_path)
+            with open(self.credentials_path, "w") as f:
+                json.dump(credential_file_content, f)
+            credential_params = [
+                '-credential_file',
+                self.credentials_path
+            ]
+        else:
+            self.log.info(
+                "The credentials are not supplied by neither key_path nor "
+                "keyfile_dict of the gcp connection %s. Falling back to "
+                "default activated account", self.gcp_conn_id)
+            credential_params = []
+
+        if not self.instance_specification:
+            project_id = connection.extra_dejson.get(
+                'extra__google_cloud_platform__project')
+            if self.project_id:
+                project_id = self.project_id
+            if not project_id:
+                raise AirflowException("For forwarding all instances, the project id "
+                                       "for GCP should be provided either "
+                                       "by project_id extra in the GCP connection or by "
+                                       "project_id provided in the operator.")
+            credential_params.extend(['-projects', project_id])
+        return credential_params
+
+    def start_proxy(self):
+        """
+        Starts Cloud SQL Proxy.
+
+        You have to remember to stop the proxy if you started it!
+        """
+        self._download_sql_proxy_if_needed()
+        if self.sql_proxy_process:
+            raise AirflowException("The sql proxy is already running: {}".format(
+                self.sql_proxy_process))
+        else:
+            command_to_run = [self.sql_proxy_path]
+            command_to_run.extend(self.command_line_parameters)
+            try:
+                self.log.info("Creating directory %s",
+                              self.cloud_sql_proxy_socket_directory)
+                os.makedirs(self.cloud_sql_proxy_socket_directory)
+            except OSError:
+                # Needed for python 2 compatibility (exists_ok missing)
+                pass
+            command_to_run.extend(self._get_credential_parameters())
+            self.log.info("Running the command: `%s`", " ".join(command_to_run))
+            self.sql_proxy_process = Popen(command_to_run,
+                                           stdin=PIPE, stdout=PIPE, stderr=PIPE)
+            self.log.info("The pid of cloud_sql_proxy: %s", self.sql_proxy_process.pid)
+            while True:
+                line = self.sql_proxy_process.stderr.readline().decode('utf-8')
+                return_code = self.sql_proxy_process.poll()
+                if line == '' and return_code is not None:
+                    self.sql_proxy_process = None
+                    raise AirflowException(
+                        "The cloud_sql_proxy finished early with return code {}!".format(
+                            return_code))
+                if line != '':
+                    self.log.info(line)
+                if "googleapi: Error" in line or "invalid instance name:" in line:
+                    self.stop_proxy()
+                    raise AirflowException(
+                        "Error when starting the cloud_sql_proxy {}!".format(
+                            line))
+                if "Ready for new connections" in line:
+                    return
+
+    def stop_proxy(self):
+        """
+        Stops running proxy.
+
+        You should stop the proxy after you stop using it.
+        """
+        if not self.sql_proxy_process:
+            raise AirflowException("The sql proxy is not started yet")
+        else:
+            self.log.info("Stopping the cloud_sql_proxy pid: %s",
+                          self.sql_proxy_process.pid)
+            self.sql_proxy_process.kill()
+            self.sql_proxy_process = None
+        # Cleanup!
+        self.log.info("Removing the socket directory: %s",
+                      self.cloud_sql_proxy_socket_directory)
+        shutil.rmtree(self.cloud_sql_proxy_socket_directory, ignore_errors=True)
+        if self.sql_proxy_was_downloaded:
+            self.log.info("Removing downloaded proxy: %s", self.sql_proxy_path)
+            # Silently ignore if the file has already been removed (concurrency)
+            try:
+                os.remove(self.sql_proxy_path)
+            except OSError as e:
+                if not e.errno == errno.ENOENT:
+                    raise
+        else:
+            self.log.info("Skipped removing proxy - it was not downloaded: %s",
+                          self.sql_proxy_path)
+        if os.path.isfile(self.credentials_path):
+            self.log.info("Removing generated credentials file %s",
+                          self.credentials_path)
+            # Here file cannot be delete by concurrent task (each task has its own copy)
+            os.remove(self.credentials_path)
+
+    def get_proxy_version(self):
+        """
+        Returns version of the Cloud SQL Proxy.
+        """
+        self._download_sql_proxy_if_needed()
+        command_to_run = [self.sql_proxy_path]
+        command_to_run.extend(['--version'])
+        command_to_run.extend(self._get_credential_parameters())
+        result = subprocess.check_output(command_to_run).decode('utf-8')
+        pattern = re.compile("^.*[V|v]ersion ([^;]*);.*$")
+        m = pattern.match(result)
+        if m:
+            return m.group(1)
+        else:
+            return None
+
+    def get_socket_path(self):
+        """
+        Retrieves UNIX socket path used by Cloud SQL Proxy.
+
+        :return: The dynamically generated path for the socket created by the proxy.
+        :rtype: str
+        """
+        return self.cloud_sql_proxy_socket_directory + "/" + self.instance_specification
+
+
+CONNECTION_URIS = {
+    "postgres": {
+        "proxy": {
+            "tcp":
+                "postgresql://{user}:{password}@127.0.0.1:{proxy_port}/{database}",
+            "socket":
+                "postgresql://{user}:{password}@{socket_path}/{database}"
+        },
+        "public": {
+            "ssl":
+                "postgresql://{user}:{password}@{public_ip}:{public_port}/{database}?"
+                "sslmode=verify-ca&"
+                "sslcert={client_cert_file}&"
+                "sslkey={client_key_file}&"
+                "sslrootcert={server_ca_file}",
+            "non-ssl":
+                "postgresql://{user}:{password}@{public_ip}:{public_port}/{database}"
+        }
+    },
+    "mysql": {
+        "proxy": {
+            "tcp":
+                "mysql://{user}:{password}@127.0.0.1:{proxy_port}/{database}",
+            "socket":
+                "mysql://{user}:{password}@localhost/{database}?"
+                "unix_socket={socket_path}"
+        },
+        "public": {
+            "ssl":
+                "mysql://{user}:{password}@{public_ip}:{public_port}/{database}?"
+                "ssl={ssl_spec}",
+            "non-ssl":
+                "mysql://{user}:{password}@{public_ip}:{public_port}/{database}"
+        }
+    }
+}
+
+CLOUD_SQL_VALID_DATABASE_TYPES = ['postgres', 'mysql']
+
+
+# noinspection PyAbstractClass
+class CloudSqlDatabaseHook(BaseHook):
+    """
+    Serves DB connection configuration for Google Cloud SQL (Connections
+    of *gcpcloudsql://* type).
+
+    The hook is a "meta" one. It does not perform an actual connection.
+    It is there to retrieve all the parameters configured in gcpcloudsql:// connection,
+    start/stop Cloud SQL Proxy if needed, dynamically generate Postgres or MySQL
+    connection in the database and return an actual Postgres or MySQL hook.
+    The returned Postgres/MySQL hooks are using direct connection or Cloud SQL
+    Proxy socket/TCP as configured.
+
+    Main parameters of the hook are retrieved from the standard URI components:
+
+    * **user** - User name to authenticate to the database (from login of the URI).
+    * **password** - Password to authenticate to the database (from password of the URI).
+    * **public_ip** - IP to connect to for public connection (from host of the URI).
+    * **public_port** - Port to connect to for public connection (from port of the URI).
+    * **database** - Database to connect to (from schema of the URI).
+
+    Remaining parameters are retrieved from the extras (URI query parameters):
+
+    * **project_id** - Optional, Google Cloud Platform project where the Cloud SQL
+       instance exists. If missing, default project id passed is used.
+    * **instance** -  Name of the instance of the Cloud SQL database instance.
+    * **location** - The location of the Cloud SQL instance (for example europe-west1).
+    * **database_type** - The type of the database instance (MySQL or Postgres).
+    * **use_proxy** - (default False) Whether SQL proxy should be used to connect to Cloud
+      SQL DB.
+    * **use_ssl** - (default False) Whether SSL should be used to connect to Cloud SQL DB.
+      You cannot use proxy and SSL together.
+    * **sql_proxy_use_tcp** - (default False) If set to true, TCP is used to connect via
+      proxy, otherwise UNIX sockets are used.
+    * **sql_proxy_binary_path** - Optional path to Cloud SQL Proxy binary. If the binary
+      is not specified or the binary is not present, it is automatically downloaded.
+    * **sql_proxy_version** -  Specific version of the proxy to download (for example
+      v1.13). If not specified, the latest version is downloaded.
+    * **sslcert** - Path to client certificate to authenticate when SSL is used.
+    * **sslkey** - Path to client private key to authenticate when SSL is used.
+    * **sslrootcert** - Path to server's certificate to authenticate when SSL is used.
+
+    :param gcp_cloudsql_conn_id: URL of the connection
+    :type gcp_cloudsql_conn_id: str
+    :param default_gcp_project_id: Default project id used if project_id not specified
+           in the connection URL
+    :type default_gcp_project_id: str
+    """
+    _conn = None
+
+    def __init__(self, gcp_cloudsql_conn_id='google_cloud_sql_default',
+                 default_gcp_project_id=None):
+        super(CloudSqlDatabaseHook, self).__init__(source=None)
+        self.gcp_cloudsql_conn_id = gcp_cloudsql_conn_id
+        self.cloudsql_connection = self.get_connection(self.gcp_cloudsql_conn_id)
+        self.extras = self.cloudsql_connection.extra_dejson
+        self.project_id = self.extras.get('project_id', default_gcp_project_id)
+        self.instance = self.extras.get('instance')
+        self.database = self.cloudsql_connection.schema
+        self.location = self.extras.get('location')
+        self.database_type = self.extras.get('database_type')
+        self.use_proxy = self._get_bool(self.extras.get('use_proxy', 'False'))
+        self.use_ssl = self._get_bool(self.extras.get('use_ssl', 'False'))
+        self.sql_proxy_use_tcp = self._get_bool(
+            self.extras.get('sql_proxy_use_tcp', 'False'))
+        self.sql_proxy_version = self.extras.get('sql_proxy_version')
+        self.sql_proxy_binary_path = self.extras.get('sql_proxy_binary_path')
+        self.user = self.cloudsql_connection.login
+        self.password = self.cloudsql_connection.password
+        self.public_ip = self.cloudsql_connection.host
+        self.public_port = self.cloudsql_connection.port
+        self.sslcert = self.extras.get('sslcert')
+        self.sslkey = self.extras.get('sslkey')
+        self.sslrootcert = self.extras.get('sslrootcert')
+        # Port and socket path and db_hook are automatically generated
+        self.sql_proxy_tcp_port = None
+        self.sql_proxy_unique_path = None
+        self.db_hook = None
+        self.reserved_tcp_socket = None
+        # Generated based on clock + clock sequence. Unique per host (!).
+        # This is important as different hosts share the database
+        self.db_conn_id = str(uuid.uuid1())
+        self._validate_inputs()
+
+    @staticmethod
+    def _get_bool(val):
+        if val == 'False':
+            return False
+        return val
+
+    @staticmethod
+    def _check_ssl_file(file_to_check, name):
+        if not file_to_check:
+            raise AirflowException("SSL connections requires {name} to be set".
+                                   format(name=name))
+        if not os.path.isfile(file_to_check):
+            raise AirflowException("The {file_to_check} must be a readable file".
+                                   format(file_to_check=file_to_check))
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required extra 'project_id' is empty")
+        if not self.location:
+            raise AirflowException("The required extra 'location' is empty or None")
+        if not self.instance:
+            raise AirflowException("The required extra 'instance' is empty or None")
+        if self.database_type not in CLOUD_SQL_VALID_DATABASE_TYPES:
+            raise AirflowException("Invalid database type '{}'. Must be one of {}".format(
+                self.database_type, CLOUD_SQL_VALID_DATABASE_TYPES
+            ))
+        if self.use_proxy and self.use_ssl:
+            raise AirflowException("Cloud SQL Proxy does not support SSL connections."
+                                   " SSL is not needed as Cloud SQL Proxy "
+                                   "provides encryption on its own")
+
+    def validate_ssl_certs(self):
+        if self.use_ssl:
+            self._check_ssl_file(self.sslcert, "sslcert")
+            self._check_ssl_file(self.sslkey, "sslkey")
+            self._check_ssl_file(self.sslrootcert, "sslrootcert")
+
+    def validate_socket_path_length(self):
+        if self.use_proxy and not self.sql_proxy_use_tcp:
+            if self.database_type == 'postgres':
+                suffix = "/.s.PGSQL.5432"
+            else:
+                suffix = ""
+            expected_path = "{}/{}:{}:{}{}".format(
+                self._generate_unique_path(),
+                self.project_id, self.instance,
+                self.database, suffix)
+            if len(expected_path) > UNIX_PATH_MAX:
+                self.log.info("Too long (%s) path: %s", len(expected_path), expected_path)
+                raise AirflowException(
+                    "The UNIX socket path length cannot exceed {} characters "
+                    "on Linux system. Either use shorter instance/database "
+                    "name or switch to TCP connection. "
+                    "The socket path for Cloud SQL proxy is now:"
+                    "{}".format(
+                        UNIX_PATH_MAX, expected_path))
+
+    @staticmethod
+    def _generate_unique_path():
+        # We are not using mkdtemp here as the path generated with mkdtemp
+        # can be close to 60 characters and there is a limitation in
+        # length of socket path to around 100 characters in total.
+        # We append project/location/instance to it later and postgres
+        # appends its own prefix, so we chose a shorter "/tmp/[8 random characters]" -
+        random.seed()
+        while True:
+            candidate = "/tmp/" + ''.join(
+                random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
+            if not os.path.exists(candidate):
+                return candidate
+
+    @staticmethod
+    def _quote(value):
+        return quote_plus(value) if value else None
+
+    def _generate_connection_uri(self):
+        if self.use_proxy:
+            if self.sql_proxy_use_tcp:
+                if not self.sql_proxy_tcp_port:
+                    self.reserve_free_tcp_port()
+            if not self.sql_proxy_unique_path:
+                self.sql_proxy_unique_path = self._generate_unique_path()
+
+        database_uris = CONNECTION_URIS[self.database_type]
+        ssl_spec = None
+        socket_path = None
+        if self.use_proxy:
+            proxy_uris = database_uris['proxy']
+            if self.sql_proxy_use_tcp:
+                format_string = proxy_uris['tcp']
+            else:
+                format_string = proxy_uris['socket']
+                socket_path = \
+                    "{sql_proxy_socket_path}/{instance_socket_name}".format(
+                        sql_proxy_socket_path=self.sql_proxy_unique_path,
+                        instance_socket_name=self._get_instance_socket_name()
+                    )
+        else:
+            public_uris = database_uris['public']
+            if self.use_ssl:
+                format_string = public_uris['ssl']
+                ssl_spec = {
+                    'cert': self.sslcert,
+                    'key': self.sslkey,
+                    'ca': self.sslrootcert
+                }
+            else:
+                format_string = public_uris['non-ssl']
+        if not self.user:
+            raise AirflowException("The login parameter needs to be set in connection")
+        if not self.public_ip:
+            raise AirflowException("The location parameter needs to be set in connection")
+        if not self.password:
+            raise AirflowException("The password parameter needs to be set in connection")
+        if not self.database:
+            raise AirflowException("The database parameter needs to be set in connection")
+
+        connection_uri = format_string.format(
+            user=quote_plus(self.user) if self.user else '',
+            password=quote_plus(self.password) if self.password else '',
+            database=quote_plus(self.database) if self.database else '',
+            public_ip=self.public_ip,
+            public_port=self.public_port,
+            proxy_port=self.sql_proxy_tcp_port,
+            socket_path=self._quote(socket_path),
+            ssl_spec=self._quote(json.dumps(ssl_spec)) if ssl_spec else '',
+            client_cert_file=self._quote(self.sslcert) if self.sslcert else '',
+            client_key_file=self._quote(self.sslkey) if self.sslcert else '',
+            server_ca_file=self._quote(self.sslrootcert if self.sslcert else '')
+        )
+        self.log.info("DB connection URI %s", connection_uri.replace(
+            quote_plus(self.password) if self.password else 'PASSWORD', 'XXXXXXXXXXXX'))
+        return connection_uri
+
+    def _get_instance_socket_name(self):
+        return self.project_id + ":" + self.location + ":" + self.instance
+
+    def _get_sqlproxy_instance_specification(self):
+        instance_specification = self._get_instance_socket_name()
+        if self.sql_proxy_use_tcp:
+            instance_specification += "=tcp:" + str(self.sql_proxy_tcp_port)
+        return instance_specification
+
+    @provide_session
+    def create_connection(self, session=None):
+        """
+        Create connection in the Connection table, according to whether it uses
+        proxy, TCP, UNIX sockets, SSL. Connection ID will be randomly generated.
+
+        :param session: Session of the SQL Alchemy ORM (automatically generated with
+                        decorator).
+        """
+        connection = Connection(conn_id=self.db_conn_id)
+        uri = self._generate_connection_uri()
+        self.log.info("Creating connection %s", self.db_conn_id)
+        connection.parse_from_uri(uri)
+        session.add(connection)
+        session.commit()
+
+    @provide_session
+    def retrieve_connection(self, session=None):
+        """
+        Retrieves the dynamically created connection from the Connection table.
+
+        :param session: Session of the SQL Alchemy ORM (automatically generated with
+                        decorator).
+        """
+        self.log.info("Retrieving connection %s", self.db_conn_id)
+        connections = session.query(Connection).filter(
+            Connection.conn_id == self.db_conn_id)
+        if connections.count():
+            return connections[0]
+        return None
+
+    @provide_session
+    def delete_connection(self, session=None):
+        """
+        Delete the dynamically created connection from the Connection table.
+
+        :param session: Session of the SQL Alchemy ORM (automatically generated with
+                        decorator).
+        """
+        self.log.info("Deleting connection %s", self.db_conn_id)
+
+        connections = session.query(Connection).filter(
+            Connection.conn_id == self.db_conn_id)
+        if connections.count():
+            connection = connections[0]
+            session.delete(connection)
+            session.commit()
+        else:
+            self.log.info("Connection was already deleted!")
+
+    def get_sqlproxy_runner(self):
+        """
+        Retrieve Cloud SQL Proxy runner. It is used to manage the proxy
+        lifecycle per task.
+
+        :return: The Cloud SQL Proxy runner.
+        :rtype: CloudSqlProxyRunner
+        """
+        if not self.use_proxy:
+            raise AirflowException("Proxy runner can only be retrieved in case of use_proxy = True")
+        return CloudSqlProxyRunner(
+            path_prefix=self.sql_proxy_unique_path,
+            instance_specification=self._get_sqlproxy_instance_specification(),
+            project_id=self.project_id,
+            sql_proxy_version=self.sql_proxy_version,
+            sql_proxy_binary_path=self.sql_proxy_binary_path
+        )
+
+    def get_database_hook(self):
+        """
+        Retrieve database hook. This is the actual Postgres or MySQL database hook
+        that uses proxy or connects directly to the Google Cloud SQL database.
+        """
+        if self.database_type == 'postgres':
+            self.db_hook = PostgresHook(postgres_conn_id=self.db_conn_id,
+                                        schema=self.database)
+        else:
+            self.db_hook = MySqlHook(mysql_conn_id=self.db_conn_id,
+                                     schema=self.database)
+        return self.db_hook
+
+    def cleanup_database_hook(self):
+        """
+        Clean up database hook after it was used.
+        """
+        if self.database_type == 'postgres':
+            if hasattr(self.db_hook,
+                       'conn') and self.db_hook.conn and self.db_hook.conn.notices:
+                for output in self.db_hook.conn.notices:
+                    self.log.info(output)
+
+    def reserve_free_tcp_port(self):
+        """
+        Reserve free TCP port to be used by Cloud SQL Proxy
+        """
+        self.reserved_tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.reserved_tcp_socket.bind(('127.0.0.1', 0))
+        self.sql_proxy_tcp_port = self.reserved_tcp_socket.getsockname()[1]
+
+    def free_reserved_port(self):
+        """
+        Free TCP port. Makes it immediately ready to be used by Cloud SQL Proxy.
+        """
+        if self.reserved_tcp_socket:
+            self.reserved_tcp_socket.close()
+            self.reserved_tcp_socket = None
diff --git a/airflow/contrib/hooks/gcp_transfer_hook.py b/airflow/contrib/hooks/gcp_transfer_hook.py
new file mode 100644
index 0000000000000..d51d7747a9644
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_transfer_hook.py
@@ -0,0 +1,392 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import time
+from copy import deepcopy
+
+import six
+from googleapiclient.discovery import build
+
+from airflow.exceptions import AirflowException
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+# Time to sleep between active checks of the operation results
+TIME_TO_SLEEP_IN_SECONDS = 10
+
+
+class GcpTransferJobsStatus:
+    ENABLED = "ENABLED"
+    DISABLED = "DISABLED"
+    DELETED = "DELETED"
+
+
+class GcpTransferOperationStatus:
+    IN_PROGRESS = "IN_PROGRESS"
+    PAUSED = "PAUSED"
+    SUCCESS = "SUCCESS"
+    FAILED = "FAILED"
+    ABORTED = "ABORTED"
+
+
+# A list of keywords used to build a request or response
+ACCESS_KEY_ID = "accessKeyId"
+ALREADY_EXISTING_IN_SINK = "overwriteObjectsAlreadyExistingInSink"
+AWS_ACCESS_KEY = "awsAccessKey"
+AWS_S3_DATA_SOURCE = 'awsS3DataSource'
+BODY = 'body'
+BUCKET_NAME = 'bucketName'
+DAY = 'day'
+DESCRIPTION = "description"
+FILTER = 'filter'
+FILTER_JOB_NAMES = 'job_names'
+FILTER_PROJECT_ID = 'project_id'
+GCS_DATA_SINK = 'gcsDataSink'
+GCS_DATA_SOURCE = 'gcsDataSource'
+HOURS = "hours"
+HTTP_DATA_SOURCE = 'httpDataSource'
+LIST_URL = 'list_url'
+METADATA = 'metadata'
+MINUTES = "minutes"
+MONTH = 'month'
+NAME = 'name'
+OBJECT_CONDITIONS = 'object_conditions'
+OPERATIONS = 'operations'
+PROJECT_ID = 'projectId'
+SCHEDULE = 'schedule'
+SCHEDULE_END_DATE = 'scheduleEndDate'
+SCHEDULE_START_DATE = 'scheduleStartDate'
+SECONDS = "seconds"
+SECRET_ACCESS_KEY = "secretAccessKey"
+START_TIME_OF_DAY = 'startTimeOfDay'
+STATUS = "status"
+STATUS1 = 'status'
+TRANSFER_JOB = 'transfer_job'
+TRANSFER_JOB_FIELD_MASK = 'update_transfer_job_field_mask'
+TRANSFER_JOBS = 'transferJobs'
+TRANSFER_OPERATIONS = 'transferOperations'
+TRANSFER_OPTIONS = 'transfer_options'
+TRANSFER_SPEC = 'transferSpec'
+YEAR = 'year'
+
+NEGATIVE_STATUSES = {GcpTransferOperationStatus.FAILED, GcpTransferOperationStatus.ABORTED}
+
+# Number of retries - used by googleapiclient method calls to perform retries
+# For requests that are "retriable"
+NUM_RETRIES = 5
+
+
+# noinspection PyAbstractClass
+class GCPTransferServiceHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Storage Transfer Service.
+    """
+
+    _conn = None
+
+    def __init__(self, api_version='v1', gcp_conn_id='google_cloud_default', delegate_to=None):
+        super(GCPTransferServiceHook, self).__init__(gcp_conn_id, delegate_to)
+        self.api_version = api_version
+
+    def get_conn(self):
+        """
+        Retrieves connection to Google Storage Transfer service.
+
+        :return: Google Storage Transfer service object
+        :rtype: dict
+        """
+        if not self._conn:
+            http_authorized = self._authorize()
+            self._conn = build(
+                'storagetransfer', self.api_version, http=http_authorized, cache_discovery=False
+            )
+        return self._conn
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def create_transfer_job(self, body):
+        """
+        Creates a transfer job that runs periodically.
+
+        :param body: (Required) A request body, as described in
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/patch#request-body
+        :type body: dict
+        :return: transfer job.
+            See:
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#TransferJob
+        :rtype: dict
+        """
+        body = self._inject_project_id(body, BODY, PROJECT_ID)
+        return self.get_conn().transferJobs().create(body=body).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    @GoogleCloudBaseHook.catch_http_exception
+    def get_transfer_job(self, job_name, project_id=None):
+        """
+        Gets the latest state of a long-running operation in Google Storage
+        Transfer Service.
+
+        :param job_name: (Required) Name of the job to be fetched
+        :type job_name: str
+        :param project_id: (Optional) the ID of the project that owns the Transfer
+            Job. If set to None or missing, the default project_id from the GCP
+            connection is used.
+        :type project_id: str
+        :return: Transfer Job
+        :rtype: dict
+        """
+        return (
+            self.get_conn()
+            .transferJobs()
+            .get(jobName=job_name, projectId=project_id)
+            .execute(num_retries=NUM_RETRIES)
+        )
+
+    def list_transfer_job(self, filter):
+        """
+        Lists long-running operations in Google Storage Transfer
+        Service that match the specified filter.
+
+        :param filter: (Required) A request filter, as described in
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/list#body.QUERY_PARAMETERS.filter
+        :type filter: dict
+        :return: List of Transfer Jobs
+        :rtype: list[dict]
+        """
+        conn = self.get_conn()
+        filter = self._inject_project_id(filter, FILTER, FILTER_PROJECT_ID)
+        request = conn.transferJobs().list(filter=json.dumps(filter))
+        jobs = []
+
+        while request is not None:
+            response = request.execute(num_retries=NUM_RETRIES)
+            jobs.extend(response[TRANSFER_JOBS])
+
+            request = conn.transferJobs().list_next(previous_request=request, previous_response=response)
+
+        return jobs
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def update_transfer_job(self, job_name, body):
+        """
+        Updates a transfer job that runs periodically.
+
+        :param job_name: (Required) Name of the job to be updated
+        :type job_name: str
+        :param body: A request body, as described in
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/patch#request-body
+        :type body: dict
+        :return: If successful, TransferJob.
+        :rtype: dict
+        """
+        body = self._inject_project_id(body, BODY, PROJECT_ID)
+        return (
+            self.get_conn().transferJobs().patch(jobName=job_name, body=body).execute(num_retries=NUM_RETRIES)
+        )
+
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    @GoogleCloudBaseHook.catch_http_exception
+    def delete_transfer_job(self, job_name, project_id):
+        """
+        Deletes a transfer job. This is a soft delete. After a transfer job is
+        deleted, the job and all the transfer executions are subject to garbage
+        collection. Transfer jobs become eligible for garbage collection
+        30 days after soft delete.
+
+        :param job_name: (Required) Name of the job to be deleted
+        :type job_name: str
+        :param project_id: (Optional) the ID of the project that owns the Transfer
+            Job. If set to None or missing, the default project_id from the GCP
+            connection is used.
+        :type project_id: str
+        :rtype: None
+        """
+
+        return (
+            self.get_conn()
+            .transferJobs()
+            .patch(
+                jobName=job_name,
+                body={
+                    PROJECT_ID: project_id,
+                    TRANSFER_JOB: {STATUS1: GcpTransferJobsStatus.DELETED},
+                    TRANSFER_JOB_FIELD_MASK: STATUS1,
+                },
+            )
+            .execute(num_retries=NUM_RETRIES)
+        )
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def cancel_transfer_operation(self, operation_name):
+        """
+        Cancels an transfer operation in Google Storage Transfer Service.
+
+        :param operation_name: Name of the transfer operation.
+        :type operation_name: str
+        :rtype: None
+        """
+        self.get_conn().transferOperations().cancel(name=operation_name).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def get_transfer_operation(self, operation_name):
+        """
+        Gets an transfer operation in Google Storage Transfer Service.
+
+        :param operation_name: (Required) Name of the transfer operation.
+        :type operation_name: str
+        :return: transfer operation
+            See:
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/Operation
+        :rtype: dict
+        """
+        return self.get_conn().transferOperations().get(name=operation_name).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def list_transfer_operations(self, filter):
+        """
+        Gets an transfer operation in Google Storage Transfer Service.
+
+        :param filter: (Required) A request filter, as described in
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/list#body.QUERY_PARAMETERS.filter
+            With one additional improvement:
+
+            * project_id is optional if you have a project id defined
+              in the connection
+              See: :ref:`howto/connection:gcp`
+
+        :type filter: dict
+        :return: transfer operation
+        :rtype: list[dict]
+        """
+        conn = self.get_conn()
+
+        filter = self._inject_project_id(filter, FILTER, FILTER_PROJECT_ID)
+
+        operations = []
+
+        request = conn.transferOperations().list(name=TRANSFER_OPERATIONS, filter=json.dumps(filter))
+
+        while request is not None:
+            response = request.execute(num_retries=NUM_RETRIES)
+            if OPERATIONS in response:
+                operations.extend(response[OPERATIONS])
+
+            request = conn.transferOperations().list_next(
+                previous_request=request, previous_response=response
+            )
+
+        return operations
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def pause_transfer_operation(self, operation_name):
+        """
+        Pauses an transfer operation in Google Storage Transfer Service.
+
+        :param operation_name: (Required) Name of the transfer operation.
+        :type operation_name: str
+        :rtype: None
+        """
+        self.get_conn().transferOperations().pause(name=operation_name).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def resume_transfer_operation(self, operation_name):
+        """
+        Resumes an transfer operation in Google Storage Transfer Service.
+
+        :param operation_name: (Required) Name of the transfer operation.
+        :type operation_name: str
+        :rtype: None
+        """
+        self.get_conn().transferOperations().resume(name=operation_name).execute(num_retries=NUM_RETRIES)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def wait_for_transfer_job(self, job, expected_statuses=(GcpTransferOperationStatus.SUCCESS,), timeout=60):
+        """
+        Waits until the job reaches the expected state.
+
+        :param job: Transfer job
+            See:
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#TransferJob
+        :type job: dict
+        :param expected_statuses: State that is expected
+            See:
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations#Status
+        :type expected_statuses: set[str]
+        :param timeout:
+        :type timeout: time in which the operation must end in seconds
+        :rtype: None
+        """
+        while timeout > 0:
+            operations = self.list_transfer_operations(
+                filter={FILTER_PROJECT_ID: job[PROJECT_ID], FILTER_JOB_NAMES: [job[NAME]]}
+            )
+
+            if GCPTransferServiceHook.operations_contain_expected_statuses(operations, expected_statuses):
+                return
+            time.sleep(TIME_TO_SLEEP_IN_SECONDS)
+            timeout -= TIME_TO_SLEEP_IN_SECONDS
+        raise AirflowException("Timeout. The operation could not be completed within the allotted time.")
+
+    def _inject_project_id(self, body, param_name, target_key):
+        body = deepcopy(body)
+        body[target_key] = body.get(target_key, self.project_id)
+        if not body.get(target_key):
+            raise AirflowException(
+                "The project id must be passed either as `{}` key in `{}` parameter or as project_id "
+                "extra in GCP connection definition. Both are not set!".format(target_key, param_name)
+            )
+        return body
+
+    @staticmethod
+    def operations_contain_expected_statuses(operations, expected_statuses):
+        """
+        Checks whether the operation list has an operation with the
+        expected status, then returns true
+        If it encounters operations in FAILED or ABORTED state
+        throw :class:`airflow.exceptions.AirflowException`.
+
+        :param operations: (Required) List of transfer operations to check.
+        :type operations: list[dict]
+        :param expected_statuses: (Required) status that is expected
+            See:
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations#Status
+        :type expected_statuses: set[str]
+        :return: If there is an operation with the expected state
+            in the operation list, returns true,
+        :raises: airflow.exceptions.AirflowException If it encounters operations
+            with a state in the list,
+        :rtype: bool
+        """
+        expected_statuses = (
+            {expected_statuses} if isinstance(expected_statuses, six.string_types) else set(expected_statuses)
+        )
+        if len(operations) == 0:
+            return False
+
+        current_statuses = {operation[METADATA][STATUS] for operation in operations}
+
+        if len(current_statuses - set(expected_statuses)) != len(current_statuses):
+            return True
+
+        if len(NEGATIVE_STATUSES - current_statuses) != len(NEGATIVE_STATUSES):
+            raise AirflowException(
+                'An unexpected operation status was encountered. Expected: {}'.format(
+                    ", ".join(expected_statuses)
+                )
+            )
+        return False
diff --git a/airflow/contrib/hooks/gcp_translate_hook.py b/airflow/contrib/hooks/gcp_translate_hook.py
new file mode 100644
index 0000000000000..2d4adbb5b2b03
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_translate_hook.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from google.cloud.translate_v2 import Client
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+
+
+class CloudTranslateHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Cloud translate APIs.
+    """
+
+    _client = None
+
+    def __init__(self, gcp_conn_id='google_cloud_default'):
+        super(CloudTranslateHook, self).__init__(gcp_conn_id)
+
+    def get_conn(self):
+        """
+        Retrieves connection to Cloud Translate
+
+        :return: Google Cloud Translate client object.
+        :rtype: Client
+        """
+        if not self._client:
+            self._client = Client(credentials=self._get_credentials())
+        return self._client
+
+    def translate(
+        self, values, target_language, format_=None, source_language=None, model=None
+    ):
+        """Translate a string or list of strings.
+
+        See https://cloud.google.com/translate/docs/translating-text
+
+        :type values: str or list
+        :param values: String or list of strings to translate.
+
+        :type target_language: str
+        :param target_language: The language to translate results into. This
+                                is required by the API and defaults to
+                                the target language of the current instance.
+
+        :type format_: str
+        :param format_: (Optional) One of ``text`` or ``html``, to specify
+                        if the input text is plain text or HTML.
+
+        :type source_language: str or None
+        :param source_language: (Optional) The language of the text to
+                                be translated.
+
+        :type model: str or None
+        :param model: (Optional) The model used to translate the text, such
+                      as ``'base'`` or ``'nmt'``.
+
+        :rtype: str or list
+        :returns: A list of dictionaries for each queried value. Each
+                  dictionary typically contains three keys (though not
+                  all will be present in all cases)
+
+                  * ``detectedSourceLanguage``: The detected language (as an
+                    ISO 639-1 language code) of the text.
+                  * ``translatedText``: The translation of the text into the
+                    target language.
+                  * ``input``: The corresponding input value.
+                  * ``model``: The model used to translate the text.
+
+                  If only a single value is passed, then only a single
+                  dictionary will be returned.
+        :raises: :class:`~exceptions.ValueError` if the number of
+                 values and translations differ.
+        """
+        client = self.get_conn()
+
+        return client.translate(
+            values=values,
+            target_language=target_language,
+            format_=format_,
+            source_language=source_language,
+            model=model,
+        )
diff --git a/airflow/contrib/hooks/gcp_vision_hook.py b/airflow/contrib/hooks/gcp_vision_hook.py
new file mode 100644
index 0000000000000..ede44bf5e5e25
--- /dev/null
+++ b/airflow/contrib/hooks/gcp_vision_hook.py
@@ -0,0 +1,454 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from copy import deepcopy
+
+from google.cloud.vision_v1 import ProductSearchClient, ImageAnnotatorClient
+from google.protobuf.json_format import MessageToDict
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
+from airflow.utils.decorators import cached_property
+
+
+class NameDeterminer:
+    """
+    Class used for checking if the entity has the 'name' attribute set.
+
+    * If so, no action is taken.
+
+    * If not, and the name can be constructed from other parameters provided, it is created and filled in
+      the entity.
+
+    * If both the entity's 'name' attribute is set and the name can be constructed from other parameters
+      provided:
+
+        * If they are the same - no action is taken
+
+        * if they are different - an exception is thrown.
+
+    """
+
+    def __init__(self, label, id_label, get_path):
+        self.label = label
+        self.id_label = id_label
+        self.get_path = get_path
+
+    def get_entity_with_name(self, entity, entity_id, location, project_id):
+        entity = deepcopy(entity)
+        explicit_name = getattr(entity, 'name')
+        if location and entity_id:
+            # Necessary parameters to construct the name are present. Checking for conflict with explicit name
+            constructed_name = self.get_path(project_id, location, entity_id)
+            if not explicit_name:
+                entity.name = constructed_name
+                return entity
+            elif explicit_name != constructed_name:
+                self._raise_ex_different_names(constructed_name, explicit_name)
+        else:
+            # Not enough parameters to construct the name. Trying to use the name from Product / ProductSet.
+            if explicit_name:
+                return entity
+            else:
+                self._raise_ex_unable_to_determine_name()
+
+    def _raise_ex_unable_to_determine_name(self):
+        raise AirflowException(
+            "Unable to determine the {label} name. Please either set the name directly in the {label} "
+            "object or provide the `location` and `{id_label}` parameters.".format(
+                label=self.label, id_label=self.id_label
+            )
+        )
+
+    def _raise_ex_different_names(self, constructed_name, explicit_name):
+        raise AirflowException(
+            "The {label} name provided in the object ({explicit_name}) is different than the name created "
+            "from the input parameters ({constructed_name}). Please either: 1) Remove the {label} name, 2) "
+            "Remove the location and {id_label} parameters, 3) Unify the {label} name and input "
+            "parameters.".format(
+                label=self.label,
+                explicit_name=explicit_name,
+                constructed_name=constructed_name,
+                id_label=self.id_label,
+            )
+        )
+
+
+class CloudVisionHook(GoogleCloudBaseHook):
+    """
+    Hook for Google Cloud Vision APIs.
+    """
+
+    _client = None
+    product_name_determiner = NameDeterminer('Product', 'product_id', ProductSearchClient.product_path)
+    product_set_name_determiner = NameDeterminer(
+        'ProductSet', 'productset_id', ProductSearchClient.product_set_path
+    )
+
+    def __init__(self, gcp_conn_id='google_cloud_default', delegate_to=None):
+        super(CloudVisionHook, self).__init__(gcp_conn_id, delegate_to)
+
+    def get_conn(self):
+        """
+        Retrieves connection to Cloud Vision.
+
+        :return: Google Cloud Vision client object.
+        :rtype: google.cloud.vision_v1.ProductSearchClient
+        """
+        if not self._client:
+            self._client = ProductSearchClient(credentials=self._get_credentials())
+        return self._client
+
+    @cached_property
+    def annotator_client(self):
+        return ImageAnnotatorClient(credentials=self._get_credentials())
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_product_set(
+        self,
+        location,
+        product_set,
+        project_id=None,
+        product_set_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator`
+        """
+        client = self.get_conn()
+        parent = ProductSearchClient.location_path(project_id, location)
+        self.log.info('Creating a new ProductSet under the parent: %s', parent)
+        response = client.create_product_set(
+            parent=parent,
+            product_set=product_set,
+            product_set_id=product_set_id,
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info('ProductSet created: %s', response.name if response else '')
+        self.log.debug('ProductSet created:\n%s', response)
+
+        if not product_set_id:
+            # Product set id was generated by the API
+            product_set_id = self._get_autogenerated_id(response)
+            self.log.info('Extracted autogenerated ProductSet ID from the response: %s', product_set_id)
+
+        return product_set_id
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_product_set(
+        self, location, product_set_id, project_id=None, retry=None, timeout=None, metadata=None
+    ):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetGetOperator`
+        """
+        client = self.get_conn()
+        name = ProductSearchClient.product_set_path(project_id, location, product_set_id)
+        self.log.info('Retrieving ProductSet: %s', name)
+        response = client.get_product_set(name=name, retry=retry, timeout=timeout, metadata=metadata)
+        self.log.info('ProductSet retrieved.')
+        self.log.debug('ProductSet retrieved:\n%s', response)
+        return MessageToDict(response)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def update_product_set(
+        self,
+        product_set,
+        location=None,
+        product_set_id=None,
+        update_mask=None,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetUpdateOperator`
+        """
+        client = self.get_conn()
+        product_set = self.product_set_name_determiner.get_entity_with_name(
+            product_set, product_set_id, location, project_id
+        )
+        self.log.info('Updating ProductSet: %s', product_set.name)
+        response = client.update_product_set(
+            product_set=product_set, update_mask=update_mask, retry=retry, timeout=timeout, metadata=metadata
+        )
+        self.log.info('ProductSet updated: %s', response.name if response else '')
+        self.log.debug('ProductSet updated:\n%s', response)
+        return MessageToDict(response)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_product_set(
+        self, location, product_set_id, project_id=None, retry=None, timeout=None, metadata=None
+    ):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetDeleteOperator`
+        """
+        client = self.get_conn()
+        name = ProductSearchClient.product_set_path(project_id, location, product_set_id)
+        self.log.info('Deleting ProductSet: %s', name)
+        client.delete_product_set(name=name, retry=retry, timeout=timeout, metadata=metadata)
+        self.log.info('ProductSet with the name [%s] deleted.', name)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_product(
+        self, location, product, project_id=None, product_id=None, retry=None, timeout=None, metadata=None
+    ):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductCreateOperator`
+        """
+        client = self.get_conn()
+        parent = ProductSearchClient.location_path(project_id, location)
+        self.log.info('Creating a new Product under the parent: %s', parent)
+        response = client.create_product(
+            parent=parent,
+            product=product,
+            product_id=product_id,
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info('Product created: %s', response.name if response else '')
+        self.log.debug('Product created:\n%s', response)
+
+        if not product_id:
+            # Product id was generated by the API
+            product_id = self._get_autogenerated_id(response)
+            self.log.info('Extracted autogenerated Product ID from the response: %s', product_id)
+
+        return product_id
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def get_product(self, location, product_id, project_id=None, retry=None, timeout=None, metadata=None):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductGetOperator`
+        """
+        client = self.get_conn()
+        name = ProductSearchClient.product_path(project_id, location, product_id)
+        self.log.info('Retrieving Product: %s', name)
+        response = client.get_product(name=name, retry=retry, timeout=timeout, metadata=metadata)
+        self.log.info('Product retrieved.')
+        self.log.debug('Product retrieved:\n%s', response)
+        return MessageToDict(response)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def update_product(
+        self,
+        product,
+        location=None,
+        product_id=None,
+        update_mask=None,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductUpdateOperator`
+        """
+        client = self.get_conn()
+        product = self.product_name_determiner.get_entity_with_name(product, product_id, location, project_id)
+        self.log.info('Updating ProductSet: %s', product.name)
+        response = client.update_product(
+            product=product, update_mask=update_mask, retry=retry, timeout=timeout, metadata=metadata
+        )
+        self.log.info('Product updated: %s', response.name if response else '')
+        self.log.debug('Product updated:\n%s', response)
+        return MessageToDict(response)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_product(self, location, product_id, project_id=None, retry=None, timeout=None, metadata=None):
+        """
+        For the documentation see:
+        :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductDeleteOperator`
+        """
+        client = self.get_conn()
+        name = ProductSearchClient.product_path(project_id, location, product_id)
+        self.log.info('Deleting ProductSet: %s', name)
+        client.delete_product(name=name, retry=retry, timeout=timeout, metadata=metadata)
+        self.log.info('Product with the name [%s] deleted:', name)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def create_reference_image(
+        self,
+        location,
+        product_id,
+        reference_image,
+        reference_image_id=None,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionReferenceImageCreateOperator`
+        """
+        client = self.get_conn()
+        self.log.info('Creating ReferenceImage')
+        parent = ProductSearchClient.product_path(project=project_id, location=location, product=product_id)
+
+        response = client.create_reference_image(
+            parent=parent,
+            reference_image=reference_image,
+            reference_image_id=reference_image_id,
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+
+        self.log.info('ReferenceImage created: %s', response.name if response else '')
+        self.log.debug('ReferenceImage created:\n%s', response)
+
+        if not reference_image_id:
+            # Refernece image  id was generated by the API
+            reference_image_id = self._get_autogenerated_id(response)
+            self.log.info(
+                'Extracted autogenerated ReferenceImage ID from the response: %s', reference_image_id
+            )
+
+        return reference_image_id
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def delete_reference_image(
+        self,
+        location,
+        product_id,
+        reference_image_id,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionReferenceImageCreateOperator`
+        """
+        client = self.get_conn()
+        self.log.info('Deleting ReferenceImage')
+        name = ProductSearchClient.reference_image_path(
+            project=project_id, location=location, product=product_id, reference_image=reference_image_id
+        )
+        response = client.delete_reference_image(name=name, retry=retry, timeout=timeout, metadata=metadata)
+        self.log.info('ReferenceImage with the name [%s] deleted.', name)
+
+        return MessageToDict(response)
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def add_product_to_product_set(
+        self,
+        product_set_id,
+        product_id,
+        location=None,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionAddProductToProductSetOperator`
+        """
+        client = self.get_conn()
+
+        product_name = ProductSearchClient.product_path(project_id, location, product_id)
+        product_set_name = ProductSearchClient.product_set_path(project_id, location, product_set_id)
+
+        self.log.info('Add Product[name=%s] to Product Set[name=%s]', product_name, product_set_name)
+
+        client.add_product_to_product_set(
+            name=product_set_name, product=product_name, retry=retry, timeout=timeout, metadata=metadata
+        )
+
+        self.log.info('Product added to Product Set')
+
+    @GoogleCloudBaseHook.catch_http_exception
+    @GoogleCloudBaseHook.fallback_to_default_project_id
+    def remove_product_from_product_set(
+        self,
+        product_set_id,
+        product_id,
+        location=None,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+    ):
+        """
+        For the documentation see:
+        :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionRemoveProductFromProductSetOperator`
+        """
+        client = self.get_conn()
+
+        product_name = ProductSearchClient.product_path(project_id, location, product_id)
+        product_set_name = ProductSearchClient.product_set_path(project_id, location, product_set_id)
+
+        self.log.info('Remove Product[name=%s] from Product Set[name=%s]', product_name, product_set_name)
+
+        client.remove_product_from_product_set(
+            name=product_set_name, product=product_name, retry=retry, timeout=timeout, metadata=metadata
+        )
+
+        self.log.info('Product removed from Product Set')
+
+    @GoogleCloudBaseHook.catch_http_exception
+    def annotate_image(self, request, retry=None, timeout=None):
+        """
+        For the documentation see:
+        :py:class:`~airflow.contrib.operators.gcp_vision_image_annotator_operator.CloudVisionAnnotateImage`
+        """
+        client = self.annotator_client
+
+        self.log.info('Annotating image')
+
+        response = client.annotate_image(request=request, retry=retry, timeout=timeout)
+
+        self.log.info('Image annotated')
+
+        return MessageToDict(response)
+
+    @staticmethod
+    def _get_autogenerated_id(response):
+        try:
+            name = response.name
+        except AttributeError as e:
+            raise AirflowException('Unable to get name from response... [{}]\n{}'.format(response, e))
+        if '/' not in name:
+            raise AirflowException('Unable to get id from name... [{}]'.format(name))
+        return name.rsplit('/', 1)[1]
diff --git a/airflow/contrib/hooks/gcs_hook.py b/airflow/contrib/hooks/gcs_hook.py
index c5e356f41cfc0..cab06b92d5dec 100644
--- a/airflow/contrib/hooks/gcs_hook.py
+++ b/airflow/contrib/hooks/gcs_hook.py
@@ -17,14 +17,18 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-from apiclient.discovery import build
-from apiclient.http import MediaFileUpload
-from googleapiclient import errors
+
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from googleapiclient.errors import HttpError
 
 from airflow.contrib.hooks.gcp_api_base_hook import GoogleCloudBaseHook
 from airflow.exceptions import AirflowException
 
+import gzip as gz
+import shutil
 import re
+import os
 
 
 class GoogleCloudStorageHook(GoogleCloudBaseHook):
@@ -57,14 +61,15 @@ def copy(self, source_bucket, source_object, destination_bucket=None,
         source bucket/object is used, but not both.
 
         :param source_bucket: The bucket of the object to copy from.
-        :type source_bucket: string
+        :type source_bucket: str
         :param source_object: The object to copy.
-        :type source_object: string
+        :type source_object: str
         :param destination_bucket: The destination of the object to copied to.
             Can be omitted; then the same bucket is used.
-        :type destination_bucket: string
+        :type destination_bucket: str
         :param destination_object: The (renamed) path of the object if given.
             Can be omitted; then the same name is used.
+        :type destination_object: str
         """
         destination_bucket = destination_bucket or source_bucket
         destination_object = destination_object or source_object
@@ -87,7 +92,7 @@ def copy(self, source_bucket, source_object, destination_bucket=None,
                       destinationObject=destination_object, body='') \
                 .execute()
             return True
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 return False
             raise
@@ -102,11 +107,11 @@ def rewrite(self, source_bucket, source_object, destination_bucket,
         destination_object can be omitted, in which case source_object is used.
 
         :param source_bucket: The bucket of the object to copy from.
-        :type source_bucket: string
+        :type source_bucket: str
         :param source_object: The object to copy.
-        :type source_object: string
+        :type source_object: str
         :param destination_bucket: The destination of the object to copied to.
-        :type destination_bucket: string
+        :type destination_bucket: str
         :param destination_object: The (renamed) path of the object if given.
             Can be omitted; then the same name is used.
         """
@@ -139,7 +144,7 @@ def rewrite(self, source_bucket, source_object, destination_bucket,
                     .execute()
                 self.log.info('Rewrite request #%s: %s', request_count, result)
             return True
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 return False
             raise
@@ -150,11 +155,11 @@ def download(self, bucket, object, filename=None):
         Get a file from Google Cloud Storage.
 
         :param bucket: The bucket to fetch from.
-        :type bucket: string
+        :type bucket: str
         :param object: The object to fetch.
-        :type object: string
+        :type object: str
         :param filename: If set, a local file path where the file should be written to.
-        :type filename: string
+        :type filename: str
         """
         service = self.get_conn()
         downloaded_file_bytes = service \
@@ -171,42 +176,90 @@ def download(self, bucket, object, filename=None):
         return downloaded_file_bytes
 
     # pylint:disable=redefined-builtin
-    def upload(self, bucket, object, filename, mime_type='application/octet-stream'):
+    def upload(self, bucket, object, filename,
+               mime_type='application/octet-stream', gzip=False,
+               multipart=False, num_retries=0):
         """
         Uploads a local file to Google Cloud Storage.
 
         :param bucket: The bucket to upload to.
-        :type bucket: string
+        :type bucket: str
         :param object: The object name to set when uploading the local file.
-        :type object: string
+        :type object: str
         :param filename: The local file path to the file to be uploaded.
-        :type filename: string
+        :type filename: str
         :param mime_type: The MIME type to set when uploading the file.
-        :type mime_type: string
+        :type mime_type: str
+        :param gzip: Option to compress file for upload
+        :type gzip: bool
+        :param multipart: If True, the upload will be split into multiple HTTP requests. The
+                          default size is 256MiB per request. Pass a number instead of True to
+                          specify the request size, which must be a multiple of 262144 (256KiB).
+        :type multipart: bool or int
+        :param num_retries: The number of times to attempt to re-upload the file (or individual
+                            chunks, in the case of multipart uploads). Retries are attempted
+                            with exponential backoff.
+        :type num_retries: int
         """
         service = self.get_conn()
-        media = MediaFileUpload(filename, mime_type)
+
+        if gzip:
+            filename_gz = filename + '.gz'
+
+            with open(filename, 'rb') as f_in:
+                with gz.open(filename_gz, 'wb') as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+                    filename = filename_gz
+
         try:
-            service \
-                .objects() \
-                .insert(bucket=bucket, name=object, media_body=media) \
-                .execute()
-            return True
-        except errors.HttpError as ex:
+            if multipart:
+                if multipart is True:
+                    chunksize = 256 * 1024 * 1024
+                else:
+                    chunksize = multipart
+
+                if chunksize % (256 * 1024) > 0 or chunksize < 0:
+                    raise ValueError("Multipart size is not a multiple of 262144 (256KiB)")
+
+                media = MediaFileUpload(filename, mimetype=mime_type,
+                                        chunksize=chunksize, resumable=True)
+
+                request = service.objects().insert(bucket=bucket, name=object, media_body=media)
+                response = None
+                while response is None:
+                    status, response = request.next_chunk(num_retries=num_retries)
+                    if status:
+                        self.log.info("Upload progress %.1f%%", status.progress() * 100)
+
+            else:
+                media = MediaFileUpload(filename, mime_type)
+
+                service \
+                    .objects() \
+                    .insert(bucket=bucket, name=object, media_body=media) \
+                    .execute(num_retries=num_retries)
+
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 return False
             raise
 
+        finally:
+            if gzip:
+                os.remove(filename)
+
+        return True
+
     # pylint:disable=redefined-builtin
     def exists(self, bucket, object):
         """
         Checks for the existence of a file in Google Cloud Storage.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to check in the Google cloud
             storage bucket.
-        :type object: string
+        :type object: str
         """
         service = self.get_conn()
         try:
@@ -215,7 +268,7 @@ def exists(self, bucket, object):
                 .get(bucket=bucket, object=object) \
                 .execute()
             return True
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 return False
             raise
@@ -226,12 +279,12 @@ def is_updated_after(self, bucket, object, ts):
         Checks if an object is updated in Google Cloud Storage.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to check in the Google cloud
             storage bucket.
-        :type object: string
+        :type object: str
         :param ts: The timestamp to check against.
-        :type ts: datetime
+        :type ts: datetime.datetime
         """
         service = self.get_conn()
         try:
@@ -253,7 +306,7 @@ def is_updated_after(self, bucket, object, ts):
                 if updated > ts:
                     return True
 
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] != '404':
                 raise
 
@@ -265,11 +318,11 @@ def delete(self, bucket, object, generation=None):
         parameter is used.
 
         :param bucket: name of the bucket, where the object resides
-        :type bucket: string
+        :type bucket: str
         :param object: name of the object to delete
-        :type object: string
+        :type object: str
         :param generation: if present, permanently delete the object of this generation
-        :type generation: string
+        :type generation: str
         :return: True if succeeded
         """
         service = self.get_conn()
@@ -280,7 +333,7 @@ def delete(self, bucket, object, generation=None):
                 .delete(bucket=bucket, object=object, generation=generation) \
                 .execute()
             return True
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 return False
             raise
@@ -290,23 +343,23 @@ def list(self, bucket, versions=None, maxResults=None, prefix=None, delimiter=No
         List all objects from the bucket with the give string prefix in name
 
         :param bucket: bucket name
-        :type bucket: string
+        :type bucket: str
         :param versions: if true, list all versions of the objects
-        :type versions: boolean
+        :type versions: bool
         :param maxResults: max count of items to return in a single page of responses
-        :type maxResults: integer
+        :type maxResults: int
         :param prefix: prefix string which filters objects whose name begin with
             this prefix
-        :type prefix: string
+        :type prefix: str
         :param delimiter: filters objects based on the delimiter (for e.g '.csv')
-        :type delimiter: string
+        :type delimiter: str
         :return: a stream of object names matching the filtering criteria
         """
         service = self.get_conn()
 
         ids = list()
         pageToken = None
-        while(True):
+        while True:
             response = service.objects().list(
                 bucket=bucket,
                 versions=versions,
@@ -343,9 +396,9 @@ def get_size(self, bucket, object):
         Gets the size of a file in Google Cloud Storage.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to check in the Google cloud storage bucket.
-        :type object: string
+        :type object: str
 
         """
         self.log.info('Checking the file size of object: %s in bucket: %s',
@@ -365,7 +418,7 @@ def get_size(self, bucket, object):
                 return size
             else:
                 raise ValueError('Object is not a file')
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 raise ValueError('Object Not Found')
 
@@ -374,10 +427,10 @@ def get_crc32c(self, bucket, object):
         Gets the CRC32c checksum of an object in Google Cloud Storage.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to check in the Google cloud
             storage bucket.
-        :type object: string
+        :type object: str
         """
         self.log.info('Retrieving the crc32c checksum of '
                       'object: %s in bucket: %s', object, bucket)
@@ -392,7 +445,7 @@ def get_crc32c(self, bucket, object):
             self.log.info('The crc32c checksum of %s is %s', object, crc32c)
             return crc32c
 
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 raise ValueError('Object Not Found')
 
@@ -401,10 +454,10 @@ def get_md5hash(self, bucket, object):
         Gets the MD5 hash of an object in Google Cloud Storage.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to check in the Google cloud
             storage bucket.
-        :type object: string
+        :type object: str
         """
         self.log.info('Retrieving the MD5 hash of '
                       'object: %s in bucket: %s', object, bucket)
@@ -419,12 +472,13 @@ def get_md5hash(self, bucket, object):
             self.log.info('The md5Hash of %s is %s', object, md5hash)
             return md5hash
 
-        except errors.HttpError as ex:
+        except HttpError as ex:
             if ex.resp['status'] == '404':
                 raise ValueError('Object Not Found')
 
     def create_bucket(self,
                       bucket_name,
+                      resource=None,
                       storage_class='MULTI_REGIONAL',
                       location='US',
                       project_id=None,
@@ -439,7 +493,11 @@ def create_bucket(self,
             https://cloud.google.com/storage/docs/bucketnaming.html#requirements
 
         :param bucket_name: The name of the bucket.
-        :type bucket_name: string
+        :type bucket_name: str
+        :param resource: An optional dict with parameters for creating the bucket.
+            For information on available parameters, see Cloud Storage API doc:
+            https://cloud.google.com/storage/docs/json_api/v1/buckets/insert
+        :type resource: dict
         :param storage_class: This defines how objects in the bucket are stored
             and determines the SLA and the cost of storage. Values include
 
@@ -448,9 +506,10 @@ def create_bucket(self,
             - ``STANDARD``
             - ``NEARLINE``
             - ``COLDLINE``.
+
             If this value is not specified when the bucket is
             created, it will default to STANDARD.
-        :type storage_class: string
+        :type storage_class: str
         :param location: The location of the bucket.
             Object data for objects in the bucket resides in physical storage
             within this region. Defaults to US.
@@ -458,9 +517,9 @@ def create_bucket(self,
             .. seealso::
                 https://developers.google.com/storage/docs/bucket-locations
 
-        :type location: string
+        :type location: str
         :param project_id: The ID of the GCP Project.
-        :type project_id: string
+        :type project_id: str
         :param labels: User-provided labels, in key/value pairs.
         :type labels: dict
         :return: If successful, it returns the ``id`` of the bucket.
@@ -477,22 +536,24 @@ def create_bucket(self,
 
         self.log.info('Creating Bucket: %s; Location: %s; Storage Class: %s',
                       bucket_name, location, storage_class)
-        assert storage_class in storage_classes, \
-            'Invalid value ({}) passed to storage_class. Value should be ' \
-            'one of {}'.format(storage_class, storage_classes)
+        if storage_class not in storage_classes:
+            raise ValueError(
+                'Invalid value ({}) passed to storage_class. Value should be '
+                'one of {}'.format(storage_class, storage_classes))
 
-        assert re.match('[a-zA-Z0-9]+', bucket_name[0]), \
-            'Bucket names must start with a number or letter.'
+        if not re.match('[a-zA-Z0-9]+', bucket_name[0]):
+            raise ValueError('Bucket names must start with a number or letter.')
 
-        assert re.match('[a-zA-Z0-9]+', bucket_name[-1]), \
-            'Bucket names must end with a number or letter.'
+        if not re.match('[a-zA-Z0-9]+', bucket_name[-1]):
+            raise ValueError('Bucket names must end with a number or letter.')
 
         service = self.get_conn()
-        bucket_resource = {
+        bucket_resource = resource or {}
+        bucket_resource.update({
             'name': bucket_name,
             'location': location,
             'storageClass': storage_class
-        }
+        })
 
         self.log.info('The Default Project ID is %s', self.project_id)
 
@@ -509,11 +570,145 @@ def create_bucket(self,
 
             return response['id']
 
-        except errors.HttpError as ex:
+        except HttpError as ex:
             raise AirflowException(
                 'Bucket creation failed. Error was: {}'.format(ex.content)
             )
 
+    def insert_bucket_acl(self, bucket, entity, role, user_project):
+        """
+        Creates a new ACL entry on the specified bucket.
+        See: https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert
+
+        :param bucket: Name of a bucket.
+        :type bucket: str
+        :param entity: The entity holding the permission, in one of the following forms:
+            user-userId, user-email, group-groupId, group-email, domain-domain,
+            project-team-projectId, allUsers, allAuthenticatedUsers.
+            See: https://cloud.google.com/storage/docs/access-control/lists#scopes
+        :type entity: str
+        :param role: The access permission for the entity.
+            Acceptable values are: "OWNER", "READER", "WRITER".
+        :type role: str
+        :param user_project: (Optional) The project to be billed for this request.
+            Required for Requester Pays buckets.
+        :type user_project: str
+        """
+        self.log.info('Creating a new ACL entry in bucket: %s', bucket)
+        service = self.get_conn()
+        try:
+            response = service.bucketAccessControls().insert(
+                bucket=bucket,
+                body={
+                    "entity": entity,
+                    "role": role
+                },
+                userProject=user_project
+            ).execute()
+            if response:
+                self.log.info('A new ACL entry created in bucket: %s', bucket)
+        except HttpError as ex:
+            raise AirflowException(
+                'Bucket ACL entry creation failed. Error was: {}'.format(ex.content)
+            )
+
+    def insert_object_acl(self, bucket, object_name, entity, role, generation,
+                          user_project):
+        """
+        Creates a new ACL entry on the specified object.
+        See: https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert
+
+        :param bucket: Name of a bucket.
+        :type bucket: str
+        :param object_name: Name of the object. For information about how to URL encode
+            object names to be path safe, see:
+            https://cloud.google.com/storage/docs/json_api/#encoding
+        :type object_name: str
+        :param entity: The entity holding the permission, in one of the following forms:
+            user-userId, user-email, group-groupId, group-email, domain-domain,
+            project-team-projectId, allUsers, allAuthenticatedUsers
+            See: https://cloud.google.com/storage/docs/access-control/lists#scopes
+        :type entity: str
+        :param role: The access permission for the entity.
+            Acceptable values are: "OWNER", "READER".
+        :type role: str
+        :param generation: (Optional) If present, selects a specific revision of this
+            object (as opposed to the latest version, the default).
+        :type generation: str
+        :param user_project: (Optional) The project to be billed for this request.
+            Required for Requester Pays buckets.
+        :type user_project: str
+        """
+        self.log.info('Creating a new ACL entry for object: %s in bucket: %s',
+                      object_name, bucket)
+        service = self.get_conn()
+        try:
+            response = service.objectAccessControls().insert(
+                bucket=bucket,
+                object=object_name,
+                body={
+                    "entity": entity,
+                    "role": role
+                },
+                generation=generation,
+                userProject=user_project
+            ).execute()
+            if response:
+                self.log.info('A new ACL entry created for object: %s in bucket: %s',
+                              object_name, bucket)
+        except HttpError as ex:
+            raise AirflowException(
+                'Object ACL entry creation failed. Error was: {}'.format(ex.content)
+            )
+
+    def compose(self, bucket, source_objects, destination_object, num_retries=5):
+        """
+        Composes a list of existing object into a new object in the same storage bucket
+
+        Currently it only supports up to 32 objects that can be concatenated
+        in a single operation
+
+        https://cloud.google.com/storage/docs/json_api/v1/objects/compose
+
+        :param bucket: The name of the bucket containing the source objects.
+            This is also the same bucket to store the composed destination object.
+        :type bucket: str
+        :param source_objects: The list of source objects that will be composed
+            into a single object.
+        :type source_objects: list
+        :param destination_object: The path of the object if given.
+        :type destination_object: str
+        """
+
+        if not source_objects or not len(source_objects):
+            raise ValueError('source_objects cannot be empty.')
+
+        if not bucket or not destination_object:
+            raise ValueError('bucket and destination_object cannot be empty.')
+
+        service = self.get_conn()
+
+        dict_source_objects = [{'name': source_object}
+                               for source_object in source_objects]
+        body = {
+            'sourceObjects': dict_source_objects
+        }
+
+        try:
+            self.log.info("Composing %s to %s in the bucket %s",
+                          source_objects, destination_object, bucket)
+            service \
+                .objects() \
+                .compose(destinationBucket=bucket,
+                         destinationObject=destination_object,
+                         body=body) \
+                .execute(num_retries=num_retries)
+            return True
+        except HttpError as ex:
+            if ex.resp['status'] == '404':
+                return False
+            raise
+
 
 def _parse_gcs_url(gsurl):
     """
diff --git a/airflow/contrib/hooks/imap_hook.py b/airflow/contrib/hooks/imap_hook.py
new file mode 100644
index 0000000000000..79f08cd5649cb
--- /dev/null
+++ b/airflow/contrib/hooks/imap_hook.py
@@ -0,0 +1,316 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import email
+import imaplib
+import os.path
+import re
+
+from airflow import LoggingMixin, AirflowException
+from airflow.hooks.base_hook import BaseHook
+
+
+class ImapHook(BaseHook):
+    """
+    This hook connects to a mail server by using the imap protocol.
+
+    :param imap_conn_id: The connection id that contains the information used to authenticate the client.
+    :type imap_conn_id: str
+    """
+
+    def __init__(self, imap_conn_id='imap_default'):
+        super(ImapHook, self).__init__(imap_conn_id)
+        self.conn = self.get_connection(imap_conn_id)
+        self.mail_client = imaplib.IMAP4_SSL(self.conn.host)
+
+    def __enter__(self):
+        self.mail_client.login(self.conn.login, self.conn.password)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.mail_client.logout()
+
+    def has_mail_attachment(self, name, mail_folder='INBOX', check_regex=False):
+        """
+        Checks the mail folder for mails containing attachments with the given name.
+
+        :param name: The name of the attachment that will be searched for.
+        :type name: str
+        :param mail_folder: The mail folder where to look at.
+        :type mail_folder: str
+        :param check_regex: Checks the name for a regular expression.
+        :type check_regex: bool
+        :returns: True if there is an attachment with the given name and False if not.
+        :rtype: bool
+        """
+        mail_attachments = self._retrieve_mails_attachments_by_name(name,
+                                                                    mail_folder,
+                                                                    check_regex,
+                                                                    latest_only=True)
+        return len(mail_attachments) > 0
+
+    def retrieve_mail_attachments(self,
+                                  name,
+                                  mail_folder='INBOX',
+                                  check_regex=False,
+                                  latest_only=False,
+                                  not_found_mode='raise'):
+        """
+        Retrieves mail's attachments in the mail folder by its name.
+
+        :param name: The name of the attachment that will be downloaded.
+        :type name: str
+        :param mail_folder: The mail folder where to look at.
+        :type mail_folder: str
+        :param check_regex: Checks the name for a regular expression.
+        :type check_regex: bool
+        :param latest_only: If set to True it will only retrieve
+                            the first matched attachment.
+        :type latest_only: bool
+        :param not_found_mode: Specify what should happen if no attachment has been found.
+                               Supported values are 'raise', 'warn' and 'ignore'.
+                               If it is set to 'raise' it will raise an exception,
+                               if set to 'warn' it will only print a warning and
+                               if set to 'ignore' it won't notify you at all.
+        :type not_found_mode: str
+        :returns: a list of tuple each containing the attachment filename and its payload.
+        :rtype: a list of tuple
+        """
+        mail_attachments = self._retrieve_mails_attachments_by_name(name,
+                                                                    mail_folder,
+                                                                    check_regex,
+                                                                    latest_only)
+        if not mail_attachments:
+            self._handle_not_found_mode(not_found_mode)
+
+        return mail_attachments
+
+    def download_mail_attachments(self,
+                                  name,
+                                  local_output_directory,
+                                  mail_folder='INBOX',
+                                  check_regex=False,
+                                  latest_only=False,
+                                  not_found_mode='raise'):
+        """
+        Downloads mail's attachments in the mail folder by its name to the local directory.
+
+        :param name: The name of the attachment that will be downloaded.
+        :type name: str
+        :param local_output_directory: The output directory on the local machine
+                                       where the files will be downloaded to.
+        :type local_output_directory: str
+        :param mail_folder: The mail folder where to look at.
+        :type mail_folder: str
+        :param check_regex: Checks the name for a regular expression.
+        :type check_regex: bool
+        :param latest_only: If set to True it will only download
+                            the first matched attachment.
+        :type latest_only: bool
+        :param not_found_mode: Specify what should happen if no attachment has been found.
+                               Supported values are 'raise', 'warn' and 'ignore'.
+                               If it is set to 'raise' it will raise an exception,
+                               if set to 'warn' it will only print a warning and
+                               if set to 'ignore' it won't notify you at all.
+        :type not_found_mode: str
+        """
+        mail_attachments = self._retrieve_mails_attachments_by_name(name,
+                                                                    mail_folder,
+                                                                    check_regex,
+                                                                    latest_only)
+
+        if not mail_attachments:
+            self._handle_not_found_mode(not_found_mode)
+
+        self._create_files(mail_attachments, local_output_directory)
+
+    def _handle_not_found_mode(self, not_found_mode):
+        if not_found_mode == 'raise':
+            raise AirflowException('No mail attachments found!')
+        elif not_found_mode == 'warn':
+            self.log.warning('No mail attachments found!')
+        elif not_found_mode == 'ignore':
+            pass  # Do not notify if the attachment has not been found.
+        else:
+            self.log.error('Invalid "not_found_mode" %s', not_found_mode)
+
+    def _retrieve_mails_attachments_by_name(self, name, mail_folder, check_regex, latest_only):
+        all_matching_attachments = []
+
+        self.mail_client.select(mail_folder)
+
+        for mail_id in self._list_mail_ids_desc():
+            response_mail_body = self._fetch_mail_body(mail_id)
+            matching_attachments = self._check_mail_body(response_mail_body, name, check_regex, latest_only)
+
+            if matching_attachments:
+                all_matching_attachments.extend(matching_attachments)
+                if latest_only:
+                    break
+
+        self.mail_client.close()
+
+        return all_matching_attachments
+
+    def _list_mail_ids_desc(self):
+        result, data = self.mail_client.search(None, 'All')
+        mail_ids = data[0].split()
+        return reversed(mail_ids)
+
+    def _fetch_mail_body(self, mail_id):
+        result, data = self.mail_client.fetch(mail_id, '(RFC822)')
+        mail_body = data[0][1]  # The mail body is always in this specific location
+        mail_body_str = mail_body.decode('utf-8')
+        return mail_body_str
+
+    def _check_mail_body(self, response_mail_body, name, check_regex, latest_only):
+        mail = Mail(response_mail_body)
+        if mail.has_attachments():
+            return mail.get_attachments_by_name(name, check_regex, find_first=latest_only)
+
+    def _create_files(self, mail_attachments, local_output_directory):
+        for name, payload in mail_attachments:
+            if self._is_symlink(name):
+                self.log.error('Can not create file because it is a symlink!')
+            elif self._is_escaping_current_directory(name):
+                self.log.error('Can not create file because it is escaping the current directory!')
+            else:
+                self._create_file(name, payload, local_output_directory)
+
+    def _is_symlink(self, name):
+        # IMPORTANT NOTE: os.path.islink is not working for windows symlinks
+        # See: https://stackoverflow.com/a/11068434
+        return os.path.islink(name)
+
+    def _is_escaping_current_directory(self, name):
+        return '../' in name
+
+    def _correct_path(self, name, local_output_directory):
+        return local_output_directory + name if local_output_directory.endswith('/') \
+            else local_output_directory + '/' + name
+
+    def _create_file(self, name, payload, local_output_directory):
+        file_path = self._correct_path(name, local_output_directory)
+
+        with open(file_path, 'wb') as file:
+            file.write(payload)
+
+
+class Mail(LoggingMixin):
+    """
+    This class simplifies working with mails returned by the imaplib client.
+
+    :param mail_body: The mail body of a mail received from imaplib client.
+    :type mail_body: str
+    """
+
+    def __init__(self, mail_body):
+        super(Mail, self).__init__()
+        self.mail = email.message_from_string(mail_body)
+
+    def has_attachments(self):
+        """
+        Checks the mail for a attachments.
+
+        :returns: True if it has attachments and False if not.
+        :rtype: bool
+        """
+        return self.mail.get_content_maintype() == 'multipart'
+
+    def get_attachments_by_name(self, name, check_regex, find_first=False):
+        """
+        Gets all attachments by name for the mail.
+
+        :param name: The name of the attachment to look for.
+        :type name: str
+        :param check_regex: Checks the name for a regular expression.
+        :type check_regex: bool
+        :param find_first: If set to True it will only find the first match and then quit.
+        :type find_first: bool
+        :returns: a list of tuples each containing name and payload
+                  where the attachments name matches the given name.
+        :rtype: list of tuple
+        """
+        attachments = []
+
+        for part in self.mail.walk():
+            mail_part = MailPart(part)
+            if mail_part.is_attachment():
+                found_attachment = mail_part.has_matching_name(name) if check_regex \
+                    else mail_part.has_equal_name(name)
+                if found_attachment:
+                    file_name, file_payload = mail_part.get_file()
+                    self.log.info('Found attachment: {}'.format(file_name))
+                    attachments.append((file_name, file_payload))
+                    if find_first:
+                        break
+
+        return attachments
+
+
+class MailPart:
+    """
+    This class is a wrapper for a Mail object's part and gives it more features.
+
+    :param part: The mail part in a Mail object.
+    :type part: any
+    """
+
+    def __init__(self, part):
+        self.part = part
+
+    def is_attachment(self):
+        """
+        Checks if the part is a valid mail attachment.
+
+        :returns: True if it is an attachment and False if not.
+        :rtype: bool
+        """
+        return self.part.get_content_maintype() != 'multipart' and self.part.get('Content-Disposition')
+
+    def has_matching_name(self, name):
+        """
+        Checks if the given name matches the part's name.
+
+        :param name: The name to look for.
+        :type name: str
+        :returns: True if it matches the name (including regular expression).
+        :rtype: tuple
+        """
+        return re.match(name, self.part.get_filename())
+
+    def has_equal_name(self, name):
+        """
+        Checks if the given name is equal to the part's name.
+
+        :param name: The name to look for.
+        :type name: str
+        :returns: True if it is equal to the given name.
+        :rtype: bool
+        """
+        return self.part.get_filename() == name
+
+    def get_file(self):
+        """
+        Gets the file including name and payload.
+
+        :returns: the part's name and payload.
+        :rtype: tuple
+        """
+        return self.part.get_filename(), self.part.get_payload(decode=True)
diff --git a/airflow/contrib/hooks/jira_hook.py b/airflow/contrib/hooks/jira_hook.py
index c6806d935acf2..e3c4a12ffe69b 100644
--- a/airflow/contrib/hooks/jira_hook.py
+++ b/airflow/contrib/hooks/jira_hook.py
@@ -21,15 +21,14 @@
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.log.logging_mixin import LoggingMixin
 
 
-class JiraHook(BaseHook, LoggingMixin):
+class JiraHook(BaseHook):
     """
     Jira interaction hook, a Wrapper around JIRA Python SDK.
 
     :param jira_conn_id: reference to a pre-defined Jira Connection
-    :type jira_conn_id: string
+    :type jira_conn_id: str
     """
     def __init__(self,
                  jira_conn_id='jira_default',
diff --git a/airflow/contrib/hooks/mongo_hook.py b/airflow/contrib/hooks/mongo_hook.py
index 6ae71a8c8d461..959d4cc86274c 100644
--- a/airflow/contrib/hooks/mongo_hook.py
+++ b/airflow/contrib/hooks/mongo_hook.py
@@ -1,20 +1,25 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 from ssl import CERT_NONE
 
 from airflow.hooks.base_hook import BaseHook
-from pymongo import MongoClient
+from pymongo import MongoClient, ReplaceOne
 
 
 class MongoHook(BaseHook):
@@ -24,10 +29,9 @@ class MongoHook(BaseHook):
     https://docs.mongodb.com/manual/reference/connection-string/index.html
     You can specify connection string options in extra field of your connection
     https://docs.mongodb.com/manual/reference/connection-string/index.html#connection-string-options
-    ex.
-        {replicaSet: test, ssl: True, connectTimeoutMS: 30000}
+    ex. ``{replicaSet: test, ssl: True, connectTimeoutMS: 30000}``
     """
-    conn_type = 'MongoDb'
+    conn_type = 'mongo'
 
     def __init__(self, conn_id='mongo_default', *args, **kwargs):
         super(MongoHook, self).__init__(source='mongo')
@@ -37,6 +41,13 @@ def __init__(self, conn_id='mongo_default', *args, **kwargs):
         self.extras = self.connection.extra_dejson
         self.client = None
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.client is not None:
+            self.close_conn()
+
     def get_conn(self):
         """
         Fetches PyMongo Client
@@ -49,11 +60,11 @@ def get_conn(self):
         uri = 'mongodb://{creds}{host}{port}/{database}'.format(
             creds='{}:{}@'.format(
                 conn.login, conn.password
-            ) if conn.login is not None else '',
+            ) if conn.login else '',
 
             host=conn.host,
             port='' if conn.port is None else ':{}'.format(conn.port),
-            database='' if conn.schema is None else conn.schema
+            database=conn.schema
         )
 
         # Mongo Connection Options dict that is unpacked when passed to MongoClient
@@ -67,6 +78,12 @@ def get_conn(self):
 
         return self.client
 
+    def close_conn(self):
+        client = self.client
+        if client is not None:
+            client.close()
+            self.client = None
+
     def get_collection(self, mongo_collection, mongo_db=None):
         """
         Fetches a mongo collection object for querying.
@@ -117,3 +134,158 @@ def insert_many(self, mongo_collection, docs, mongo_db=None, **kwargs):
         collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
 
         return collection.insert_many(docs, **kwargs)
+
+    def update_one(self, mongo_collection, filter_doc, update_doc,
+                   mongo_db=None, **kwargs):
+        """
+        Updates a single document in a mongo collection.
+        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.update_one
+
+        :param mongo_collection: The name of the collection to update.
+        :type mongo_collection: str
+        :param filter_doc: A query that matches the documents to update.
+        :type filter_doc: dict
+        :param update_doc: The modifications to apply.
+        :type update_doc: dict
+        :param mongo_db: The name of the database to use.
+            Can be omitted; then the database from the connection string is used.
+        :type mongo_db: str
+
+        """
+        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
+
+        return collection.update_one(filter_doc, update_doc, **kwargs)
+
+    def update_many(self, mongo_collection, filter_doc, update_doc,
+                    mongo_db=None, **kwargs):
+        """
+        Updates one or more documents in a mongo collection.
+        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.update_many
+
+        :param mongo_collection: The name of the collection to update.
+        :type mongo_collection: str
+        :param filter_doc: A query that matches the documents to update.
+        :type filter_doc: dict
+        :param update_doc: The modifications to apply.
+        :type update_doc: dict
+        :param mongo_db: The name of the database to use.
+            Can be omitted; then the database from the connection string is used.
+        :type mongo_db: str
+
+        """
+        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
+
+        return collection.update_many(filter_doc, update_doc, **kwargs)
+
+    def replace_one(self, mongo_collection, doc, filter_doc=None,
+                    mongo_db=None, **kwargs):
+        """
+        Replaces a single document in a mongo collection.
+        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.replace_one
+
+        .. note::
+            If no ``filter_doc`` is given, it is assumed that the replacement
+            document contain the ``_id`` field which is then used as filters.
+
+        :param mongo_collection: The name of the collection to update.
+        :type mongo_collection: str
+        :param doc: The new document.
+        :type doc: dict
+        :param filter_doc: A query that matches the documents to replace.
+            Can be omitted; then the _id field from doc will be used.
+        :type filter_doc: dict
+        :param mongo_db: The name of the database to use.
+            Can be omitted; then the database from the connection string is used.
+        :type mongo_db: str
+        """
+        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
+
+        if not filter_doc:
+            filter_doc = {'_id': doc['_id']}
+
+        return collection.replace_one(filter_doc, doc, **kwargs)
+
+    def replace_many(self, mongo_collection, docs,
+                     filter_docs=None, mongo_db=None, upsert=False, collation=None,
+                     **kwargs):
+        """
+        Replaces many documents in a mongo collection.
+
+        Uses bulk_write with multiple ReplaceOne operations
+        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.bulk_write
+
+        .. note::
+            If no ``filter_docs``are given, it is assumed that all
+            replacement documents contain the ``_id`` field which are then
+            used as filters.
+
+        :param mongo_collection: The name of the collection to update.
+        :type mongo_collection: str
+        :param docs: The new documents.
+        :type docs: list[dict]
+        :param filter_docs: A list of queries that match the documents to replace.
+            Can be omitted; then the _id fields from docs will be used.
+        :type filter_docs: list[dict]
+        :param mongo_db: The name of the database to use.
+            Can be omitted; then the database from the connection string is used.
+        :type mongo_db: str
+        :param upsert: If ``True``, perform an insert if no documents
+            match the filters for the replace operation.
+        :type upsert: bool
+        :param collation: An instance of
+            :class:`~pymongo.collation.Collation`. This option is only
+            supported on MongoDB 3.4 and above.
+        :type collation: pymongo.collation.Collation
+
+        """
+        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
+
+        if not filter_docs:
+            filter_docs = [{'_id': doc['_id']} for doc in docs]
+
+        requests = [
+            ReplaceOne(
+                filter_docs[i],
+                docs[i],
+                upsert=upsert,
+                collation=collation)
+            for i in range(len(docs))
+        ]
+
+        return collection.bulk_write(requests, **kwargs)
+
+    def delete_one(self, mongo_collection, filter_doc, mongo_db=None, **kwargs):
+        """
+        Deletes a single document in a mongo collection.
+        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.delete_one
+
+        :param mongo_collection: The name of the collection to delete from.
+        :type mongo_collection: str
+        :param filter_doc: A query that matches the document to delete.
+        :type filter_doc: dict
+        :param mongo_db: The name of the database to use.
+            Can be omitted; then the database from the connection string is used.
+        :type mongo_db: str
+
+        """
+        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
+
+        return collection.delete_one(filter_doc, **kwargs)
+
+    def delete_many(self, mongo_collection, filter_doc, mongo_db=None, **kwargs):
+        """
+        Deletes one or more documents in a mongo collection.
+        https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.delete_many
+
+        :param mongo_collection: The name of the collection to delete from.
+        :type mongo_collection: str
+        :param filter_doc: A query that matches the documents to delete.
+        :type filter_doc: dict
+        :param mongo_db: The name of the database to use.
+            Can be omitted; then the database from the connection string is used.
+        :type mongo_db: str
+
+        """
+        collection = self.get_collection(mongo_collection, mongo_db=mongo_db)
+
+        return collection.delete_many(filter_doc, **kwargs)
diff --git a/airflow/contrib/hooks/openfaas_hook.py b/airflow/contrib/hooks/openfaas_hook.py
new file mode 100644
index 0000000000000..8f5062cbc708f
--- /dev/null
+++ b/airflow/contrib/hooks/openfaas_hook.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.hooks.base_hook import BaseHook
+import requests
+from airflow import AirflowException
+
+OK_STATUS_CODE = 202
+
+
+class OpenFaasHook(BaseHook):
+    """
+    Interact with Openfaas to query, deploy, invoke and update function
+
+    :param function_name: Name of the function, Defaults to None
+    :type query: str
+    :param conn_id: openfass connection to use, Defaults to open_faas_default
+        for example host : http://openfaas.faas.com, Conn Type : Http
+    :type conn_id: str
+    """
+
+    GET_FUNCTION = "/system/function/"
+    INVOKE_ASYNC_FUNCTION = "/async-function/"
+    DEPLOY_FUNCTION = "/system/functions"
+    UPDATE_FUNCTION = "/system/functions"
+
+    def __init__(self,
+                 function_name=None,
+                 conn_id='open_faas_default',
+                 *args, **kwargs):
+        self.function_name = function_name
+        self.conn_id = conn_id
+        super(BaseHook, self).__init__(*args, **kwargs)
+
+    def get_conn(self):
+        conn = self.get_connection(self.conn_id)
+        return conn
+
+    def deploy_function(self, overwrite_function_if_exist, body):
+        if overwrite_function_if_exist:
+            self.log.info("Function already exist " + self.function_name + " going to update")
+            self.update_function(body)
+        else:
+            url = self.get_conn().host + self.DEPLOY_FUNCTION
+            self.log.info("Deploying function " + url)
+            response = requests.post(url, body)
+            if (response.status_code != OK_STATUS_CODE):
+                self.log.error("Response status " + str(response.status_code))
+                self.log.error("Failed to deploy")
+                raise AirflowException('failed to deploy')
+            else:
+                self.log.info("Function deployed " + self.function_name)
+
+    def invoke_async_function(self, body):
+        url = self.get_conn().host + self.INVOKE_ASYNC_FUNCTION + self.function_name
+        self.log.info("Invoking  function " + url)
+        response = requests.post(url, body)
+        if (response.ok):
+            self.log.info("Invoked " + self.function_name)
+        else:
+            self.log.error("Response status " + str(response.status_code))
+            raise AirflowException('failed to invoke function')
+
+    def update_function(self, body):
+        url = self.get_conn().host + self.UPDATE_FUNCTION
+        self.log.info("Updating function " + url)
+        response = requests.put(url, body)
+        if (response.status_code != OK_STATUS_CODE):
+            self.log.error("Response status " + str(response.status_code))
+            self.log.error("Failed to update response " + response.content.decode("utf-8"))
+            raise AirflowException('failed to update ' + self.function_name)
+        else:
+            self.log.info("Function was updated")
+
+    def does_function_exist(self):
+        url = self.get_conn().host + self.GET_FUNCTION + self.function_name
+
+        response = requests.get(url)
+        if (response.ok):
+            return True
+        else:
+            self.log.error("Failed to find function " + self.function_name)
+            return False
diff --git a/airflow/contrib/hooks/opsgenie_alert_hook.py b/airflow/contrib/hooks/opsgenie_alert_hook.py
new file mode 100644
index 0000000000000..d576c08c23982
--- /dev/null
+++ b/airflow/contrib/hooks/opsgenie_alert_hook.py
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import json
+
+import requests
+
+from airflow.hooks.http_hook import HttpHook
+from airflow import AirflowException
+
+
+class OpsgenieAlertHook(HttpHook):
+    """
+    This hook allows you to post alerts to Opsgenie.
+    Accepts a connection that has an Opsgenie API key as the connection's password.
+    This hook sets the domain to conn_id.host, and if not set will default
+    to ``https://api.opsgenie.com``.
+
+    Each Opsgenie API key can be pre-configured to a team integration.
+    You can override these defaults in this hook.
+
+    :param opsgenie_conn_id: The name of the Opsgenie connection to use
+    :type opsgenie_conn_id: str
+
+    """
+    def __init__(self,
+                 opsgenie_conn_id='opsgenie_default',
+                 *args,
+                 **kwargs
+                 ):
+        super(OpsgenieAlertHook, self).__init__(http_conn_id=opsgenie_conn_id, *args, **kwargs)
+
+    def _get_api_key(self):
+        """
+        Get Opsgenie api_key for creating alert
+        """
+        conn = self.get_connection(self.http_conn_id)
+        api_key = conn.password
+        if not api_key:
+            raise AirflowException('Opsgenie API Key is required for this hook, '
+                                   'please check your conn_id configuration.')
+        return api_key
+
+    def get_conn(self, headers=None):
+        """
+        Overwrite HttpHook get_conn because this hook just needs base_url
+        and headers, and does not need generic params
+
+        :param headers: additional headers to be passed through as a dictionary
+        :type headers: dict
+        """
+        conn = self.get_connection(self.http_conn_id)
+        self.base_url = conn.host if conn.host else 'https://api.opsgenie.com'
+        session = requests.Session()
+        if headers:
+            session.headers.update(headers)
+        return session
+
+    def execute(self, payload={}):
+        """
+        Execute the Opsgenie Alert call
+
+        :param payload: Opsgenie API Create Alert payload values
+            See https://docs.opsgenie.com/docs/alert-api#section-create-alert
+        :type payload: dict
+        """
+        api_key = self._get_api_key()
+        return self.run(endpoint='v2/alerts',
+                        data=json.dumps(payload),
+                        headers={'Content-Type': 'application/json',
+                                 'Authorization': 'GenieKey %s' % api_key})
diff --git a/airflow/contrib/hooks/qubole_hook.py b/airflow/contrib/hooks/qubole_hook.py
index fb45862e88e02..43eb5e6f82e15 100755
--- a/airflow/contrib/hooks/qubole_hook.py
+++ b/airflow/contrib/hooks/qubole_hook.py
@@ -123,7 +123,7 @@ def execute(self, context):
 
     def kill(self, ti):
         """
-        Kill (cancel) a Qubole commmand
+        Kill (cancel) a Qubole command
         :param ti: Task Instance of the dag, used to determine the Quboles command id
         :return: response from Qubole
         """
@@ -176,7 +176,7 @@ def get_jobs_id(self, ti):
         """
         Get jobs associated with a Qubole commands
         :param ti: Task Instance of the dag, used to determine the Quboles command id
-        :return: Job informations assoiciated with command
+        :return: Job information associated with command
         """
         if self.cmd is None:
             cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=self.task_id)
diff --git a/airflow/contrib/hooks/redis_hook.py b/airflow/contrib/hooks/redis_hook.py
index 1de75dbca905f..a63c839e34c85 100644
--- a/airflow/contrib/hooks/redis_hook.py
+++ b/airflow/contrib/hooks/redis_hook.py
@@ -20,16 +20,13 @@
 """
 RedisHook module
 """
-from redis import StrictRedis
-
-from airflow.exceptions import AirflowException
+from redis import Redis
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.log.logging_mixin import LoggingMixin
 
 
-class RedisHook(BaseHook, LoggingMixin):
+class RedisHook(BaseHook):
     """
-    Hook to interact with Redis database
+    Wrapper for connection to interact with Redis in-memory data structure store
     """
     def __init__(self, redis_conn_id='redis_default'):
         """
@@ -39,55 +36,31 @@ def __init__(self, redis_conn_id='redis_default'):
                             we need to connect to Redis.
         """
         self.redis_conn_id = redis_conn_id
-        self.client = None
-        conn = self.get_connection(self.redis_conn_id)
-        self.host = conn.host
-        self.port = int(conn.port)
-        self.password = conn.password
-        self.db = int(conn.extra_dejson.get('db', 0))
-
-        self.log.debug(
-            '''Connection "{conn}":
-            \thost: {host}
-            \tport: {port}
-            \textra: {extra}
-            '''.format(
-                conn=self.redis_conn_id,
-                host=self.host,
-                port=self.port,
-                extra=conn.extra_dejson
-            )
-        )
+        self.redis = None
+        self.host = None
+        self.port = None
+        self.password = None
+        self.db = None
 
     def get_conn(self):
         """
         Returns a Redis connection.
         """
-        if not self.client:
+        conn = self.get_connection(self.redis_conn_id)
+        self.host = conn.host
+        self.port = conn.port
+        self.password = None if str(conn.password).lower() in ['none', 'false', ''] else conn.password
+        self.db = conn.extra_dejson.get('db', None)
+
+        if not self.redis:
             self.log.debug(
-                'generating Redis client for conn_id "%s" on %s:%s:%s',
+                'Initializing redis object for conn_id "%s" on %s:%s:%s',
                 self.redis_conn_id, self.host, self.port, self.db
             )
-            try:
-                self.client = StrictRedis(
-                    host=self.host,
-                    port=self.port,
-                    password=self.password,
-                    db=self.db)
-            except Exception as general_error:
-                raise AirflowException(
-                    'Failed to create Redis client, error: {error}'.format(
-                        error=str(general_error)
-                    )
-                )
-
-        return self.client
-
-    def key_exists(self, key):
-        """
-        Checks if a key exists in Redis database
+            self.redis = Redis(
+                host=self.host,
+                port=self.port,
+                password=self.password,
+                db=self.db)
 
-        :param key: The key to check the existence.
-        :type key: string
-        """
-        return self.get_conn().exists(key)
+        return self.redis
diff --git a/airflow/contrib/hooks/sagemaker_hook.py b/airflow/contrib/hooks/sagemaker_hook.py
new file mode 100644
index 0000000000000..80777821fef39
--- /dev/null
+++ b/airflow/contrib/hooks/sagemaker_hook.py
@@ -0,0 +1,770 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import tarfile
+import tempfile
+import time
+import os
+import collections
+
+import botocore.config
+from botocore.exceptions import ClientError
+
+from airflow.exceptions import AirflowException
+from airflow.contrib.hooks.aws_hook import AwsHook
+from airflow.hooks.S3_hook import S3Hook
+from airflow.utils import timezone
+
+
+class LogState(object):
+    STARTING = 1
+    WAIT_IN_PROGRESS = 2
+    TAILING = 3
+    JOB_COMPLETE = 4
+    COMPLETE = 5
+
+
+# Position is a tuple that includes the last read timestamp and the number of items that were read
+# at that time. This is used to figure out which event to start with on the next read.
+Position = collections.namedtuple('Position', ['timestamp', 'skip'])
+
+
+def argmin(arr, f):
+    """Return the index, i, in arr that minimizes f(arr[i])"""
+    m = None
+    i = None
+    for idx, item in enumerate(arr):
+        if item is not None:
+            if m is None or f(item) < m:
+                m = f(item)
+                i = idx
+    return i
+
+
+def secondary_training_status_changed(current_job_description, prev_job_description):
+    """
+    Returns true if training job's secondary status message has changed.
+
+    :param current_job_description: Current job description, returned from DescribeTrainingJob call.
+    :type current_job_description: dict
+    :param prev_job_description: Previous job description, returned from DescribeTrainingJob call.
+    :type prev_job_description: dict
+
+    :return: Whether the secondary status message of a training job changed or not.
+    """
+    current_secondary_status_transitions = current_job_description.get('SecondaryStatusTransitions')
+    if current_secondary_status_transitions is None or len(current_secondary_status_transitions) == 0:
+        return False
+
+    prev_job_secondary_status_transitions = prev_job_description.get('SecondaryStatusTransitions') \
+        if prev_job_description is not None else None
+
+    last_message = prev_job_secondary_status_transitions[-1]['StatusMessage'] \
+        if prev_job_secondary_status_transitions is not None \
+        and len(prev_job_secondary_status_transitions) > 0 else ''
+
+    message = current_job_description['SecondaryStatusTransitions'][-1]['StatusMessage']
+
+    return message != last_message
+
+
+def secondary_training_status_message(job_description, prev_description):
+    """
+    Returns a string contains start time and the secondary training job status message.
+
+    :param job_description: Returned response from DescribeTrainingJob call
+    :type job_description: dict
+    :param prev_description: Previous job description from DescribeTrainingJob call
+    :type prev_description: dict
+
+    :return: Job status string to be printed.
+    """
+
+    if job_description is None or job_description.get('SecondaryStatusTransitions') is None\
+            or len(job_description.get('SecondaryStatusTransitions')) == 0:
+        return ''
+
+    prev_description_secondary_transitions = prev_description.get('SecondaryStatusTransitions')\
+        if prev_description is not None else None
+    prev_transitions_num = len(prev_description['SecondaryStatusTransitions'])\
+        if prev_description_secondary_transitions is not None else 0
+    current_transitions = job_description['SecondaryStatusTransitions']
+
+    transitions_to_print = current_transitions[-1:] if len(current_transitions) == prev_transitions_num else \
+        current_transitions[prev_transitions_num - len(current_transitions):]
+
+    status_strs = []
+    for transition in transitions_to_print:
+        message = transition['StatusMessage']
+        time_str = timezone.convert_to_utc(job_description['LastModifiedTime']).strftime('%Y-%m-%d %H:%M:%S')
+        status_strs.append('{} {} - {}'.format(time_str, transition['Status'], message))
+
+    return '\n'.join(status_strs)
+
+
+class SageMakerHook(AwsHook):
+    """
+    Interact with Amazon SageMaker.
+    """
+    non_terminal_states = {'InProgress', 'Stopping'}
+    endpoint_non_terminal_states = {'Creating', 'Updating', 'SystemUpdating',
+                                    'RollingBack', 'Deleting'}
+    failed_states = {'Failed'}
+
+    def __init__(self,
+                 *args, **kwargs):
+        super(SageMakerHook, self).__init__(*args, **kwargs)
+        self.s3_hook = S3Hook(aws_conn_id=self.aws_conn_id)
+
+    def tar_and_s3_upload(self, path, key, bucket):
+        """
+        Tar the local file or directory and upload to s3
+
+        :param path: local file or directory
+        :type path: str
+        :param key: s3 key
+        :type key: str
+        :param bucket: s3 bucket
+        :type bucket: str
+        :return: None
+        """
+        with tempfile.TemporaryFile() as temp_file:
+            if os.path.isdir(path):
+                files = [os.path.join(path, name) for name in os.listdir(path)]
+            else:
+                files = [path]
+            with tarfile.open(mode='w:gz', fileobj=temp_file) as tar_file:
+                for f in files:
+                    tar_file.add(f, arcname=os.path.basename(f))
+            temp_file.seek(0)
+            self.s3_hook.load_file_obj(temp_file, key, bucket, replace=True)
+
+    def configure_s3_resources(self, config):
+        """
+        Extract the S3 operations from the configuration and execute them.
+
+        :param config: config of SageMaker operation
+        :type config: dict
+        :rtype: dict
+        """
+        s3_operations = config.pop('S3Operations', None)
+
+        if s3_operations is not None:
+            create_bucket_ops = s3_operations.get('S3CreateBucket', [])
+            upload_ops = s3_operations.get('S3Upload', [])
+            for op in create_bucket_ops:
+                self.s3_hook.create_bucket(bucket_name=op['Bucket'])
+            for op in upload_ops:
+                if op['Tar']:
+                    self.tar_and_s3_upload(op['Path'], op['Key'],
+                                           op['Bucket'])
+                else:
+                    self.s3_hook.load_file(op['Path'], op['Key'],
+                                           op['Bucket'])
+
+    def check_s3_url(self, s3url):
+        """
+        Check if an S3 URL exists
+
+        :param s3url: S3 url
+        :type s3url: str
+        :rtype: bool
+        """
+        bucket, key = S3Hook.parse_s3_url(s3url)
+        if not self.s3_hook.check_for_bucket(bucket_name=bucket):
+            raise AirflowException(
+                "The input S3 Bucket {} does not exist ".format(bucket))
+        if key and not self.s3_hook.check_for_key(key=key, bucket_name=bucket)\
+           and not self.s3_hook.check_for_prefix(
+                prefix=key, bucket_name=bucket, delimiter='/'):
+            # check if s3 key exists in the case user provides a single file
+            # or if s3 prefix exists in the case user provides multiple files in
+            # a prefix
+            raise AirflowException("The input S3 Key "
+                                   "or Prefix {} does not exist in the Bucket {}"
+                                   .format(s3url, bucket))
+        return True
+
+    def check_training_config(self, training_config):
+        """
+        Check if a training configuration is valid
+
+        :param training_config: training_config
+        :type training_config: dict
+        :return: None
+        """
+        for channel in training_config['InputDataConfig']:
+            self.check_s3_url(channel['DataSource']['S3DataSource']['S3Uri'])
+
+    def check_tuning_config(self, tuning_config):
+        """
+        Check if a tuning configuration is valid
+
+        :param tuning_config: tuning_config
+        :type tuning_config: dict
+        :return: None
+        """
+        for channel in tuning_config['TrainingJobDefinition']['InputDataConfig']:
+            self.check_s3_url(channel['DataSource']['S3DataSource']['S3Uri'])
+
+    def get_conn(self):
+        """
+        Establish an AWS connection for SageMaker
+
+        :rtype: :py:class:`SageMaker.Client`
+        """
+        return self.get_client_type('sagemaker')
+
+    def get_log_conn(self):
+        """
+        Establish an AWS connection for retrieving logs during training
+
+        :rtype: CloudWatchLogs.Client
+        """
+        config = botocore.config.Config(retries={'max_attempts': 15})
+        return self.get_client_type('logs', config=config)
+
+    def log_stream(self, log_group, stream_name, start_time=0, skip=0):
+        """
+        A generator for log items in a single stream. This will yield all the
+        items that are available at the current moment.
+
+        :param log_group: The name of the log group.
+        :type log_group: str
+        :param stream_name: The name of the specific stream.
+        :type stream_name: str
+        :param start_time: The time stamp value to start reading the logs from (default: 0).
+        :type start_time: int
+        :param skip: The number of log entries to skip at the start (default: 0).
+            This is for when there are multiple entries at the same timestamp.
+        :type skip: int
+        :rtype: dict
+        :return: | A CloudWatch log event with the following key-value pairs:
+                 |   'timestamp' (int): The time in milliseconds of the event.
+                 |   'message' (str): The log event data.
+                 |   'ingestionTime' (int): The time in milliseconds the event was ingested.
+        """
+
+        next_token = None
+
+        event_count = 1
+        while event_count > 0:
+            if next_token is not None:
+                token_arg = {'nextToken': next_token}
+            else:
+                token_arg = {}
+
+            response = self.get_log_conn().get_log_events(logGroupName=log_group,
+                                                          logStreamName=stream_name,
+                                                          startTime=start_time,
+                                                          startFromHead=True,
+                                                          **token_arg)
+            next_token = response['nextForwardToken']
+            events = response['events']
+            event_count = len(events)
+            if event_count > skip:
+                events = events[skip:]
+                skip = 0
+            else:
+                skip = skip - event_count
+                events = []
+            for ev in events:
+                yield ev
+
+    def multi_stream_iter(self, log_group, streams, positions=None):
+        """
+        Iterate over the available events coming from a set of log streams in a single log group
+        interleaving the events from each stream so they're yielded in timestamp order.
+
+        :param log_group: The name of the log group.
+        :type log_group: str
+        :param streams: A list of the log stream names. The position of the stream in this list is
+            the stream number.
+        :type streams: list
+        :param positions: A list of pairs of (timestamp, skip) which represents the last record
+            read from each stream.
+        :type positions: list
+        :return: A tuple of (stream number, cloudwatch log event).
+        """
+        positions = positions or {s: Position(timestamp=0, skip=0) for s in streams}
+        event_iters = [self.log_stream(log_group, s, positions[s].timestamp, positions[s].skip)
+                       for s in streams]
+        events = []
+        for s in event_iters:
+            if not s:
+                events.append(None)
+                continue
+            try:
+                events.append(next(s))
+            except StopIteration:
+                events.append(None)
+
+        while any(events):
+            i = argmin(events, lambda x: x['timestamp'] if x else 9999999999)
+            yield (i, events[i])
+            try:
+                events[i] = next(event_iters[i])
+            except StopIteration:
+                events[i] = None
+
+    def create_training_job(self, config, wait_for_completion=True, print_log=True,
+                            check_interval=30, max_ingestion_time=None):
+        """
+        Create a training job
+
+        :param config: the config for training
+        :type config: dict
+        :param wait_for_completion: if the program should keep running until job finishes
+        :type wait_for_completion: bool
+        :param check_interval: the time interval in seconds which the operator
+            will check the status of any SageMaker job
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :return: A response to training job creation
+        """
+
+        self.check_training_config(config)
+
+        response = self.get_conn().create_training_job(**config)
+        if print_log:
+            self.check_training_status_with_log(config['TrainingJobName'],
+                                                self.non_terminal_states,
+                                                self.failed_states,
+                                                wait_for_completion,
+                                                check_interval, max_ingestion_time
+                                                )
+        elif wait_for_completion:
+            describe_response = self.check_status(config['TrainingJobName'],
+                                                  'TrainingJobStatus',
+                                                  self.describe_training_job,
+                                                  check_interval, max_ingestion_time
+                                                  )
+
+            billable_time = \
+                (describe_response['TrainingEndTime'] - describe_response['TrainingStartTime']) * \
+                describe_response['ResourceConfig']['InstanceCount']
+            self.log.info('Billable seconds:{}'.format(int(billable_time.total_seconds()) + 1))
+
+        return response
+
+    def create_tuning_job(self, config, wait_for_completion=True,
+                          check_interval=30, max_ingestion_time=None):
+        """
+        Create a tuning job
+
+        :param config: the config for tuning
+        :type config: dict
+        :param wait_for_completion: if the program should keep running until job finishes
+        :type wait_for_completion: bool
+        :param check_interval: the time interval in seconds which the operator
+            will check the status of any SageMaker job
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :return: A response to tuning job creation
+        """
+
+        self.check_tuning_config(config)
+
+        response = self.get_conn().create_hyper_parameter_tuning_job(**config)
+        if wait_for_completion:
+            self.check_status(config['HyperParameterTuningJobName'],
+                              'HyperParameterTuningJobStatus',
+                              self.describe_tuning_job,
+                              check_interval, max_ingestion_time
+                              )
+        return response
+
+    def create_transform_job(self, config, wait_for_completion=True,
+                             check_interval=30, max_ingestion_time=None):
+        """
+        Create a transform job
+
+        :param config: the config for transform job
+        :type config: dict
+        :param wait_for_completion: if the program should keep running until job finishes
+        :type wait_for_completion: bool
+        :param check_interval: the time interval in seconds which the operator
+            will check the status of any SageMaker job
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :return: A response to transform job creation
+        """
+
+        self.check_s3_url(config['TransformInput']['DataSource']['S3DataSource']['S3Uri'])
+
+        response = self.get_conn().create_transform_job(**config)
+        if wait_for_completion:
+            self.check_status(config['TransformJobName'],
+                              'TransformJobStatus',
+                              self.describe_transform_job,
+                              check_interval, max_ingestion_time
+                              )
+        return response
+
+    def create_model(self, config):
+        """
+        Create a model job
+
+        :param config: the config for model
+        :type config: dict
+        :return: A response to model creation
+        """
+
+        return self.get_conn().create_model(**config)
+
+    def create_endpoint_config(self, config):
+        """
+        Create an endpoint config
+
+        :param config: the config for endpoint-config
+        :type config: dict
+        :return: A response to endpoint config creation
+        """
+
+        return self.get_conn().create_endpoint_config(**config)
+
+    def create_endpoint(self, config, wait_for_completion=True,
+                        check_interval=30, max_ingestion_time=None):
+        """
+        Create an endpoint
+
+        :param config: the config for endpoint
+        :type config: dict
+        :param wait_for_completion: if the program should keep running until job finishes
+        :type wait_for_completion: bool
+        :param check_interval: the time interval in seconds which the operator
+            will check the status of any SageMaker job
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :return: A response to endpoint creation
+        """
+
+        response = self.get_conn().create_endpoint(**config)
+        if wait_for_completion:
+            self.check_status(config['EndpointName'],
+                              'EndpointStatus',
+                              self.describe_endpoint,
+                              check_interval, max_ingestion_time,
+                              non_terminal_states=self.endpoint_non_terminal_states
+                              )
+        return response
+
+    def update_endpoint(self, config, wait_for_completion=True,
+                        check_interval=30, max_ingestion_time=None):
+        """
+        Update an endpoint
+
+        :param config: the config for endpoint
+        :type config: dict
+        :param wait_for_completion: if the program should keep running until job finishes
+        :type wait_for_completion: bool
+        :param check_interval: the time interval in seconds which the operator
+            will check the status of any SageMaker job
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :return: A response to endpoint update
+        """
+
+        response = self.get_conn().update_endpoint(**config)
+        if wait_for_completion:
+            self.check_status(config['EndpointName'],
+                              'EndpointStatus',
+                              self.describe_endpoint,
+                              check_interval, max_ingestion_time,
+                              non_terminal_states=self.endpoint_non_terminal_states
+                              )
+        return response
+
+    def describe_training_job(self, name):
+        """
+        Return the training job info associated with the name
+
+        :param name: the name of the training job
+        :type name: str
+        :return: A dict contains all the training job info
+        """
+
+        return self.get_conn().describe_training_job(TrainingJobName=name)
+
+    def describe_training_job_with_log(self, job_name, positions, stream_names,
+                                       instance_count, state, last_description,
+                                       last_describe_job_call):
+        """
+        Return the training job info associated with job_name and print CloudWatch logs
+        """
+        log_group = '/aws/sagemaker/TrainingJobs'
+
+        if len(stream_names) < instance_count:
+            # Log streams are created whenever a container starts writing to stdout/err, so this list
+            # may be dynamic until we have a stream for every instance.
+            logs_conn = self.get_log_conn()
+            try:
+                streams = logs_conn.describe_log_streams(
+                    logGroupName=log_group,
+                    logStreamNamePrefix=job_name + '/',
+                    orderBy='LogStreamName',
+                    limit=instance_count
+                )
+                stream_names = [s['logStreamName'] for s in streams['logStreams']]
+                positions.update([(s, Position(timestamp=0, skip=0))
+                                  for s in stream_names if s not in positions])
+            except logs_conn.exceptions.ResourceNotFoundException:
+                # On the very first training job run on an account, there's no log group until
+                # the container starts logging, so ignore any errors thrown about that
+                pass
+
+        if len(stream_names) > 0:
+            for idx, event in self.multi_stream_iter(log_group, stream_names, positions):
+                self.log.info(event['message'])
+                ts, count = positions[stream_names[idx]]
+                if event['timestamp'] == ts:
+                    positions[stream_names[idx]] = Position(timestamp=ts, skip=count + 1)
+                else:
+                    positions[stream_names[idx]] = Position(timestamp=event['timestamp'], skip=1)
+
+        if state == LogState.COMPLETE:
+            return state, last_description, last_describe_job_call
+
+        if state == LogState.JOB_COMPLETE:
+            state = LogState.COMPLETE
+        elif time.time() - last_describe_job_call >= 30:
+            description = self.describe_training_job(job_name)
+            last_describe_job_call = time.time()
+
+            if secondary_training_status_changed(description, last_description):
+                self.log.info(secondary_training_status_message(description, last_description))
+                last_description = description
+
+            status = description['TrainingJobStatus']
+
+            if status not in self.non_terminal_states:
+                state = LogState.JOB_COMPLETE
+        return state, last_description, last_describe_job_call
+
+    def describe_tuning_job(self, name):
+        """
+        Return the tuning job info associated with the name
+
+        :param name: the name of the tuning job
+        :type name: string
+        :return: A dict contains all the tuning job info
+        """
+
+        return self.get_conn().describe_hyper_parameter_tuning_job(HyperParameterTuningJobName=name)
+
+    def describe_model(self, name):
+        """
+        Return the SageMaker model info associated with the name
+
+        :param name: the name of the SageMaker model
+        :type name: string
+        :return: A dict contains all the model info
+        """
+
+        return self.get_conn().describe_model(ModelName=name)
+
+    def describe_transform_job(self, name):
+        """
+        Return the transform job info associated with the name
+
+        :param name: the name of the transform job
+        :type name: string
+        :return: A dict contains all the transform job info
+        """
+
+        return self.get_conn().describe_transform_job(TransformJobName=name)
+
+    def describe_endpoint_config(self, name):
+        """
+        Return the endpoint config info associated with the name
+
+        :param name: the name of the endpoint config
+        :type name: string
+        :return: A dict contains all the endpoint config info
+        """
+
+        return self.get_conn().describe_endpoint_config(EndpointConfigName=name)
+
+    def describe_endpoint(self, name):
+        """
+        :param name: the name of the endpoint
+        :type name: string
+        :return: A dict contains all the endpoint info
+        """
+
+        return self.get_conn().describe_endpoint(EndpointName=name)
+
+    def check_status(self, job_name, key,
+                     describe_function, check_interval,
+                     max_ingestion_time,
+                     non_terminal_states=None):
+        """
+        Check status of a SageMaker job
+
+        :param job_name: name of the job to check status
+        :type job_name: str
+        :param key: the key of the response dict
+            that points to the state
+        :type key: str
+        :param describe_function: the function used to retrieve the status
+        :type describe_function: python callable
+        :param args: the arguments for the function
+        :param check_interval: the time interval in seconds which the operator
+            will check the status of any SageMaker job
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :param non_terminal_states: the set of nonterminal states
+        :type non_terminal_states: set
+        :return: response of describe call after job is done
+        """
+        if not non_terminal_states:
+            non_terminal_states = self.non_terminal_states
+
+        sec = 0
+        running = True
+
+        while running:
+            time.sleep(check_interval)
+            sec = sec + check_interval
+
+            try:
+                response = describe_function(job_name)
+                status = response[key]
+                self.log.info('Job still running for %s seconds... '
+                              'current status is %s' % (sec, status))
+            except KeyError:
+                raise AirflowException('Could not get status of the SageMaker job')
+            except ClientError:
+                raise AirflowException('AWS request failed, check logs for more info')
+
+            if status in non_terminal_states:
+                running = True
+            elif status in self.failed_states:
+                raise AirflowException('SageMaker job failed because %s' % response['FailureReason'])
+            else:
+                running = False
+
+            if max_ingestion_time and sec > max_ingestion_time:
+                # ensure that the job gets killed if the max ingestion time is exceeded
+                raise AirflowException('SageMaker job took more than %s seconds', max_ingestion_time)
+
+        self.log.info('SageMaker Job Compeleted')
+        response = describe_function(job_name)
+        return response
+
+    def check_training_status_with_log(self, job_name, non_terminal_states, failed_states,
+                                       wait_for_completion, check_interval, max_ingestion_time):
+        """
+        Display the logs for a given training job, optionally tailing them until the
+        job is complete.
+
+        :param job_name: name of the training job to check status and display logs for
+        :type job_name: str
+        :param non_terminal_states: the set of non_terminal states
+        :type non_terminal_states: set
+        :param failed_states: the set of failed states
+        :type failed_states: set
+        :param wait_for_completion: Whether to keep looking for new log entries
+            until the job completes
+        :type wait_for_completion: bool
+        :param check_interval: The interval in seconds between polling for new log entries and job completion
+        :type check_interval: int
+        :param max_ingestion_time: the maximum ingestion time in seconds. Any
+            SageMaker jobs that run longer than this will fail. Setting this to
+            None implies no timeout for any SageMaker job.
+        :type max_ingestion_time: int
+        :return: None
+        """
+
+        sec = 0
+        description = self.describe_training_job(job_name)
+        self.log.info(secondary_training_status_message(description, None))
+        instance_count = description['ResourceConfig']['InstanceCount']
+        status = description['TrainingJobStatus']
+
+        stream_names = []  # The list of log streams
+        positions = {}     # The current position in each stream, map of stream name -> position
+
+        job_already_completed = status not in non_terminal_states
+
+        state = LogState.TAILING if wait_for_completion and not job_already_completed else LogState.COMPLETE
+
+        # The loop below implements a state machine that alternates between checking the job status and
+        # reading whatever is available in the logs at this point. Note, that if we were called with
+        # wait_for_completion == False, we never check the job status.
+        #
+        # If wait_for_completion == TRUE and job is not completed, the initial state is TAILING
+        # If wait_for_completion == FALSE, the initial state is COMPLETE
+        # (doesn't matter if the job really is complete).
+        #
+        # The state table:
+        #
+        # STATE               ACTIONS                        CONDITION             NEW STATE
+        # ----------------    ----------------               -----------------     ----------------
+        # TAILING             Read logs, Pause, Get status   Job complete          JOB_COMPLETE
+        #                                                    Else                  TAILING
+        # JOB_COMPLETE        Read logs, Pause               Any                   COMPLETE
+        # COMPLETE            Read logs, Exit                                      N/A
+        #
+        # Notes:
+        # - The JOB_COMPLETE state forces us to do an extra pause and read any items that
+        # got to Cloudwatch after the job was marked complete.
+        last_describe_job_call = time.time()
+        last_description = description
+
+        while True:
+            time.sleep(check_interval)
+            sec = sec + check_interval
+
+            state, last_description, last_describe_job_call = \
+                self.describe_training_job_with_log(job_name, positions, stream_names,
+                                                    instance_count, state, last_description,
+                                                    last_describe_job_call)
+            if state == LogState.COMPLETE:
+                break
+
+            if max_ingestion_time and sec > max_ingestion_time:
+                # ensure that the job gets killed if the max ingestion time is exceeded
+                raise AirflowException('SageMaker job took more than %s seconds', max_ingestion_time)
+
+        if wait_for_completion:
+            status = last_description['TrainingJobStatus']
+            if status in failed_states:
+                reason = last_description.get('FailureReason', '(No reason provided)')
+                raise AirflowException('Error training {}: {} Reason: {}'.format(job_name, status, reason))
+            billable_time = (last_description['TrainingEndTime'] - last_description['TrainingStartTime']) \
+                * instance_count
+            self.log.info('Billable seconds:{}'.format(int(billable_time.total_seconds()) + 1))
diff --git a/airflow/contrib/hooks/salesforce_hook.py b/airflow/contrib/hooks/salesforce_hook.py
index ee18b353d2e82..a1756b6530b6a 100644
--- a/airflow/contrib/hooks/salesforce_hook.py
+++ b/airflow/contrib/hooks/salesforce_hook.py
@@ -37,7 +37,7 @@
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 
-class SalesforceHook(BaseHook, LoggingMixin):
+class SalesforceHook(BaseHook):
     def __init__(
             self,
             conn_id,
@@ -53,14 +53,14 @@ def __init__(
 
         :param conn_id:     the name of the connection that has the parameters
                             we need to connect to Salesforce.
-                            The conenction shoud be type `http` and include a
+                            The connection should be type `http` and include a
                             user's security token in the `Extras` field.
         .. note::
             For the HTTP connection type, you can include a
             JSON structure in the `Extras` field.
             We need a user's security token to connect to Salesforce.
             So we define it in the `Extras` field as:
-                `{"security_token":"YOUR_SECRUITY_TOKEN"}`
+                `{"security_token":"YOUR_SECURITY_TOKEN"}`
         """
         self.conn_id = conn_id
         self._args = args
@@ -135,7 +135,8 @@ def get_available_fields(self, obj):
 
         return [f['name'] for f in desc['fields']]
 
-    def _build_field_list(self, fields):
+    @staticmethod
+    def _build_field_list(fields):
         # join all of the fields in a comma separated list
         return ",".join(fields)
 
@@ -275,7 +276,7 @@ def write_object_to_file(
 
             schema = self.describe_object(object_name)
 
-            # possible columns that can be convereted to timestamps
+            # possible columns that can be converted to timestamps
             # are the ones that are either date or datetime types
             # strings are too general and we risk unintentional conversion
             possible_timestamp_cols = [
diff --git a/airflow/contrib/hooks/segment_hook.py b/airflow/contrib/hooks/segment_hook.py
index 874d35d0743b9..a072a9f1eaf99 100644
--- a/airflow/contrib/hooks/segment_hook.py
+++ b/airflow/contrib/hooks/segment_hook.py
@@ -29,10 +29,8 @@
 from airflow.hooks.base_hook import BaseHook
 from airflow.exceptions import AirflowException
 
-from airflow.utils.log.logging_mixin import LoggingMixin
 
-
-class SegmentHook(BaseHook, LoggingMixin):
+class SegmentHook(BaseHook):
     def __init__(
             self,
             segment_conn_id='segment_default',
@@ -54,7 +52,7 @@ def __init__(
         :type segment_conn_id: str
         :param segment_debug_mode: Determines whether Segment should run in debug mode.
         Defaults to False
-        :type segment_debug_mode: boolean
+        :type segment_debug_mode: bool
         .. note::
             You must include a JSON structure in the `Extras` field.
             We need a user's security token to connect to Segment.
diff --git a/airflow/contrib/hooks/sftp_hook.py b/airflow/contrib/hooks/sftp_hook.py
index 33c8b19e0a732..6fc01babe5ab2 100644
--- a/airflow/contrib/hooks/sftp_hook.py
+++ b/airflow/contrib/hooks/sftp_hook.py
@@ -19,52 +19,98 @@
 
 import stat
 import pysftp
-import logging
 import datetime
-from airflow.hooks.base_hook import BaseHook
+from airflow.contrib.hooks.ssh_hook import SSHHook
 
 
-class SFTPHook(BaseHook):
+class SFTPHook(SSHHook):
     """
+    This hook is inherited from SSH hook. Please refer to SSH hook for the input
+    arguments.
+
     Interact with SFTP. Aims to be interchangeable with FTPHook.
 
-    Pitfalls: - In contrast with FTPHook describe_directory only returns size, type and
-                modify. It doesn't return unix.owner, unix.mode, perm, unix.group and
-                unique.
-              - retrieve_file and store_file only take a local full path and not a
-                buffer.
-              - If no mode is passed to create_directory it will be created with 777
-                permissions.
+    :Pitfalls::
+
+        - In contrast with FTPHook describe_directory only returns size, type and
+          modify. It doesn't return unix.owner, unix.mode, perm, unix.group and
+          unique.
+        - retrieve_file and store_file only take a local full path and not a
+           buffer.
+        - If no mode is passed to create_directory it will be created with 777
+          permissions.
 
     Errors that may occur throughout but should be handled downstream.
     """
 
-    def __init__(self, ftp_conn_id='sftp_default'):
-        self.ftp_conn_id = ftp_conn_id
+    def __init__(self, ftp_conn_id='sftp_default', *args, **kwargs):
+        kwargs['ssh_conn_id'] = ftp_conn_id
+        super(SFTPHook, self).__init__(*args, **kwargs)
+
         self.conn = None
+        self.private_key_pass = None
+
+        # Fail for unverified hosts, unless this is explicitly allowed
+        self.no_host_key_check = False
+
+        if self.ssh_conn_id is not None:
+            conn = self.get_connection(self.ssh_conn_id)
+            if conn.extra is not None:
+                extra_options = conn.extra_dejson
+                if 'private_key_pass' in extra_options:
+                    self.private_key_pass = extra_options.get('private_key_pass', None)
+
+                # For backward compatibility
+                # TODO: remove in Airflow 2.1
+                import warnings
+                if 'ignore_hostkey_verification' in extra_options:
+                    warnings.warn(
+                        'Extra option `ignore_hostkey_verification` is deprecated.'
+                        'Please use `no_host_key_check` instead.'
+                        'This option will be removed in Airflow 2.1',
+                        DeprecationWarning,
+                        stacklevel=2,
+                    )
+                    self.no_host_key_check = str(
+                        extra_options['ignore_hostkey_verification']
+                    ).lower() == 'true'
+
+                if 'no_host_key_check' in extra_options:
+                    self.no_host_key_check = str(
+                        extra_options['no_host_key_check']).lower() == 'true'
+
+                if 'private_key' in extra_options:
+                    warnings.warn(
+                        'Extra option `private_key` is deprecated.'
+                        'Please use `key_file` instead.'
+                        'This option will be removed in Airflow 2.1',
+                        DeprecationWarning,
+                        stacklevel=2,
+                    )
+                    self.key_file = extra_options.get('private_key')
 
     def get_conn(self):
         """
         Returns an SFTP connection object
         """
         if self.conn is None:
-            params = self.get_connection(self.ftp_conn_id)
             cnopts = pysftp.CnOpts()
-            if ('ignore_hostkey_verification' in params.extra_dejson and
-                    params.extra_dejson['ignore_hostkey_verification']):
+            if self.no_host_key_check:
                 cnopts.hostkeys = None
+            cnopts.compression = self.compress
             conn_params = {
-                'host': params.host,
-                'port': params.port,
-                'username': params.login,
+                'host': self.remote_host,
+                'port': self.port,
+                'username': self.username,
                 'cnopts': cnopts
             }
-            if params.password is not None:
-                conn_params['password'] = params.password
-            if 'private_key' in params.extra_dejson:
-                conn_params['private_key'] = params.extra_dejson['private_key']
-            if 'private_key_pass' in params.extra_dejson:
-                conn_params['private_key_pass'] = params.extra_dejson['private_key_pass']
+            if self.password and self.password.strip():
+                conn_params['password'] = self.password
+            if self.key_file:
+                conn_params['private_key'] = self.key_file
+            if self.private_key_pass:
+                conn_params['private_key_pass'] = self.private_key_pass
+
             self.conn = pysftp.Connection(**conn_params)
         return self.conn
 
@@ -136,10 +182,9 @@ def retrieve_file(self, remote_full_path, local_full_path):
         :type local_full_path: str
         """
         conn = self.get_conn()
-        logging.info('Retrieving file from FTP: {}'.format(remote_full_path))
+        self.log.info('Retrieving file from FTP: %s', remote_full_path)
         conn.get(remote_full_path, local_full_path)
-        logging.info('Finished retrieving file from FTP: {}'.format(
-            remote_full_path))
+        self.log.info('Finished retrieving file from FTP: %s', remote_full_path)
 
     def store_file(self, remote_full_path, local_full_path):
         """
diff --git a/airflow/contrib/hooks/slack_webhook_hook.py b/airflow/contrib/hooks/slack_webhook_hook.py
index 670d40105e669..aca10385ff901 100644
--- a/airflow/contrib/hooks/slack_webhook_hook.py
+++ b/airflow/contrib/hooks/slack_webhook_hook.py
@@ -38,6 +38,9 @@ class SlackWebhookHook(HttpHook):
     :type webhook_token: str
     :param message: The message you want to send on Slack
     :type message: str
+    :param attachments: The attachments to send on Slack. Should be a list of
+                        dictionaries representing Slack attachments.
+    :type attachments: list
     :param channel: The channel the message should be posted to
     :type channel: str
     :param username: The username to post to slack with
@@ -54,6 +57,7 @@ def __init__(self,
                  http_conn_id=None,
                  webhook_token=None,
                  message="",
+                 attachments=None,
                  channel=None,
                  username=None,
                  icon_emoji=None,
@@ -63,9 +67,9 @@ def __init__(self,
                  **kwargs
                  ):
         super(SlackWebhookHook, self).__init__(*args, **kwargs)
-        self.http_conn_id = http_conn_id
         self.webhook_token = self._get_token(webhook_token, http_conn_id)
         self.message = message
+        self.attachments = attachments
         self.channel = channel
         self.username = username
         self.icon_emoji = icon_emoji
@@ -76,7 +80,9 @@ def _get_token(self, token, http_conn_id):
         """
         Given either a manually set token or a conn_id, return the webhook_token to use
         :param token: The manually provided token
-        :param conn_id: The conn_id provided
+        :type token: str
+        :param http_conn_id: The conn_id provided
+        :type http_conn_id: str
         :return: webhook_token (str) to use
         """
         if token:
@@ -105,17 +111,15 @@ def _build_slack_message(self):
             cmd['icon_emoji'] = self.icon_emoji
         if self.link_names:
             cmd['link_names'] = 1
+        if self.attachments:
+            cmd['attachments'] = self.attachments
 
-        # there should always be a message to post ;-)
         cmd['text'] = self.message
         return json.dumps(cmd)
 
     def execute(self):
         """
         Remote Popen (actually execute the slack webhook call)
-
-        :param cmd: command to remotely execute
-        :param kwargs: extra arguments to Popen (see subprocess.Popen)
         """
         proxies = {}
         if self.proxy:
diff --git a/airflow/contrib/hooks/spark_jdbc_hook.py b/airflow/contrib/hooks/spark_jdbc_hook.py
index b55e4ef060419..c188b1e863de2 100644
--- a/airflow/contrib/hooks/spark_jdbc_hook.py
+++ b/airflow/contrib/hooks/spark_jdbc_hook.py
@@ -62,7 +62,7 @@ class SparkJDBCHook(SparkSubmitHook):
     :param jdbc_table: The name of the JDBC table
     :type jdbc_table: str
     :param jdbc_conn_id: Connection id used for connection to JDBC database
-    :type: jdbc_conn_id: str
+    :type jdbc_conn_id: str
     :param jdbc_driver: Name of the JDBC driver to use for the JDBC connection. This
                         driver (usually a jar) should be passed in the 'jars' parameter
     :type jdbc_driver: str
diff --git a/airflow/contrib/hooks/spark_sql_hook.py b/airflow/contrib/hooks/spark_sql_hook.py
index c1fd2ce21c89f..25c0bc53c5576 100644
--- a/airflow/contrib/hooks/spark_sql_hook.py
+++ b/airflow/contrib/hooks/spark_sql_hook.py
@@ -27,6 +27,7 @@ class SparkSqlHook(BaseHook):
     """
     This hook is a wrapper around the spark-sql binary. It requires that the
     "spark-sql" binary is in the PATH.
+
     :param sql: The SQL query to execute
     :type sql: str
     :param conf: arbitrary Spark configuration property
@@ -91,6 +92,7 @@ def _prepare_command(self, cmd):
         """
         Construct the spark-sql command to execute. Verbose output is enabled
         as default.
+
         :param cmd: command to append to the spark-sql command
         :type cmd: str
         :return: full command to be executed
diff --git a/airflow/contrib/hooks/spark_submit_hook.py b/airflow/contrib/hooks/spark_submit_hook.py
index 0185cab283345..677ea9000843b 100644
--- a/airflow/contrib/hooks/spark_submit_hook.py
+++ b/airflow/contrib/hooks/spark_submit_hook.py
@@ -26,7 +26,6 @@
 from airflow.exceptions import AirflowException
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.contrib.kubernetes import kube_client
-from kubernetes.client.rest import ApiException
 
 
 class SparkSubmitHook(BaseHook, LoggingMixin):
@@ -34,37 +33,40 @@ class SparkSubmitHook(BaseHook, LoggingMixin):
     This hook is a wrapper around the spark-submit binary to kick off a spark-submit job.
     It requires that the "spark-submit" binary is in the PATH or the spark_home to be
     supplied.
+
     :param conf: Arbitrary Spark configuration properties
     :type conf: dict
     :param conn_id: The connection id as configured in Airflow administration. When an
-                    invalid connection_id is supplied, it will default to yarn.
+        invalid connection_id is supplied, it will default to yarn.
     :type conn_id: str
     :param files: Upload additional files to the executor running the job, separated by a
-                  comma. Files will be placed in the working directory of each executor.
-                  For example, serialized objects.
+        comma. Files will be placed in the working directory of each executor.
+        For example, serialized objects.
     :type files: str
     :param py_files: Additional python files used by the job, can be .zip, .egg or .py.
     :type py_files: str
-    :param driver_classpath: Additional, driver-specific, classpath settings.
-    :type driver_classpath: str
+    :param: archives: Archives that spark should unzip (and possibly tag with #ALIAS) into
+        the application working directory.
+    :param driver_class_path: Additional, driver-specific, classpath settings.
+    :type driver_class_path: str
     :param jars: Submit additional jars to upload and place them in executor classpath.
     :type jars: str
     :param java_class: the main class of the Java application
     :type java_class: str
     :param packages: Comma-separated list of maven coordinates of jars to include on the
-    driver and executor classpaths
+        driver and executor classpaths
     :type packages: str
     :param exclude_packages: Comma-separated list of maven coordinates of jars to exclude
-    while resolving the dependencies provided in 'packages'
+        while resolving the dependencies provided in 'packages'
     :type exclude_packages: str
     :param repositories: Comma-separated list of additional remote repositories to search
-    for the maven coordinates given with 'packages'
+        for the maven coordinates given with 'packages'
     :type repositories: str
     :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors
-    (Default: all the available cores on the worker)
+        (Default: all the available cores on the worker)
     :type total_executor_cores: int
     :param executor_cores: (Standalone, YARN and Kubernetes only) Number of cores per
-    executor (Default: 2)
+        executor (Default: 2)
     :type executor_cores: int
     :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)
     :type executor_memory: str
@@ -81,17 +83,21 @@ class SparkSubmitHook(BaseHook, LoggingMixin):
     :param application_args: Arguments for the application being submitted
     :type application_args: list
     :param env_vars: Environment variables for spark-submit. It
-                     supports yarn and k8s mode too.
+        supports yarn and k8s mode too.
     :type env_vars: dict
     :param verbose: Whether to pass the verbose flag to spark-submit process for debugging
     :type verbose: bool
+    :param spark_binary: The command to use for spark submit.
+                         Some distros may use spark2-submit.
+    :type spark_binary: str
     """
     def __init__(self,
                  conf=None,
                  conn_id='spark_default',
                  files=None,
                  py_files=None,
-                 driver_classpath=None,
+                 archives=None,
+                 driver_class_path=None,
                  jars=None,
                  java_class=None,
                  packages=None,
@@ -107,12 +113,14 @@ def __init__(self,
                  num_executors=None,
                  application_args=None,
                  env_vars=None,
-                 verbose=False):
+                 verbose=False,
+                 spark_binary="spark-submit"):
         self._conf = conf
         self._conn_id = conn_id
         self._files = files
         self._py_files = py_files
-        self._driver_classpath = driver_classpath
+        self._archives = archives
+        self._driver_class_path = driver_class_path
         self._jars = jars
         self._java_class = java_class
         self._packages = packages
@@ -132,10 +140,15 @@ def __init__(self,
         self._submit_sp = None
         self._yarn_application_id = None
         self._kubernetes_driver_pod = None
+        self._spark_binary = spark_binary
 
         self._connection = self._resolve_connection()
         self._is_yarn = 'yarn' in self._connection['master']
         self._is_kubernetes = 'k8s' in self._connection['master']
+        if self._is_kubernetes and kube_client is None:
+            raise RuntimeError(
+                "{} specified by kubernetes dependencies are not installed!".format(
+                    self._connection['master']))
 
         self._should_track_driver_status = self._resolve_should_track_driver_status()
         self._driver_id = None
@@ -157,7 +170,7 @@ def _resolve_connection(self):
                      'queue': None,
                      'deploy_mode': None,
                      'spark_home': None,
-                     'spark_binary': 'spark-submit',
+                     'spark_binary': self._spark_binary,
                      'namespace': 'default'}
 
         try:
@@ -174,7 +187,7 @@ def _resolve_connection(self):
             conn_data['queue'] = extra.get('queue', None)
             conn_data['deploy_mode'] = extra.get('deploy-mode', None)
             conn_data['spark_home'] = extra.get('spark-home', None)
-            conn_data['spark_binary'] = extra.get('spark-binary', 'spark-submit')
+            conn_data['spark_binary'] = extra.get('spark-binary', "spark-submit")
             conn_data['namespace'] = extra.get('namespace', 'default')
         except AirflowException:
             self.log.debug(
@@ -235,8 +248,10 @@ def _build_spark_submit_command(self, application):
             connection_cmd += ["--files", self._files]
         if self._py_files:
             connection_cmd += ["--py-files", self._py_files]
-        if self._driver_classpath:
-            connection_cmd += ["--driver-classpath", self._driver_classpath]
+        if self._archives:
+            connection_cmd += ["--archives", self._archives]
+        if self._driver_class_path:
+            connection_cmd += ["--driver-class-path", self._driver_class_path]
         if self._jars:
             connection_cmd += ["--jars", self._jars]
         if self._packages:
@@ -387,14 +402,14 @@ def _process_spark_submit_log(self, itr):
             # If we run Kubernetes cluster mode, we want to extract the driver pod id
             # from the logs so we can kill the application when we stop it unexpectedly
             elif self._is_kubernetes:
-                match = re.search('\s*pod name: ((.+?)-([a-z0-9]+)-driver)', line)
+                match = re.search(r'\s*pod name: ((.+?)-([a-z0-9]+)-driver)', line)
                 if match:
                     self._kubernetes_driver_pod = match.groups()[0]
                     self.log.info("Identified spark driver pod: %s",
                                   self._kubernetes_driver_pod)
 
                 # Store the Spark Exit code
-                match_exit_code = re.search('\s*Exit code: (\d+)', line)
+                match_exit_code = re.search(r'\s*Exit code: (\d+)', line)
                 if match_exit_code:
                     self._spark_exit_code = int(match_exit_code.groups()[0])
 
@@ -402,7 +417,7 @@ def _process_spark_submit_log(self, itr):
             # we need to extract the driver id from the logs. This allows us to poll for
             # the status using the driver id. Also, we can kill the driver when needed.
             elif self._should_track_driver_status and not self._driver_id:
-                match_driver_id = re.search('(driver-[0-9\-]+)', line)
+                match_driver_id = re.search(r'(driver-[0-9\-]+)', line)
                 if match_driver_id:
                     self._driver_id = match_driver_id.groups()[0]
                     self.log.info("identified spark driver id: {}"
@@ -437,16 +452,25 @@ def _start_driver_status_tracking(self):
         Finish failed when the status is ERROR/UNKNOWN/KILLED/FAILED.
 
         Possible status:
-            SUBMITTED: Submitted but not yet scheduled on a worker
-            RUNNING: Has been allocated to a worker to run
-            FINISHED: Previously ran and exited cleanly
-            RELAUNCHING: Exited non-zero or due to worker failure, but has not yet
+
+        SUBMITTED
+            Submitted but not yet scheduled on a worker
+        RUNNING
+            Has been allocated to a worker to run
+        FINISHED
+            Previously ran and exited cleanly
+        RELAUNCHING
+            Exited non-zero or due to worker failure, but has not yet
             started running again
-            UNKNOWN: The status of the driver is temporarily not known due to
-             master failure recovery
-            KILLED: A user manually killed this driver
-            FAILED: The driver exited non-zero and was not supervised
-            ERROR: Unable to run or restart due to an unrecoverable error
+        UNKNOWN
+            The status of the driver is temporarily not known due to
+            master failure recovery
+        KILLED
+            A user manually killed this driver
+        FAILED
+            The driver exited non-zero and was not supervised
+        ERROR
+            Unable to run or restart due to an unrecoverable error
             (e.g. missing jar file)
         """
 
@@ -559,6 +583,6 @@ def on_kill(self):
 
                     self.log.info("Spark on K8s killed with response: %s", api_response)
 
-                except ApiException as e:
+                except kube_client.ApiException as e:
                     self.log.info("Exception when attempting to kill Spark on K8s:")
                     self.log.exception(e)
diff --git a/airflow/contrib/hooks/sqoop_hook.py b/airflow/contrib/hooks/sqoop_hook.py
index 74cddc2b21027..aa5ec5222afd0 100644
--- a/airflow/contrib/hooks/sqoop_hook.py
+++ b/airflow/contrib/hooks/sqoop_hook.py
@@ -25,24 +25,24 @@
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.log.logging_mixin import LoggingMixin
 from copy import deepcopy
 
 
-class SqoopHook(BaseHook, LoggingMixin):
+class SqoopHook(BaseHook):
     """
     This hook is a wrapper around the sqoop 1 binary. To be able to use the hook
     it is required that "sqoop" is in the PATH.
 
     Additional arguments that can be passed via the 'extra' JSON field of the
     sqoop connection:
-    * job_tracker: Job tracker local|jobtracker:port.
-    * namenode: Namenode.
-    * lib_jars: Comma separated jar files to include in the classpath.
-    * files: Comma separated files to be copied to the map reduce cluster.
-    * archives: Comma separated archives to be unarchived on the compute
-        machines.
-    * password_file: Path to file containing the password.
+
+        * ``job_tracker``: Job tracker local|jobtracker:port.
+        * ``namenode``: Namenode.
+        * ``lib_jars``: Comma separated jar files to include in the classpath.
+        * ``files``: Comma separated files to be copied to the map reduce cluster.
+        * ``archives``: Comma separated archives to be unarchived on the compute
+            machines.
+        * ``password_file``: Path to file containing the password.
 
     :param conn_id: Reference to the sqoop connection.
     :type conn_id: str
@@ -195,7 +195,7 @@ def _import_cmd(self, target_dir, append, file_type, split_by, direct,
             for key, value in extra_import_options.items():
                 cmd += ['--{}'.format(key)]
                 if value:
-                    cmd += [value]
+                    cmd += [str(value)]
 
         return cmd
 
@@ -205,6 +205,7 @@ def import_table(self, table, target_dir=None, append=False, file_type="text",
         """
         Imports table from remote location to target dir. Arguments are
         copies of direct sqoop command line arguments
+
         :param table: Table to read
         :param target_dir: HDFS destination dir
         :param append: Append data to an existing dataset in HDFS
@@ -235,6 +236,7 @@ def import_query(self, query, target_dir, append=False, file_type="text",
                      split_by=None, direct=None, driver=None, extra_import_options=None):
         """
         Imports a specific query from the rdbms to hdfs
+
         :param query: Free format query to run
         :param target_dir: HDFS destination dir
         :param append: Append data to an existing dataset in HDFS
@@ -302,7 +304,7 @@ def _export_cmd(self, table, export_dir, input_null_string,
             for key, value in extra_export_options.items():
                 cmd += ['--{}'.format(key)]
                 if value:
-                    cmd += [value]
+                    cmd += [str(value)]
 
         # The required option
         cmd += ["--table", table]
@@ -319,6 +321,7 @@ def export_table(self, table, export_dir, input_null_string,
         """
         Exports Hive table to remote location. Arguments are copies of direct
         sqoop command line Arguments
+
         :param table: Table remote destination
         :param export_dir: Hive table to export
         :param input_null_string: The string to be interpreted as null for
diff --git a/airflow/contrib/hooks/ssh_hook.py b/airflow/contrib/hooks/ssh_hook.py
index f51f0fbd11948..956a7fd4419d5 100755
--- a/airflow/contrib/hooks/ssh_hook.py
+++ b/airflow/contrib/hooks/ssh_hook.py
@@ -1,8 +1,5 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2012-2015 Spotify AB
-# Ported to Airflow by Bolke de Bruin
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -22,17 +19,17 @@
 
 import getpass
 import os
+import warnings
 
 import paramiko
 from paramiko.config import SSH_PORT
+from sshtunnel import SSHTunnelForwarder
 
-from contextlib import contextmanager
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.log.logging_mixin import LoggingMixin
 
 
-class SSHHook(BaseHook, LoggingMixin):
+class SSHHook(BaseHook):
     """
     Hook for ssh remote execution using Paramiko.
     ref: https://github.com/paramiko/paramiko
@@ -65,7 +62,7 @@ def __init__(self,
                  username=None,
                  password=None,
                  key_file=None,
-                 port=SSH_PORT,
+                 port=None,
                  timeout=10,
                  keepalive_interval=30
                  ):
@@ -75,162 +72,177 @@ def __init__(self,
         self.username = username
         self.password = password
         self.key_file = key_file
+        self.port = port
         self.timeout = timeout
         self.keepalive_interval = keepalive_interval
+
         # Default values, overridable from Connection
         self.compress = True
         self.no_host_key_check = True
+        self.allow_host_key_change = False
+        self.host_proxy = None
+
+        # Placeholder for deprecated __enter__
         self.client = None
-        self.port = port
+
+        # Use connection to override defaults
+        if self.ssh_conn_id is not None:
+            conn = self.get_connection(self.ssh_conn_id)
+            if self.username is None:
+                self.username = conn.login
+            if self.password is None:
+                self.password = conn.password
+            if self.remote_host is None:
+                self.remote_host = conn.host
+            if self.port is None:
+                self.port = conn.port
+            if conn.extra is not None:
+                extra_options = conn.extra_dejson
+                self.key_file = extra_options.get("key_file")
+
+                if "timeout" in extra_options:
+                    self.timeout = int(extra_options["timeout"], 10)
+
+                if "compress" in extra_options\
+                        and str(extra_options["compress"]).lower() == 'false':
+                    self.compress = False
+                if "no_host_key_check" in extra_options\
+                        and\
+                        str(extra_options["no_host_key_check"]).lower() == 'false':
+                    self.no_host_key_check = False
+                if "allow_host_key_change" in extra_options\
+                        and\
+                        str(extra_options["allow_host_key_change"]).lower() == 'true':
+                    self.allow_host_key_change = True
+
+        if not self.remote_host:
+            raise AirflowException("Missing required param: remote_host")
+
+        # Auto detecting username values from system
+        if not self.username:
+            self.log.debug(
+                "username to ssh to host: %s is not specified for connection id"
+                " %s. Using system's default provided by getpass.getuser()",
+                self.remote_host, self.ssh_conn_id
+            )
+            self.username = getpass.getuser()
+
+        user_ssh_config_filename = os.path.expanduser('~/.ssh/config')
+        if os.path.isfile(user_ssh_config_filename):
+            ssh_conf = paramiko.SSHConfig()
+            ssh_conf.parse(open(user_ssh_config_filename))
+            host_info = ssh_conf.lookup(self.remote_host)
+            if host_info and host_info.get('proxycommand'):
+                self.host_proxy = paramiko.ProxyCommand(host_info.get('proxycommand'))
+
+            if not (self.password or self.key_file):
+                if host_info and host_info.get('identityfile'):
+                    self.key_file = host_info.get('identityfile')[0]
+
+        self.port = self.port or SSH_PORT
 
     def get_conn(self):
-        if not self.client:
-            self.log.debug('Creating SSH client for conn_id: %s', self.ssh_conn_id)
-            if self.ssh_conn_id is not None:
-                conn = self.get_connection(self.ssh_conn_id)
-                if self.username is None:
-                    self.username = conn.login
-                if self.password is None:
-                    self.password = conn.password
-                if self.remote_host is None:
-                    self.remote_host = conn.host
-                if conn.port is not None:
-                    self.port = conn.port
-                if conn.extra is not None:
-                    extra_options = conn.extra_dejson
-                    self.key_file = extra_options.get("key_file")
-
-                    if "timeout" in extra_options:
-                        self.timeout = int(extra_options["timeout"], 10)
-
-                    if "compress" in extra_options \
-                            and str(extra_options["compress"]).lower() == 'false':
-                        self.compress = False
-                    if "no_host_key_check" in extra_options \
-                            and \
-                            str(extra_options["no_host_key_check"]).lower() == 'false':
-                        self.no_host_key_check = False
-
-            if not self.remote_host:
-                raise AirflowException("Missing required param: remote_host")
-
-            # Auto detecting username values from system
-            if not self.username:
-                self.log.debug(
-                    "username to ssh to host: %s is not specified for connection id"
-                    " %s. Using system's default provided by getpass.getuser()",
-                    self.remote_host, self.ssh_conn_id
-                )
-                self.username = getpass.getuser()
-
-            host_proxy = None
-            user_ssh_config_filename = os.path.expanduser('~/.ssh/config')
-            if os.path.isfile(user_ssh_config_filename):
-                ssh_conf = paramiko.SSHConfig()
-                ssh_conf.parse(open(user_ssh_config_filename))
-                host_info = ssh_conf.lookup(self.remote_host)
-                if host_info and host_info.get('proxycommand'):
-                    host_proxy = paramiko.ProxyCommand(host_info.get('proxycommand'))
-
-                if not (self.password or self.key_file):
-                    if host_info and host_info.get('identityfile'):
-                        self.key_file = host_info.get('identityfile')[0]
-
-            try:
-                client = paramiko.SSHClient()
-                client.load_system_host_keys()
-                if self.no_host_key_check:
-                    # Default is RejectPolicy
-                    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-                if self.password and self.password.strip():
-                    client.connect(hostname=self.remote_host,
-                                   username=self.username,
-                                   password=self.password,
-                                   timeout=self.timeout,
-                                   compress=self.compress,
-                                   port=self.port,
-                                   sock=host_proxy)
-                else:
-                    client.connect(hostname=self.remote_host,
-                                   username=self.username,
-                                   key_filename=self.key_file,
-                                   timeout=self.timeout,
-                                   compress=self.compress,
-                                   port=self.port,
-                                   sock=host_proxy)
-
-                if self.keepalive_interval:
-                    client.get_transport().set_keepalive(self.keepalive_interval)
-
-                self.client = client
-            except paramiko.AuthenticationException as auth_error:
-                self.log.error(
-                    "Auth failed while connecting to host: %s, error: %s",
-                    self.remote_host, auth_error
-                )
-            except paramiko.SSHException as ssh_error:
-                self.log.error(
-                    "Failed connecting to host: %s, error: %s",
-                    self.remote_host, ssh_error
-                )
-            except Exception as error:
-                self.log.error(
-                    "Error connecting to host: %s, error: %s",
-                    self.remote_host, error
-                )
-        return self.client
-
-    @contextmanager
-    def create_tunnel(self, local_port, remote_port=None, remote_host="localhost"):
         """
-        Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.
-        Remember to close() the returned "tunnel" object in order to clean up
-        after yourself when you are done with the tunnel.
+        Opens a ssh connection to the remote host.
 
-        :param local_port:
-        :type local_port: int
-        :param remote_port:
-        :type remote_port: int
-        :param remote_host:
-        :type remote_host: str
-        :return:
+        :rtype: paramiko.client.SSHClient
         """
 
-        import subprocess
-        # this will ensure the connection to the ssh.remote_host from where the tunnel
-        # is getting created
-        self.get_conn()
-
-        tunnel_host = "{0}:{1}:{2}".format(local_port, remote_host, remote_port)
-
-        ssh_cmd = ["ssh", "{0}@{1}".format(self.username, self.remote_host),
-                   "-o", "ControlMaster=no",
-                   "-o", "UserKnownHostsFile=/dev/null",
-                   "-o", "StrictHostKeyChecking=no"]
-
-        ssh_tunnel_cmd = ["-L", tunnel_host,
-                          "echo -n ready && cat"
-                          ]
-
-        ssh_cmd += ssh_tunnel_cmd
-        self.log.debug("Creating tunnel with cmd: %s", ssh_cmd)
-
-        proc = subprocess.Popen(ssh_cmd,
-                                stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE,
-                                close_fds=True)
-        ready = proc.stdout.read(5)
-        assert ready == b"ready", \
-            "Did not get 'ready' from remote, got '{0}' instead".format(ready)
-        yield
-        proc.communicate()
-        assert proc.returncode == 0, \
-            "Tunnel process did unclean exit (returncode {}".format(proc.returncode)
+        self.log.debug('Creating SSH client for conn_id: %s', self.ssh_conn_id)
+        client = paramiko.SSHClient()
+        if not self.allow_host_key_change:
+            self.log.warning('Remote Identification Change is not verified. '
+                             'This wont protect against Man-In-The-Middle attacks')
+            client.load_system_host_keys()
+        if self.no_host_key_check:
+            self.log.warning('No Host Key Verification. This wont protect '
+                             'against Man-In-The-Middle attacks')
+            # Default is RejectPolicy
+            client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        if self.password and self.password.strip():
+            client.connect(hostname=self.remote_host,
+                           username=self.username,
+                           password=self.password,
+                           key_filename=self.key_file,
+                           timeout=self.timeout,
+                           compress=self.compress,
+                           port=self.port,
+                           sock=self.host_proxy)
+        else:
+            client.connect(hostname=self.remote_host,
+                           username=self.username,
+                           key_filename=self.key_file,
+                           timeout=self.timeout,
+                           compress=self.compress,
+                           port=self.port,
+                           sock=self.host_proxy)
+
+        if self.keepalive_interval:
+            client.get_transport().set_keepalive(self.keepalive_interval)
+
+        self.client = client
+        return client
 
     def __enter__(self):
+        warnings.warn('The contextmanager of SSHHook is deprecated.'
+                      'Please use get_conn() as a contextmanager instead.'
+                      'This method will be removed in Airflow 2.0',
+                      category=DeprecationWarning)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.client is not None:
             self.client.close()
+            self.client = None
+
+    def get_tunnel(self, remote_port, remote_host="localhost", local_port=None):
+        """
+        Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.
+
+        :param remote_port: The remote port to create a tunnel to
+        :type remote_port: int
+        :param remote_host: The remote host to create a tunnel to (default localhost)
+        :type remote_host: str
+        :param local_port:  The local port to attach the tunnel to
+        :type local_port: int
+
+        :return: sshtunnel.SSHTunnelForwarder object
+        """
+
+        if local_port:
+            local_bind_address = ('localhost', local_port)
+        else:
+            local_bind_address = ('localhost',)
+
+        if self.password and self.password.strip():
+            client = SSHTunnelForwarder(self.remote_host,
+                                        ssh_port=self.port,
+                                        ssh_username=self.username,
+                                        ssh_password=self.password,
+                                        ssh_pkey=self.key_file,
+                                        ssh_proxy=self.host_proxy,
+                                        local_bind_address=local_bind_address,
+                                        remote_bind_address=(remote_host, remote_port),
+                                        logger=self.log)
+        else:
+            client = SSHTunnelForwarder(self.remote_host,
+                                        ssh_port=self.port,
+                                        ssh_username=self.username,
+                                        ssh_pkey=self.key_file,
+                                        ssh_proxy=self.host_proxy,
+                                        local_bind_address=local_bind_address,
+                                        remote_bind_address=(remote_host, remote_port),
+                                        host_pkey_directories=[],
+                                        logger=self.log)
+
+        return client
+
+    def create_tunnel(self, local_port, remote_port=None, remote_host="localhost"):
+        warnings.warn('SSHHook.create_tunnel is deprecated, Please'
+                      'use get_tunnel() instead. But please note that the'
+                      'order of the parameters have changed'
+                      'This method will be removed in Airflow 2.0',
+                      category=DeprecationWarning)
+
+        return self.get_tunnel(remote_port, remote_host, local_port)
diff --git a/airflow/contrib/hooks/vertica_hook.py b/airflow/contrib/hooks/vertica_hook.py
index f3411de994d7f..e6b36b51d5937 100644
--- a/airflow/contrib/hooks/vertica_hook.py
+++ b/airflow/contrib/hooks/vertica_hook.py
@@ -41,9 +41,9 @@ def get_conn(self):
             "user": conn.login,
             "password": conn.password or '',
             "database": conn.schema,
+            "host": conn.host or 'localhost'
         }
 
-        conn_config["host"] = conn.host or 'localhost'
         if not conn.port:
             conn_config["port"] = 5433
         else:
diff --git a/airflow/contrib/hooks/wasb_hook.py b/airflow/contrib/hooks/wasb_hook.py
index 1d73abd78b3dd..d3a766cf69de7 100644
--- a/airflow/contrib/hooks/wasb_hook.py
+++ b/airflow/contrib/hooks/wasb_hook.py
@@ -18,6 +18,7 @@
 # under the License.
 #
 
+from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
 
 from azure.storage.blob import BlockBlobService
@@ -58,7 +59,7 @@ def check_for_blob(self, container_name, blob_name, **kwargs):
             `BlockBlobService.exists()` takes.
         :type kwargs: object
         :return: True if the blob exists, False otherwise.
-        :rtype bool
+        :rtype: bool
         """
         return self.connection.exists(container_name, blob_name, **kwargs)
 
@@ -74,7 +75,7 @@ def check_for_prefix(self, container_name, prefix, **kwargs):
             `BlockBlobService.list_blobs()` takes.
         :type kwargs: object
         :return: True if blobs matching the prefix exist, False otherwise.
-        :rtype bool
+        :rtype: bool
         """
         matches = self.connection.list_blobs(container_name, prefix,
                                              num_results=1, **kwargs)
@@ -148,3 +149,43 @@ def read_file(self, container_name, blob_name, **kwargs):
         return self.connection.get_blob_to_text(container_name,
                                                 blob_name,
                                                 **kwargs).content
+
+    def delete_file(self, container_name, blob_name, is_prefix=False,
+                    ignore_if_missing=False, **kwargs):
+        """
+        Delete a file from Azure Blob Storage.
+
+        :param container_name: Name of the container.
+        :type container_name: str
+        :param blob_name: Name of the blob.
+        :type blob_name: str
+        :param is_prefix: If blob_name is a prefix, delete all matching files
+        :type is_prefix: bool
+        :param ignore_if_missing: if True, then return success even if the
+            blob does not exist.
+        :type ignore_if_missing: bool
+        :param kwargs: Optional keyword arguments that
+            `BlockBlobService.create_blob_from_path()` takes.
+        :type kwargs: object
+        """
+
+        if is_prefix:
+            blobs_to_delete = [
+                blob.name for blob in self.connection.list_blobs(
+                    container_name, prefix=blob_name, **kwargs
+                )
+            ]
+        elif self.check_for_blob(container_name, blob_name):
+            blobs_to_delete = [blob_name]
+        else:
+            blobs_to_delete = []
+
+        if not ignore_if_missing and len(blobs_to_delete) == 0:
+            raise AirflowException('Blob(s) not found: {}'.format(blob_name))
+
+        for blob_uri in blobs_to_delete:
+            self.log.info("Deleting blob: " + blob_uri)
+            self.connection.delete_blob(container_name,
+                                        blob_uri,
+                                        delete_snapshots='include',
+                                        **kwargs)
diff --git a/airflow/contrib/hooks/winrm_hook.py b/airflow/contrib/hooks/winrm_hook.py
index 0be904b9bcdca..336dc2d818c62 100644
--- a/airflow/contrib/hooks/winrm_hook.py
+++ b/airflow/contrib/hooks/winrm_hook.py
@@ -17,121 +17,234 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+
 import getpass
+
 from winrm.protocol import Protocol
+
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.log.logging_mixin import LoggingMixin
 
 
-class WinRMHook(BaseHook, LoggingMixin):
-
+class WinRMHook(BaseHook):
     """
     Hook for winrm remote execution using pywinrm.
 
+    :seealso: https://github.com/diyan/pywinrm/blob/master/winrm/protocol.py
+
     :param ssh_conn_id: connection id from airflow Connections from where all
-        the required parameters can be fetched like username, password or key_file.
+        the required parameters can be fetched like username and password.
         Thought the priority is given to the param passed during init
     :type ssh_conn_id: str
-    :param remote_host: remote host to connect
+    :param endpoint: When set to `None`, endpoint will be constructed like this:
+        'http://{remote_host}:{remote_port}/wsman'
+    :type endpoint: str
+    :param remote_host: Remote host to connect to.
+        Ignored if `endpoint` is not `None`.
     :type remote_host: str
+    :param remote_port: Remote port to connect to.
+        Ignored if `endpoint` is not `None`.
+    :type remote_port: int
+    :param transport: transport type, one of 'plaintext' (default), 'kerberos', 'ssl',
+        'ntlm', 'credssp'
+    :type transport: str
     :param username: username to connect to the remote_host
     :type username: str
     :param password: password of the username to connect to the remote_host
     :type password: str
-    :param key_file: key file to use to connect to the remote_host.
-    :type key_file: str
-    :param timeout: timeout for the attempt to connect to the remote_host.
-    :type timeout: int
-    :param keepalive_interval: send a keepalive packet to remote host
-        every keepalive_interval seconds
-    :type keepalive_interval: int
+    :param service: the service name, default is HTTP
+    :type service: str
+    :param keytab: the path to a keytab file if you are using one
+    :type keytab: str
+    :param ca_trust_path: Certification Authority trust path
+    :type ca_trust_path: str
+    :param cert_pem: client authentication certificate file path in PEM format
+    :type cert_pem: str
+    :param cert_key_pem: client authentication certificate key file path in PEM format
+    :type cert_key_pem: str
+    :param server_cert_validation: whether server certificate should be validated on
+        Python versions that suppport it; one of 'validate' (default), 'ignore'
+    :type server_cert_validation: str
+    :param kerberos_delegation: if True, TGT is sent to target server to
+        allow multiple hops
+    :type kerberos_delegation: bool
+    :param read_timeout_sec: maximum seconds to wait before an HTTP connect/read times out
+        (default 30). This value should be slightly higher than operation_timeout_sec,
+        as the server can block *at least* that long.
+    :type read_timeout_sec: int
+    :param operation_timeout_sec: maximum allowed time in seconds for any single wsman
+        HTTP operation (default 20). Note that operation timeouts while receiving output
+        (the only wsman operation that should take any significant time, and where these
+        timeouts are expected) will be silently retried indefinitely.
+    :type operation_timeout_sec: int
+    :param kerberos_hostname_override: the hostname to use for the kerberos exchange
+        (defaults to the hostname in the endpoint URL)
+    :type kerberos_hostname_override: str
+    :param message_encryption_enabled: Will encrypt the WinRM messages if set to True and
+        the transport auth supports message encryption (Default True).
+    :type message_encryption_enabled: bool
+    :param credssp_disable_tlsv1_2: Whether to disable TLSv1.2 support and work with older
+        protocols like TLSv1.0, default is False
+    :type credssp_disable_tlsv1_2: bool
+    :param send_cbt: Will send the channel bindings over a HTTPS channel (Default: True)
+    :type send_cbt: bool
     """
-
     def __init__(self,
                  ssh_conn_id=None,
+                 endpoint=None,
                  remote_host=None,
+                 remote_port=5985,
+                 transport='plaintext',
                  username=None,
                  password=None,
-                 key_file=None,
-                 timeout=10,
-                 keepalive_interval=30
+                 service='HTTP',
+                 keytab=None,
+                 ca_trust_path=None,
+                 cert_pem=None,
+                 cert_key_pem=None,
+                 server_cert_validation='validate',
+                 kerberos_delegation=False,
+                 read_timeout_sec=30,
+                 operation_timeout_sec=20,
+                 kerberos_hostname_override=None,
+                 message_encryption='auto',
+                 credssp_disable_tlsv1_2=False,
+                 send_cbt=True,
                  ):
         super(WinRMHook, self).__init__(ssh_conn_id)
-        # TODO make new win rm connection class
         self.ssh_conn_id = ssh_conn_id
+        self.endpoint = endpoint
         self.remote_host = remote_host
+        self.remote_port = remote_port
+        self.transport = transport
         self.username = username
         self.password = password
-        self.key_file = key_file
-        self.timeout = timeout
-        self.keepalive_interval = keepalive_interval
-        # Default values, overridable from Connection
-        self.compress = True
-        self.no_host_key_check = True
+        self.service = service
+        self.keytab = keytab
+        self.ca_trust_path = ca_trust_path
+        self.cert_pem = cert_pem
+        self.cert_key_pem = cert_key_pem
+        self.server_cert_validation = server_cert_validation
+        self.kerberos_delegation = kerberos_delegation
+        self.read_timeout_sec = read_timeout_sec
+        self.operation_timeout_sec = operation_timeout_sec
+        self.kerberos_hostname_override = kerberos_hostname_override
+        self.message_encryption = message_encryption
+        self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2
+        self.send_cbt = send_cbt
+
         self.client = None
         self.winrm_protocol = None
 
     def get_conn(self):
-        if not self.client:
-            self.log.debug('Creating WinRM client for conn_id: %s', self.ssh_conn_id)
-            if self.ssh_conn_id is not None:
-                conn = self.get_connection(self.ssh_conn_id)
-                if self.username is None:
-                    self.username = conn.login
-                if self.password is None:
-                    self.password = conn.password
-                if self.remote_host is None:
-                    self.remote_host = conn.host
-                if conn.extra is not None:
-                    extra_options = conn.extra_dejson
-                    self.key_file = extra_options.get("key_file")
-
-                    if "timeout" in extra_options:
-                        self.timeout = int(extra_options["timeout"], 10)
-
-                    if "compress" in extra_options \
-                            and extra_options["compress"].lower() == 'false':
-                        self.compress = False
-                    if "no_host_key_check" in extra_options \
-                            and extra_options["no_host_key_check"].lower() == 'false':
-                        self.no_host_key_check = False
-
-            if not self.remote_host:
-                raise AirflowException("Missing required param: remote_host")
-
-            # Auto detecting username values from system
-            if not self.username:
-                self.log.debug(
-                    "username to ssh to host: %s is not specified for connection id"
-                    " %s. Using system's default provided by getpass.getuser()",
-                    self.remote_host, self.ssh_conn_id
-                )
-                self.username = getpass.getuser()
-
-            try:
-
-                if self.password and self.password.strip():
-                    self.winrm_protocol = Protocol(
-                        # TODO pass in port from ssh conn
-                        endpoint='http://' + self.remote_host + ':5985/wsman',
-                        # TODO get cert transport working
-                        # transport='certificate',
-                        transport='plaintext',
-                        # cert_pem=r'publickey.pem',
-                        # cert_key_pem=r'dev.pem',
-                        read_timeout_sec=70,
-                        operation_timeout_sec=60,
-                        username=self.username,
-                        password=self.password,
-                        server_cert_validation='ignore')
-
-                self.log.info("Opening WinRM shell")
-                self.client = self.winrm_protocol.open_shell()
-
-            except Exception as error:
-                self.log.error(
-                    "Error connecting to host: %s, error: %s",
-                    self.remote_host, error
+        if self.client:
+            return self.client
+
+        self.log.debug('Creating WinRM client for conn_id: %s', self.ssh_conn_id)
+        if self.ssh_conn_id is not None:
+            conn = self.get_connection(self.ssh_conn_id)
+
+            if self.username is None:
+                self.username = conn.login
+            if self.password is None:
+                self.password = conn.password
+            if self.remote_host is None:
+                self.remote_host = conn.host
+
+            if conn.extra is not None:
+                extra_options = conn.extra_dejson
+
+                if "endpoint" in extra_options:
+                    self.endpoint = str(extra_options["endpoint"])
+                if "remote_port" in extra_options:
+                    self.remote_port = int(extra_options["remote_port"])
+                if "transport" in extra_options:
+                    self.transport = str(extra_options["transport"])
+                if "service" in extra_options:
+                    self.service = str(extra_options["service"])
+                if "keytab" in extra_options:
+                    self.keytab = str(extra_options["keytab"])
+                if "ca_trust_path" in extra_options:
+                    self.ca_trust_path = str(extra_options["ca_trust_path"])
+                if "cert_pem" in extra_options:
+                    self.cert_pem = str(extra_options["cert_pem"])
+                if "cert_key_pem" in extra_options:
+                    self.cert_key_pem = str(extra_options["cert_key_pem"])
+                if "server_cert_validation" in extra_options:
+                    self.server_cert_validation = \
+                        str(extra_options["server_cert_validation"])
+                if "kerberos_delegation" in extra_options:
+                    self.kerberos_delegation = \
+                        str(extra_options["kerberos_delegation"]).lower() == 'true'
+                if "read_timeout_sec" in extra_options:
+                    self.read_timeout_sec = int(extra_options["read_timeout_sec"])
+                if "operation_timeout_sec" in extra_options:
+                    self.operation_timeout_sec = \
+                        int(extra_options["operation_timeout_sec"])
+                if "kerberos_hostname_override" in extra_options:
+                    self.kerberos_hostname_override = \
+                        str(extra_options["kerberos_hostname_override"])
+                if "message_encryption" in extra_options:
+                    self.message_encryption = str(extra_options["message_encryption"])
+                if "credssp_disable_tlsv1_2" in extra_options:
+                    self.credssp_disable_tlsv1_2 = \
+                        str(extra_options["credssp_disable_tlsv1_2"]).lower() == 'true'
+                if "send_cbt" in extra_options:
+                    self.send_cbt = str(extra_options["send_cbt"]).lower() == 'true'
+
+        if not self.remote_host:
+            raise AirflowException("Missing required param: remote_host")
+
+        # Auto detecting username values from system
+        if not self.username:
+            self.log.debug(
+                "username to WinRM to host: %s is not specified for connection id"
+                " %s. Using system's default provided by getpass.getuser()",
+                self.remote_host, self.ssh_conn_id
+            )
+            self.username = getpass.getuser()
+
+        # If endpoint is not set, then build a standard wsman endpoint from host and port.
+        if not self.endpoint:
+            self.endpoint = 'http://{0}:{1}/wsman'.format(
+                self.remote_host,
+                self.remote_port
+            )
+
+        try:
+            if self.password and self.password.strip():
+                self.winrm_protocol = Protocol(
+                    endpoint=self.endpoint,
+                    transport=self.transport,
+                    username=self.username,
+                    password=self.password,
+                    service=self.service,
+                    keytab=self.keytab,
+                    ca_trust_path=self.ca_trust_path,
+                    cert_pem=self.cert_pem,
+                    cert_key_pem=self.cert_key_pem,
+                    server_cert_validation=self.server_cert_validation,
+                    kerberos_delegation=self.kerberos_delegation,
+                    read_timeout_sec=self.read_timeout_sec,
+                    operation_timeout_sec=self.operation_timeout_sec,
+                    kerberos_hostname_override=self.kerberos_hostname_override,
+                    message_encryption=self.message_encryption,
+                    credssp_disable_tlsv1_2=self.credssp_disable_tlsv1_2,
+                    send_cbt=self.send_cbt
                 )
+
+            self.log.info(
+                "Establishing WinRM connection to host: %s",
+                self.remote_host
+            )
+            self.client = self.winrm_protocol.open_shell()
+
+        except Exception as error:
+            error_msg = "Error connecting to host: {0}, error: {1}".format(
+                self.remote_host,
+                error
+            )
+            self.log.error(error_msg)
+            raise AirflowException(error_msg)
+
         return self.client
diff --git a/airflow/contrib/kubernetes/kube_client.py b/airflow/contrib/kubernetes/kube_client.py
index 8b71f41242329..cc699e7104475 100644
--- a/airflow/contrib/kubernetes/kube_client.py
+++ b/airflow/contrib/kubernetes/kube_client.py
@@ -17,9 +17,21 @@
 from airflow.configuration import conf
 from six import PY2
 
+try:
+    from kubernetes import config, client
+    from kubernetes.client.rest import ApiException
+    has_kubernetes = True
+except ImportError as e:
+    # We need an exception class to be able to use it in ``except`` elsewhere
+    # in the code base
+    ApiException = BaseException
+    has_kubernetes = False
+    _import_err = e
+
 
 def _load_kube_config(in_cluster, cluster_context, config_file):
-    from kubernetes import config, client
+    if not has_kubernetes:
+        raise _import_err
     if in_cluster:
         config.load_incluster_config()
     else:
@@ -36,4 +48,9 @@ def _load_kube_config(in_cluster, cluster_context, config_file):
 def get_kube_client(in_cluster=conf.getboolean('kubernetes', 'in_cluster'),
                     cluster_context=None,
                     config_file=None):
+    if not in_cluster:
+        if cluster_context is None:
+            cluster_context = conf.get('kubernetes', 'cluster_context', fallback=None)
+        if config_file is None:
+            config_file = conf.get('kubernetes', 'config_file', fallback=None)
     return _load_kube_config(in_cluster, cluster_context, config_file)
diff --git a/airflow/contrib/kubernetes/kubernetes_request_factory/kubernetes_request_factory.py b/airflow/contrib/kubernetes/kubernetes_request_factory/kubernetes_request_factory.py
index 7133125ab14e3..d1b6d4eaf4dca 100644
--- a/airflow/contrib/kubernetes/kubernetes_request_factory/kubernetes_request_factory.py
+++ b/airflow/contrib/kubernetes/kubernetes_request_factory/kubernetes_request_factory.py
@@ -75,6 +75,12 @@ def extract_affinity(pod, req):
         for k, v in six.iteritems(pod.affinity):
             req['spec']['affinity'][k] = v
 
+    @staticmethod
+    def extract_node_selector(pod, req):
+        req['spec']['nodeSelector'] = req['spec'].get('nodeSelector', {})
+        for k, v in six.iteritems(pod.node_selectors):
+            req['spec']['nodeSelector'][k] = v
+
     @staticmethod
     def extract_cmds(pod, req):
         req['spec']['containers'][0]['command'] = pod.cmds
@@ -83,11 +89,6 @@ def extract_cmds(pod, req):
     def extract_args(pod, req):
         req['spec']['containers'][0]['args'] = pod.args
 
-    @staticmethod
-    def extract_node_selector(pod, req):
-        if len(pod.node_selectors) > 0:
-            req['spec']['nodeSelector'] = pod.node_selectors
-
     @staticmethod
     def attach_volumes(pod, req):
         req['spec']['volumes'] = (
@@ -130,15 +131,21 @@ def extract_volume_secrets(pod, req):
 
     @staticmethod
     def extract_env_and_secrets(pod, req):
-        env_secrets = [s for s in pod.secrets if s.deploy_type == 'env']
-        if len(pod.envs) > 0 or len(env_secrets) > 0:
+        envs_from_key_secrets = [
+            env for env in pod.secrets if env.deploy_type == 'env' and env.key is not None
+        ]
+
+        if len(pod.envs) > 0 or len(envs_from_key_secrets) > 0:
             env = []
             for k in pod.envs.keys():
                 env.append({'name': k, 'value': pod.envs[k]})
-            for secret in env_secrets:
+            for secret in envs_from_key_secrets:
                 KubernetesRequestFactory.add_secret_to_env(env, secret)
+
             req['spec']['containers'][0]['env'] = env
 
+        KubernetesRequestFactory._apply_env_from(pod, req)
+
     @staticmethod
     def extract_resources(pod, req):
         if not pod.resources or pod.resources.is_empty_resource_request():
@@ -174,9 +181,51 @@ def extract_service_account_name(pod, req):
         if pod.service_account_name:
             req['spec']['serviceAccountName'] = pod.service_account_name
 
+    @staticmethod
+    def extract_hostnetwork(pod, req):
+        if pod.hostnetwork:
+            req['spec']['hostNetwork'] = pod.hostnetwork
+
     @staticmethod
     def extract_image_pull_secrets(pod, req):
         if pod.image_pull_secrets:
             req['spec']['imagePullSecrets'] = [{
                 'name': pull_secret
             } for pull_secret in pod.image_pull_secrets.split(',')]
+
+    @staticmethod
+    def extract_tolerations(pod, req):
+        if pod.tolerations:
+            req['spec']['tolerations'] = pod.tolerations
+
+    @staticmethod
+    def extract_security_context(pod, req):
+        if pod.security_context:
+            req['spec']['securityContext'] = pod.security_context
+
+    @staticmethod
+    def _apply_env_from(pod, req):
+        envs_from_secrets = [
+            env for env in pod.secrets if env.deploy_type == 'env' and env.key is None
+        ]
+
+        if pod.configmaps or envs_from_secrets:
+            req['spec']['containers'][0]['envFrom'] = []
+
+        for secret in envs_from_secrets:
+            req['spec']['containers'][0]['envFrom'].append(
+                {
+                    'secretRef': {
+                        'name': secret.secret
+                    }
+                }
+            )
+
+        for configmap in pod.configmaps:
+            req['spec']['containers'][0]['envFrom'].append(
+                {
+                    'configMapRef': {
+                        'name': configmap
+                    }
+                }
+            )
diff --git a/airflow/contrib/kubernetes/kubernetes_request_factory/pod_request_factory.py b/airflow/contrib/kubernetes/kubernetes_request_factory/pod_request_factory.py
index 95d6c829dec59..3aea85df3ea18 100644
--- a/airflow/contrib/kubernetes/kubernetes_request_factory/pod_request_factory.py
+++ b/airflow/contrib/kubernetes/kubernetes_request_factory/pod_request_factory.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import yaml
+from airflow.contrib.kubernetes.pod import Pod
 from airflow.contrib.kubernetes.kubernetes_request_factory.kubernetes_request_factory \
     import KubernetesRequestFactory
 
@@ -59,16 +60,18 @@ def create(self, pod):
         self.extract_image_pull_secrets(pod, req)
         self.extract_annotations(pod, req)
         self.extract_affinity(pod, req)
+        self.extract_hostnetwork(pod, req)
+        self.extract_tolerations(pod, req)
+        self.extract_security_context(pod, req)
         return req
 
 
 class ExtractXcomPodRequestFactory(KubernetesRequestFactory):
-
-    XCOM_MOUNT_PATH = '/airflow/xcom'
-    SIDECAR_CONTAINER_NAME = 'airflow-xcom-sidecar'
     """
     Request generator for a pod with sidecar container.
     """
+    XCOM_MOUNT_PATH = '/airflow/xcom'
+    SIDECAR_CONTAINER_NAME = 'airflow-xcom-sidecar'
     _yaml = """apiVersion: v1
 kind: Pod
 metadata:
@@ -86,7 +89,16 @@ class ExtractXcomPodRequestFactory(KubernetesRequestFactory):
           mountPath: {xcomMountPath}
     - name: {sidecarContainerName}
       image: python:3.5-alpine
-      command: ["python", "-m", "http.server"]
+      command:
+        - python
+        - -c
+        - |
+            import time
+            while True:
+                try:
+                    time.sleep(3600)
+                except KeyboardInterrupt:
+                    exit(0)
       volumeMounts:
         - name: xcom
           mountPath: {xcomMountPath}
@@ -116,4 +128,7 @@ def create(self, pod):
         self.extract_image_pull_secrets(pod, req)
         self.extract_annotations(pod, req)
         self.extract_affinity(pod, req)
+        self.extract_hostnetwork(pod, req)
+        self.extract_tolerations(pod, req)
+        self.extract_security_context(pod, req)
         return req
diff --git a/airflow/contrib/kubernetes/pod.py b/airflow/contrib/kubernetes/pod.py
index c42221482ff88..f8c0bdffaee08 100644
--- a/airflow/contrib/kubernetes/pod.py
+++ b/airflow/contrib/kubernetes/pod.py
@@ -46,16 +46,29 @@ class Pod:
     :param envs: A dict containing the environment variables
     :type envs: dict
     :param cmds: The command to be run on the pod
-    :type cmds: list str
+    :type cmds: list[str]
     :param secrets: Secrets to be launched to the pod
-    :type secrets: list Secret
+    :type secrets: list[airflow.contrib.kubernetes.secret.Secret]
     :param result: The result that will be returned to the operator after
-                   successful execution of the pod
+        successful execution of the pod
     :type result: any
     :param image_pull_policy: Specify a policy to cache or always pull an image
     :type image_pull_policy: str
+    :param image_pull_secrets: Any image pull secrets to be given to the pod.
+        If more than one secret is required, provide a comma separated list:
+        secret_a,secret_b
+    :type image_pull_secrets: str
     :param affinity: A dict containing a group of affinity scheduling rules
     :type affinity: dict
+    :param hostnetwork: If True enable host networking on the pod
+    :type hostnetwork: bool
+    :param tolerations: A list of kubernetes tolerations
+    :type tolerations: list
+    :param security_context: A dict containing the security context for the pod
+    :type security_context: dict
+    :param configmaps: A list containing names of configmaps object
+        mounting env variables to the pod
+    :type configmaps: list[str]
     """
     def __init__(
             self,
@@ -77,7 +90,11 @@ def __init__(
             service_account_name=None,
             resources=None,
             annotations=None,
-            affinity=None
+            affinity=None,
+            hostnetwork=False,
+            tolerations=None,
+            security_context=None,
+            configmaps=None
     ):
         self.image = image
         self.envs = envs or {}
@@ -89,7 +106,7 @@ def __init__(
         self.name = name
         self.volumes = volumes or []
         self.volume_mounts = volume_mounts or []
-        self.node_selectors = node_selectors or []
+        self.node_selectors = node_selectors or {}
         self.namespace = namespace
         self.image_pull_policy = image_pull_policy
         self.image_pull_secrets = image_pull_secrets
@@ -98,3 +115,7 @@ def __init__(
         self.resources = resources or Resources()
         self.annotations = annotations or {}
         self.affinity = affinity or {}
+        self.hostnetwork = hostnetwork or False
+        self.tolerations = tolerations or []
+        self.security_context = security_context
+        self.configmaps = configmaps or []
diff --git a/airflow/contrib/kubernetes/pod_generator.py b/airflow/contrib/kubernetes/pod_generator.py
index 6d8d83ef054a9..bee7f5b9572a0 100644
--- a/airflow/contrib/kubernetes/pod_generator.py
+++ b/airflow/contrib/kubernetes/pod_generator.py
@@ -149,7 +149,7 @@ def make_pod(self, namespace, image, pod_id, cmds, arguments, labels):
 
         return Pod(
             namespace=namespace,
-            name=pod_id + "-" + str(uuid.uuid1())[:8],
+            name=pod_id + "-" + str(uuid.uuid4())[:8],
             image=image,
             cmds=cmds,
             args=arguments,
diff --git a/airflow/contrib/kubernetes/pod_launcher.py b/airflow/contrib/kubernetes/pod_launcher.py
index 8ac5108507345..2d46d277b8b8e 100644
--- a/airflow/contrib/kubernetes/pod_launcher.py
+++ b/airflow/contrib/kubernetes/pod_launcher.py
@@ -17,12 +17,14 @@
 
 import json
 import time
+from typing import Tuple, Optional
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import State
 from datetime import datetime as dt
+from airflow.contrib.kubernetes.pod import Pod
 from airflow.contrib.kubernetes.kubernetes_request_factory import \
     pod_request_factory as pod_factory
-from kubernetes import watch
+from kubernetes import watch, client
 from kubernetes.client.rest import ApiException
 from kubernetes.stream import stream as kubernetes_stream
 from airflow import AirflowException
@@ -59,8 +61,17 @@ def run_pod_async(self, pod):
             raise
         return resp
 
+    def delete_pod(self, pod):
+        try:
+            self._client.delete_namespaced_pod(
+                pod.name, pod.namespace, body=client.V1DeleteOptions())
+        except ApiException as e:
+            # If the pod is already deleted
+            if e.status != 404:
+                raise
+
     def run_pod(self, pod, startup_timeout=120, get_logs=True):
-        # type: (Pod) -> (State, result)
+        # type: (Pod, int, bool) -> Tuple[State, Optional[str]]
         """
         Launches the pod synchronously and waits for completion.
         Args:
@@ -81,7 +92,7 @@ def run_pod(self, pod, startup_timeout=120, get_logs=True):
         return self._monitor_pod(pod, get_logs)
 
     def _monitor_pod(self, pod, get_logs):
-        # type: (Pod) -> (State, content)
+        # type: (Pod, bool) -> Tuple[State, Optional[str]]
 
         if get_logs:
             logs = self._client.read_namespaced_pod_log(
@@ -104,7 +115,7 @@ def _monitor_pod(self, pod, get_logs):
         while self.pod_is_running(pod):
             self.log.info('Pod %s has state %s', pod.name, State.RUNNING)
             time.sleep(2)
-        return (self._task_status(self.read_pod(pod)), result)
+        return self._task_status(self.read_pod(pod)), result
 
     def _task_status(self, event):
         self.log.info(
@@ -154,7 +165,7 @@ def _extract_xcom(self, pod):
 
     def _exec_pod_command(self, resp, command):
         if resp.is_open():
-            self.log.info('Running command... %s\n' % command)
+            self.log.info('Running command... %s\n', command)
             resp.write_stdin(command + '\n')
             while resp.is_open():
                 resp.update(timeout=1)
diff --git a/airflow/contrib/kubernetes/secret.py b/airflow/contrib/kubernetes/secret.py
index 5c1038cd84e9a..73c51e900acf9 100644
--- a/airflow/contrib/kubernetes/secret.py
+++ b/airflow/contrib/kubernetes/secret.py
@@ -14,28 +14,55 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from airflow.exceptions import AirflowConfigException
 
 
-class Secret:
+class Secret(object):
     """Defines Kubernetes Secret Volume"""
 
-    def __init__(self, deploy_type, deploy_target, secret, key):
+    def __init__(self, deploy_type, deploy_target, secret, key=None):
         """Initialize a Kubernetes Secret Object. Used to track requested secrets from
         the user.
         :param deploy_type: The type of secret deploy in Kubernetes, either `env` or
             `volume`
-        :type deploy_type: ``str``
-        :param deploy_target: The environment variable when `deploy_type` `env` or
-            file path when `deploy_type` `volume` where expose secret
-        :type deploy_target: ``str``
+        :type deploy_type: str
+        :param deploy_target: (Optional) The environment variable when
+            `deploy_type` `env` or file path when `deploy_type` `volume` where
+            expose secret. If `key` is not provided deploy target should be None.
+        :type deploy_target: str or None
         :param secret: Name of the secrets object in Kubernetes
-        :type secret: ``str``
-        :param key: Key of the secret within the Kubernetes Secret
-        :type key: ``str``
+        :type secret: str
+        :param key: (Optional) Key of the secret within the Kubernetes Secret
+            if not provided in `deploy_type` `env` it will mount all secrets in object
+        :type key: str or None
         """
         self.deploy_type = deploy_type
-        self.deploy_target = deploy_target.upper()
-        if deploy_type == 'volume':
-            self.deploy_target = deploy_target
+        self.deploy_target = deploy_target
+
+        if deploy_target is not None and deploy_type == 'env':
+            # if deploying to env, capitalize the deploy target
+            self.deploy_target = deploy_target.upper()
+
+        if key is not None and deploy_target is None:
+            raise AirflowConfigException(
+                'If `key` is set, `deploy_target` should not be None'
+            )
+
         self.secret = secret
         self.key = key
+
+    def __eq__(self, other):
+        return (
+            self.deploy_type == other.deploy_type and
+            self.deploy_target == other.deploy_target and
+            self.secret == other.secret and
+            self.key == other.key
+        )
+
+    def __repr__(self):
+        return 'Secret({}, {}, {}, {})'.format(
+            self.deploy_type,
+            self.deploy_target,
+            self.secret,
+            self.key
+        )
diff --git a/airflow/contrib/kubernetes/volume.py b/airflow/contrib/kubernetes/volume.py
index d5b4f60cc3850..94003fe48dcb3 100644
--- a/airflow/contrib/kubernetes/volume.py
+++ b/airflow/contrib/kubernetes/volume.py
@@ -23,11 +23,11 @@ def __init__(self, name, configs):
         """ Adds Kubernetes Volume to pod. allows pod to access features like ConfigMaps
         and Persistent Volumes
         :param name: the name of the volume mount
-        :type: name: str
+        :type name: str
         :param configs: dictionary of any features needed for volume.
         We purposely keep this vague since there are multiple volume types with changing
         configs.
-        :type: configs: dict
+        :type configs: dict
         """
         self.name = name
         self.configs = configs
diff --git a/airflow/contrib/kubernetes/worker_configuration.py b/airflow/contrib/kubernetes/worker_configuration.py
index 26540856f980d..b9896b5b886d9 100644
--- a/airflow/contrib/kubernetes/worker_configuration.py
+++ b/airflow/contrib/kubernetes/worker_configuration.py
@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import copy
 import os
 import six
 
+from airflow.configuration import conf
 from airflow.contrib.kubernetes.pod import Pod, Resources
 from airflow.contrib.kubernetes.secret import Secret
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -27,17 +27,26 @@
 class WorkerConfiguration(LoggingMixin):
     """Contains Kubernetes Airflow Worker configuration logic"""
 
+    dags_volume_name = 'airflow-dags'
+    logs_volume_name = 'airflow-logs'
+    git_sync_ssh_secret_volume_name = 'git-sync-ssh-key'
+    git_ssh_key_secret_key = 'gitSshKey'
+    git_sync_ssh_known_hosts_volume_name = 'git-sync-known-hosts'
+    git_ssh_known_hosts_configmap_key = 'known_hosts'
+
     def __init__(self, kube_config):
         self.kube_config = kube_config
         self.worker_airflow_home = self.kube_config.airflow_home
         self.worker_airflow_dags = self.kube_config.dags_folder
         self.worker_airflow_logs = self.kube_config.base_log_folder
+
         super(WorkerConfiguration, self).__init__()
 
-    def _get_init_containers(self, volume_mounts):
+    def _get_init_containers(self):
         """When using git to retrieve the DAGs, use the GitSync Init Container"""
         # If we're using volume claims to mount the dags, no init container is needed
-        if self.kube_config.dags_volume_claim:
+        if self.kube_config.dags_volume_claim or \
+           self.kube_config.dags_volume_host or self.kube_config.dags_in_image:
             return []
 
         # Otherwise, define a git-sync init container
@@ -49,10 +58,13 @@ def _get_init_containers(self, volume_mounts):
             'value': self.kube_config.git_branch
         }, {
             'name': 'GIT_SYNC_ROOT',
-            'value': '/tmp'
+            'value': self.kube_config.git_sync_root
         }, {
             'name': 'GIT_SYNC_DEST',
-            'value': 'dags'
+            'value': self.kube_config.git_sync_dest
+        }, {
+            'name': 'GIT_SYNC_DEPTH',
+            'value': '1'
         }, {
             'name': 'GIT_SYNC_ONE_TIME',
             'value': 'true'
@@ -68,32 +80,103 @@ def _get_init_containers(self, volume_mounts):
                 'value': self.kube_config.git_password
             })
 
-        volume_mounts[0]['readOnly'] = False
+        volume_mounts = [{
+            'mountPath': self.kube_config.git_sync_root,
+            'name': self.dags_volume_name,
+            'readOnly': False
+        }]
+        if self.kube_config.git_ssh_key_secret_name:
+            volume_mounts.append({
+                'name': self.git_sync_ssh_secret_volume_name,
+                'mountPath': '/etc/git-secret/ssh',
+                'subPath': 'ssh'
+            })
+            init_environment.extend([
+                {
+                    'name': 'GIT_SSH_KEY_FILE',
+                    'value': '/etc/git-secret/ssh'
+                },
+                {
+                    'name': 'GIT_SYNC_SSH',
+                    'value': 'true'
+                }])
+        if self.kube_config.git_ssh_known_hosts_configmap_name:
+            volume_mounts.append({
+                'name': self.git_sync_ssh_known_hosts_volume_name,
+                'mountPath': '/etc/git-secret/known_hosts',
+                'subPath': 'known_hosts'
+            })
+            init_environment.extend([
+                {
+                    'name': 'GIT_KNOWN_HOSTS',
+                    'value': 'true'
+                },
+                {
+                    'name': 'GIT_SSH_KNOWN_HOSTS_FILE',
+                    'value': '/etc/git-secret/known_hosts'
+                }
+            ])
+        else:
+            init_environment.append({
+                'name': 'GIT_KNOWN_HOSTS',
+                'value': 'false'
+            })
+
         return [{
             'name': self.kube_config.git_sync_init_container_name,
             'image': self.kube_config.git_sync_container,
-            'securityContext': {'runAsUser': 0},
+            'securityContext': {'runAsUser': 65533},  # git-sync user
             'env': init_environment,
             'volumeMounts': volume_mounts
         }]
 
     def _get_environment(self):
         """Defines any necessary environment variables for the pod executor"""
-        env = {
-            'AIRFLOW__CORE__DAGS_FOLDER': '/tmp/dags',
-            'AIRFLOW__CORE__EXECUTOR': 'LocalExecutor'
-        }
+        env = {}
+
+        for env_var_name, env_var_val in six.iteritems(self.kube_config.kube_env_vars):
+            env[env_var_name] = env_var_val
+
+        env["AIRFLOW__CORE__EXECUTOR"] = "LocalExecutor"
+
         if self.kube_config.airflow_configmap:
-            env['AIRFLOW__CORE__AIRFLOW_HOME'] = self.worker_airflow_home
+            env['AIRFLOW_HOME'] = self.worker_airflow_home
+            env['AIRFLOW__CORE__DAGS_FOLDER'] = self.worker_airflow_dags
+        if (not self.kube_config.airflow_configmap and
+                'AIRFLOW__CORE__SQL_ALCHEMY_CONN' not in self.kube_config.kube_secrets):
+            env['AIRFLOW__CORE__SQL_ALCHEMY_CONN'] = conf.get("core", "SQL_ALCHEMY_CONN")
+        if self.kube_config.git_dags_folder_mount_point:
+            # /root/airflow/dags/repo/dags
+            dag_volume_mount_path = os.path.join(
+                self.kube_config.git_dags_folder_mount_point,
+                self.kube_config.git_sync_dest,  # repo
+                self.kube_config.git_subpath     # dags
+            )
+            env['AIRFLOW__CORE__DAGS_FOLDER'] = dag_volume_mount_path
         return env
 
+    def _get_configmaps(self):
+        """Extracts any configmapRefs to envFrom"""
+        if not self.kube_config.env_from_configmap_ref:
+            return []
+        return self.kube_config.env_from_configmap_ref.split(',')
+
     def _get_secrets(self):
         """Defines any necessary secrets for the pod executor"""
         worker_secrets = []
+
         for env_var_name, obj_key_pair in six.iteritems(self.kube_config.kube_secrets):
             k8s_secret_obj, k8s_secret_key = obj_key_pair.split('=')
             worker_secrets.append(
-                Secret('env', env_var_name, k8s_secret_obj, k8s_secret_key))
+                Secret('env', env_var_name, k8s_secret_obj, k8s_secret_key)
+            )
+
+        if self.kube_config.env_from_secret_ref:
+            for secret_ref in self.kube_config.env_from_secret_ref.split(','):
+                worker_secrets.append(
+                    Secret('env', None, secret_ref)
+                )
+
         return worker_secrets
 
     def _get_image_pull_secrets(self):
@@ -102,79 +185,129 @@ def _get_image_pull_secrets(self):
             return []
         return self.kube_config.image_pull_secrets.split(',')
 
-    def init_volumes_and_mounts(self):
-        dags_volume_name = 'airflow-dags'
-        logs_volume_name = 'airflow-logs'
+    def _get_security_context(self):
+        """Defines the security context"""
+        security_context = {}
+
+        if self.kube_config.worker_run_as_user:
+            security_context['runAsUser'] = self.kube_config.worker_run_as_user
+
+        if self.kube_config.worker_fs_group:
+            security_context['fsGroup'] = self.kube_config.worker_fs_group
 
-        def _construct_volume(name, claim, subpath=None):
-            vo = {
+        # set fs_group to 65533 if not explicitly specified and using git ssh keypair auth
+        if self.kube_config.git_ssh_key_secret_name and security_context.get('fsGroup') is None:
+            security_context['fsGroup'] = 65533
+
+        return security_context
+
+    def _get_volumes_and_mounts(self):
+        def _construct_volume(name, claim, host):
+            volume = {
                 'name': name
             }
             if claim:
-                vo['persistentVolumeClaim'] = {
+                volume['persistentVolumeClaim'] = {
                     'claimName': claim
                 }
-                if subpath:
-                    vo['subPath'] = subpath
+            elif host:
+                volume['hostPath'] = {
+                    'path': host,
+                    'type': ''
+                }
             else:
-                vo['emptyDir'] = {}
-            return vo
+                volume['emptyDir'] = {}
+            return volume
 
-        volumes = [
-            _construct_volume(
-                dags_volume_name,
+        volumes = {
+            self.dags_volume_name: _construct_volume(
+                self.dags_volume_name,
                 self.kube_config.dags_volume_claim,
-                self.kube_config.dags_volume_subpath
+                self.kube_config.dags_volume_host
             ),
-            _construct_volume(
-                logs_volume_name,
+            self.logs_volume_name: _construct_volume(
+                self.logs_volume_name,
                 self.kube_config.logs_volume_claim,
-                self.kube_config.logs_volume_subpath
+                self.kube_config.logs_volume_host
             )
-        ]
+        }
 
-        dag_volume_mount_path = ""
-        if self.kube_config.dags_volume_claim:
-            dag_volume_mount_path = self.worker_airflow_dags
-        else:
-            dag_volume_mount_path = os.path.join(
-                self.worker_airflow_dags,
-                self.kube_config.git_subpath
-            )
+        volume_mounts = {
+            self.dags_volume_name: {
+                'name': self.dags_volume_name,
+                'mountPath': self.generate_dag_volume_mount_path(),
+                'readOnly': True,
+            },
+            self.logs_volume_name: {
+                'name': self.logs_volume_name,
+                'mountPath': self.worker_airflow_logs,
+            }
+        }
 
-        volume_mounts = [{
-            'name': dags_volume_name,
-            'mountPath': dag_volume_mount_path,
-            'readOnly': True
-        }, {
-            'name': logs_volume_name,
-            'mountPath': self.worker_airflow_logs
-        }]
+        if self.kube_config.dags_volume_subpath:
+            volume_mounts[self.dags_volume_name]['subPath'] = self.kube_config.dags_volume_subpath
+
+        if self.kube_config.logs_volume_subpath:
+            volume_mounts[self.logs_volume_name]['subPath'] = self.kube_config.logs_volume_subpath
+
+        if self.kube_config.dags_in_image:
+            del volumes[self.dags_volume_name]
+            del volume_mounts[self.dags_volume_name]
+
+        # Get the SSH key from secrets as a volume
+        if self.kube_config.git_ssh_key_secret_name:
+            volumes[self.git_sync_ssh_secret_volume_name] = {
+                'name': self.git_sync_ssh_secret_volume_name,
+                'secret': {
+                    'secretName': self.kube_config.git_ssh_key_secret_name,
+                    'items': [{
+                        'key': self.git_ssh_key_secret_key,
+                        'path': 'ssh',
+                        'mode': 0o440
+                    }]
+                }
+            }
+
+        if self.kube_config.git_ssh_known_hosts_configmap_name:
+            volumes[self.git_sync_ssh_known_hosts_volume_name] = {
+                'name': self.git_sync_ssh_known_hosts_volume_name,
+                'configMap': {
+                    'name': self.kube_config.git_ssh_known_hosts_configmap_name
+                },
+                'mode': 0o440
+            }
 
         # Mount the airflow.cfg file via a configmap the user has specified
         if self.kube_config.airflow_configmap:
             config_volume_name = 'airflow-config'
             config_path = '{}/airflow.cfg'.format(self.worker_airflow_home)
-            volumes.append({
+            volumes[config_volume_name] = {
                 'name': config_volume_name,
                 'configMap': {
                     'name': self.kube_config.airflow_configmap
                 }
-            })
-            volume_mounts.append({
+            }
+            volume_mounts[config_volume_name] = {
                 'name': config_volume_name,
                 'mountPath': config_path,
                 'subPath': 'airflow.cfg',
                 'readOnly': True
-            })
+            }
 
         return volumes, volume_mounts
 
+    def generate_dag_volume_mount_path(self):
+        if self.kube_config.dags_volume_claim or self.kube_config.dags_volume_host:
+            dag_volume_mount_path = self.worker_airflow_dags
+        else:
+            dag_volume_mount_path = self.kube_config.git_dags_folder_mount_point
+
+        return dag_volume_mount_path
+
     def make_pod(self, namespace, worker_uuid, pod_id, dag_id, task_id, execution_date,
-                 airflow_command, kube_executor_config):
-        volumes, volume_mounts = self.init_volumes_and_mounts()
-        worker_init_container_spec = self._get_init_containers(
-            copy.deepcopy(volume_mounts))
+                 try_number, airflow_command, kube_executor_config):
+        volumes_dict, volume_mounts_dict = self._get_volumes_and_mounts()
+        worker_init_container_spec = self._get_init_containers()
         resources = Resources(
             request_memory=kube_executor_config.request_memory,
             request_cpu=kube_executor_config.request_cpu,
@@ -182,9 +315,15 @@ def make_pod(self, namespace, worker_uuid, pod_id, dag_id, task_id, execution_da
             limit_cpu=kube_executor_config.limit_cpu
         )
         gcp_sa_key = kube_executor_config.gcp_service_account_key
-        annotations = {
-            'iam.cloud.google.com/service-account': gcp_sa_key
-        } if gcp_sa_key else {}
+        annotations = dict(kube_executor_config.annotations) or self.kube_config.kube_annotations
+        if gcp_sa_key:
+            annotations['iam.cloud.google.com/service-account'] = gcp_sa_key
+
+        volumes = [value for value in volumes_dict.values()] + kube_executor_config.volumes
+        volume_mounts = [value for value in volume_mounts_dict.values()] + kube_executor_config.volume_mounts
+
+        affinity = kube_executor_config.affinity or self.kube_config.kube_affinity
+        tolerations = kube_executor_config.tolerations or self.kube_config.kube_tolerations
 
         return Pod(
             namespace=namespace,
@@ -192,13 +331,13 @@ def make_pod(self, namespace, worker_uuid, pod_id, dag_id, task_id, execution_da
             image=kube_executor_config.image or self.kube_config.kube_image,
             image_pull_policy=(kube_executor_config.image_pull_policy or
                                self.kube_config.kube_image_pull_policy),
-            cmds=['bash', '-cx', '--'],
-            args=[airflow_command],
+            cmds=airflow_command,
             labels={
                 'airflow-worker': worker_uuid,
                 'dag_id': dag_id,
                 'task_id': task_id,
-                'execution_date': execution_date
+                'execution_date': execution_date,
+                'try_number': str(try_number),
             },
             envs=self._get_environment(),
             secrets=self._get_secrets(),
@@ -208,5 +347,11 @@ def make_pod(self, namespace, worker_uuid, pod_id, dag_id, task_id, execution_da
             volumes=volumes,
             volume_mounts=volume_mounts,
             resources=resources,
-            annotations=annotations
+            annotations=annotations,
+            node_selectors=(kube_executor_config.node_selectors or
+                            self.kube_config.kube_node_selectors),
+            affinity=affinity,
+            tolerations=tolerations,
+            security_context=self._get_security_context(),
+            configmaps=self._get_configmaps()
         )
diff --git a/airflow/contrib/operators/adls_list_operator.py b/airflow/contrib/operators/adls_list_operator.py
new file mode 100644
index 0000000000000..33c99064aa27b
--- /dev/null
+++ b/airflow/contrib/operators/adls_list_operator.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Iterable
+
+from airflow.contrib.hooks.azure_data_lake_hook import AzureDataLakeHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class AzureDataLakeStorageListOperator(BaseOperator):
+    """
+    List all files from the specified path
+
+    This operator returns a python list with the names of files which can be used by
+     `xcom` in the downstream tasks.
+
+    :param path: The Azure Data Lake path to find the objects. Supports glob
+        strings (templated)
+    :type path: str
+    :param azure_data_lake_conn_id: The connection ID to use when
+        connecting to Azure Data Lake Storage.
+    :type azure_data_lake_conn_id: str
+
+    **Example**:
+        The following Operator would list all the Parquet files from ``folder/output/``
+        folder in the specified ADLS account ::
+
+            adls_files = AzureDataLakeStorageListOperator(
+                task_id='adls_files',
+                path='folder/output/*.parquet',
+                azure_data_lake_conn_id='azure_data_lake_default'
+            )
+    """
+    template_fields = ('path',)  # type: Iterable[str]
+    ui_color = '#901dd2'
+
+    @apply_defaults
+    def __init__(self,
+                 path,
+                 azure_data_lake_conn_id='azure_data_lake_default',
+                 *args,
+                 **kwargs):
+        super(AzureDataLakeStorageListOperator, self).__init__(*args, **kwargs)
+        self.path = path
+        self.azure_data_lake_conn_id = azure_data_lake_conn_id
+
+    def execute(self, context):
+
+        hook = AzureDataLakeHook(
+            azure_data_lake_conn_id=self.azure_data_lake_conn_id
+        )
+
+        self.log.info('Getting list of ADLS files in path: %s', self.path)
+
+        return hook.list(path=self.path)
diff --git a/airflow/contrib/operators/adls_to_gcs.py b/airflow/contrib/operators/adls_to_gcs.py
new file mode 100644
index 0000000000000..5d99251d3f6d2
--- /dev/null
+++ b/airflow/contrib/operators/adls_to_gcs.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from tempfile import NamedTemporaryFile
+
+from airflow.contrib.hooks.azure_data_lake_hook import AzureDataLakeHook
+from airflow.contrib.operators.adls_list_operator import AzureDataLakeStorageListOperator
+from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook, _parse_gcs_url
+from airflow.utils.decorators import apply_defaults
+
+
+class AdlsToGoogleCloudStorageOperator(AzureDataLakeStorageListOperator):
+    """
+    Synchronizes an Azure Data Lake Storage path with a GCS bucket
+
+    :param src_adls: The Azure Data Lake path to find the objects (templated)
+    :type src_adls: str
+    :param dest_gcs: The Google Cloud Storage bucket and prefix to
+        store the objects. (templated)
+    :type dest_gcs: str
+    :param replace: If true, replaces same-named files in GCS
+    :type replace: bool
+    :param azure_data_lake_conn_id: The connection ID to use when
+        connecting to Azure Data Lake Storage.
+    :type azure_data_lake_conn_id: str
+    :param google_cloud_storage_conn_id: The connection ID to use when
+        connecting to Google Cloud Storage.
+    :type google_cloud_storage_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide delegation enabled.
+    :type delegate_to: str
+
+    **Examples**:
+        The following Operator would copy a single file named
+        ``hello/world.avro`` from ADLS to the GCS bucket ``mybucket``. Its full
+        resulting gcs path will be ``gs://mybucket/hello/world.avro`` ::
+
+            copy_single_file = AdlsToGoogleCloudStorageOperator(
+                task_id='copy_single_file',
+                src_adls='hello/world.avro',
+                dest_gcs='gs://mybucket',
+                replace=False,
+                azure_data_lake_conn_id='azure_data_lake_default',
+                google_cloud_storage_conn_id='google_cloud_default'
+            )
+
+        The following Operator would copy all parquet files from ADLS
+        to the GCS bucket ``mybucket``. ::
+
+            copy_all_files = AdlsToGoogleCloudStorageOperator(
+                task_id='copy_all_files',
+                src_adls='*.parquet',
+                dest_gcs='gs://mybucket',
+                replace=False,
+                azure_data_lake_conn_id='azure_data_lake_default',
+                google_cloud_storage_conn_id='google_cloud_default'
+            )
+
+         The following Operator would copy all parquet files from ADLS
+         path ``/hello/world``to the GCS bucket ``mybucket``. ::
+
+            copy_world_files = AdlsToGoogleCloudStorageOperator(
+                task_id='copy_world_files',
+                src_adls='hello/world/*.parquet',
+                dest_gcs='gs://mybucket',
+                replace=False,
+                azure_data_lake_conn_id='azure_data_lake_default',
+                google_cloud_storage_conn_id='google_cloud_default'
+            )
+    """
+    template_fields = ('src_adls', 'dest_gcs')
+    ui_color = '#f0eee4'
+
+    @apply_defaults
+    def __init__(self,
+                 src_adls,
+                 dest_gcs,
+                 azure_data_lake_conn_id,
+                 google_cloud_storage_conn_id,
+                 delegate_to=None,
+                 replace=False,
+                 *args,
+                 **kwargs):
+
+        super(AdlsToGoogleCloudStorageOperator, self).__init__(
+            path=src_adls,
+            azure_data_lake_conn_id=azure_data_lake_conn_id,
+            *args,
+            **kwargs
+        )
+        self.src_adls = src_adls
+        self.dest_gcs = dest_gcs
+        self.replace = replace
+        self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
+        self.delegate_to = delegate_to
+
+    def execute(self, context):
+        # use the super to list all files in an Azure Data Lake path
+        files = super(AdlsToGoogleCloudStorageOperator, self).execute(context)
+        g_hook = GoogleCloudStorageHook(
+            google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
+            delegate_to=self.delegate_to)
+
+        if not self.replace:
+            # if we are not replacing -> list all files in the ADLS path
+            # and only keep those files which are present in
+            # ADLS and not in Google Cloud Storage
+            bucket_name, prefix = _parse_gcs_url(self.dest_gcs)
+            existing_files = g_hook.list(bucket=bucket_name, prefix=prefix)
+            files = set(files) - set(existing_files)
+
+        if files:
+            hook = AzureDataLakeHook(
+                azure_data_lake_conn_id=self.azure_data_lake_conn_id
+            )
+
+            for obj in files:
+                with NamedTemporaryFile(mode='wb', delete=True) as f:
+                    hook.download_file(local_path=f.name, remote_path=obj)
+                    f.flush()
+                    dest_gcs_bucket, dest_gcs_prefix = _parse_gcs_url(self.dest_gcs)
+                    dest_path = os.path.join(dest_gcs_prefix, obj)
+                    self.log.info("Saving file to %s", dest_path)
+
+                    g_hook.upload(bucket=dest_gcs_bucket, object=dest_path, filename=f.name)
+
+            self.log.info("All done, uploaded %d files to GCS", len(files))
+        else:
+            self.log.info("In sync, no files needed to be uploaded to GCS")
+
+        return files
diff --git a/airflow/contrib/operators/aws_athena_operator.py b/airflow/contrib/operators/aws_athena_operator.py
new file mode 100644
index 0000000000000..28054d320168a
--- /dev/null
+++ b/airflow/contrib/operators/aws_athena_operator.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from uuid import uuid4
+
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook
+
+
+class AWSAthenaOperator(BaseOperator):
+    """
+    An operator that submit presto query to athena.
+
+    :param query: Presto to be run on athena. (templated)
+    :type query: str
+    :param database: Database to select. (templated)
+    :type database: str
+    :param output_location: s3 path to write the query results into. (templated)
+    :type output_location: str
+    :param aws_conn_id: aws connection to use
+    :type aws_conn_id: str
+    :param sleep_time: Time to wait between two consecutive call to check query status on athena
+    :type sleep_time: int
+    """
+
+    ui_color = '#44b5e2'
+    template_fields = ('query', 'database', 'output_location')
+
+    @apply_defaults
+    def __init__(self, query, database, output_location, aws_conn_id='aws_default', client_request_token=None,
+                 query_execution_context=None, result_configuration=None, sleep_time=30, *args, **kwargs):
+        super(AWSAthenaOperator, self).__init__(*args, **kwargs)
+        self.query = query
+        self.database = database
+        self.output_location = output_location
+        self.aws_conn_id = aws_conn_id
+        self.client_request_token = client_request_token or str(uuid4())
+        self.query_execution_context = query_execution_context or {}
+        self.result_configuration = result_configuration or {}
+        self.sleep_time = sleep_time
+        self.query_execution_id = None
+        self.hook = None
+
+    def get_hook(self):
+        return AWSAthenaHook(self.aws_conn_id, self.sleep_time)
+
+    def execute(self, context):
+        """
+        Run Presto Query on Athena
+        """
+        self.hook = self.get_hook()
+        self.hook.get_conn()
+
+        self.query_execution_context['Database'] = self.database
+        self.result_configuration['OutputLocation'] = self.output_location
+        self.query_execution_id = self.hook.run_query(self.query, self.query_execution_context,
+                                                      self.result_configuration, self.client_request_token)
+        self.hook.poll_query_status(self.query_execution_id)
+
+    def on_kill(self):
+        """
+        Cancel the submitted athena query
+        """
+        if self.query_execution_id:
+            self.log.info('⚰️⚰️⚰️ Received a kill Signal. Time to Die')
+            self.log.info(
+                'Stopping Query with executionId - %s', self.query_execution_id
+            )
+            response = self.hook.stop_query(self.query_execution_id)
+            http_status_code = None
+            try:
+                http_status_code = response['ResponseMetadata']['HTTPStatusCode']
+            except Exception as ex:
+                self.log.error('Exception while cancelling query', ex)
+            finally:
+                if http_status_code is None or http_status_code != 200:
+                    self.log.error('Unable to request query cancel on athena. Exiting')
+                else:
+                    self.log.info(
+                        'Polling Athena for query with id %s to reach final state', self.query_execution_id
+                    )
+                    self.hook.poll_query_status(self.query_execution_id)
diff --git a/airflow/contrib/operators/awsbatch_operator.py b/airflow/contrib/operators/awsbatch_operator.py
index d23b44e0d4471..c0b96d819c820 100644
--- a/airflow/contrib/operators/awsbatch_operator.py
+++ b/airflow/contrib/operators/awsbatch_operator.py
@@ -33,7 +33,7 @@ class AWSBatchOperator(BaseOperator):
     """
     Execute a job on AWS Batch Service
 
-    .. warning: the queue parameter was renamed to job_queue to segreggate the
+    .. warning: the queue parameter was renamed to job_queue to segregate the
                 internal CeleryExecutor queue from the AWS Batch internal queue.
 
     :param job_name: the name for the job that will run on AWS Batch
@@ -42,18 +42,20 @@ class AWSBatchOperator(BaseOperator):
     :type job_definition: str
     :param job_queue: the queue name on AWS Batch
     :type job_queue: str
-    :param: overrides: the same parameter that boto3 will receive on
-            containerOverrides (templated):
-            http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job
-    :type: overrides: dict
-    :param max_retries: exponential backoff retries while waiter is not merged, 4200 = 48 hours
+    :param overrides: the same parameter that boto3 will receive on
+        containerOverrides (templated):
+        http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job
+    :type overrides: dict
+    :param max_retries: exponential backoff retries while waiter is not
+        merged, 4200 = 48 hours
     :type max_retries: int
     :param aws_conn_id: connection id of AWS credentials / region name. If None,
-            credential boto3 strategy will be used
-            (http://boto3.readthedocs.io/en/latest/guide/configuration.html).
+        credential boto3 strategy will be used
+        (http://boto3.readthedocs.io/en/latest/guide/configuration.html).
     :type aws_conn_id: str
     :param region_name: region name to use in AWS Hook.
         Override the region_name in connection (if provided)
+    :type region_name: str
     """
 
     ui_color = '#c3dae0'
@@ -139,7 +141,7 @@ def _wait_for_task_ended(self):
                 if response['jobs'][-1]['status'] in ['SUCCEEDED', 'FAILED']:
                     retry = False
 
-                sleep( 1 + pow(retries * 0.1, 2))
+                sleep(1 + pow(retries * 0.1, 2))
                 retries += 1
 
     def _check_success_task(self):
@@ -152,17 +154,19 @@ def _check_success_task(self):
             raise AirflowException('No job found for {}'.format(response))
 
         for job in response['jobs']:
-            if 'attempts' in job:
-                containers = job['attempts']
-                for container in containers:
-                    if (job['status'] == 'FAILED' or
-                            container['container']['exitCode'] != 0):
-                        raise AirflowException(
-                            'This containers encounter an error during '
-                            'execution {}'.format(job))
-            elif job['status'] is not 'SUCCEEDED':
+            job_status = job['status']
+            if job_status == 'FAILED':
+                reason = job['statusReason']
+                raise AirflowException('Job failed with status {}'.format(reason))
+            elif job_status in [
+                'SUBMITTED',
+                'PENDING',
+                'RUNNABLE',
+                'STARTING',
+                'RUNNING'
+            ]:
                 raise AirflowException(
-                    'This task is still pending {}'.format(job['status']))
+                    'This task is still pending {}'.format(job_status))
 
     def get_hook(self):
         return AwsHook(
diff --git a/airflow/contrib/operators/azure_container_instances_operator.py b/airflow/contrib/operators/azure_container_instances_operator.py
new file mode 100644
index 0000000000000..2d01e13f65d1b
--- /dev/null
+++ b/airflow/contrib/operators/azure_container_instances_operator.py
@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from collections import namedtuple
+from time import sleep
+from typing import Dict, Sequence
+
+from airflow.contrib.hooks.azure_container_instance_hook import AzureContainerInstanceHook
+from airflow.contrib.hooks.azure_container_registry_hook import AzureContainerRegistryHook
+from airflow.contrib.hooks.azure_container_volume_hook import AzureContainerVolumeHook
+
+from airflow.exceptions import AirflowException, AirflowTaskTimeout
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+from azure.mgmt.containerinstance.models import (EnvironmentVariable,
+                                                 VolumeMount,
+                                                 ResourceRequests,
+                                                 ResourceRequirements,
+                                                 Container,
+                                                 ContainerGroup)
+from msrestazure.azure_exceptions import CloudError
+
+
+Volume = namedtuple(
+    'Volume',
+    ['conn_id', 'account_name', 'share_name', 'mount_path', 'read_only'],
+)
+
+DEFAULT_ENVIRONMENT_VARIABLES = {}  # type: Dict[str, str]
+DEFAULT_VOLUMES = []  # type: Sequence[Volume]
+DEFAULT_MEMORY_IN_GB = 2.0
+DEFAULT_CPU = 1.0
+
+
+class AzureContainerInstancesOperator(BaseOperator):
+    """
+    Start a container on Azure Container Instances
+
+    :param ci_conn_id: connection id of a service principal which will be used
+        to start the container instance
+    :type ci_conn_id: str
+    :param registry_conn_id: connection id of a user which can login to a
+        private docker registry. If None, we assume a public registry
+    :type registry_conn_id: str
+    :param resource_group: name of the resource group wherein this container
+        instance should be started
+    :type resource_group: str
+    :param name: name of this container instance. Please note this name has
+        to be unique in order to run containers in parallel.
+    :type name: str
+    :param image: the docker image to be used
+    :type image: str
+    :param region: the region wherein this container instance should be started
+    :type region: str
+    :param environment_variables: key,value pairs containing environment
+        variables which will be passed to the running container
+    :type environment_variables: dict
+    :param volumes: list of volumes to be mounted to the container.
+        Currently only Azure Fileshares are supported.
+    :type volumes: list[<conn_id, account_name, share_name, mount_path, read_only>]
+    :param memory_in_gb: the amount of memory to allocate to this container
+    :type memory_in_gb: double
+    :param cpu: the number of cpus to allocate to this container
+    :type cpu: double
+    :param command: the command to run inside the container
+    :type command: str
+
+    :Example:
+
+    >>>  a = AzureContainerInstancesOperator(
+                'azure_service_principal',
+                'azure_registry_user',
+                'my-resource-group',
+                'my-container-name-{{ ds }}',
+                'myprivateregistry.azurecr.io/my_container:latest',
+                'westeurope',
+                {'EXECUTION_DATE': '{{ ds }}'},
+                [('azure_wasb_conn_id',
+                  'my_storage_container',
+                  'my_fileshare',
+                  '/input-data',
+                  True),],
+                memory_in_gb=14.0,
+                cpu=4.0,
+                command='python /app/myfile.py',
+                task_id='start_container'
+            )
+    """
+
+    template_fields = ('name', 'environment_variables')
+
+    @apply_defaults
+    def __init__(self, ci_conn_id, registry_conn_id, resource_group, name, image, region,
+                 environment_variables=None, volumes=None, memory_in_gb=None, cpu=None,
+                 command=None, remove_on_error=True, fail_if_exists=True, *args, **kwargs):
+        super(AzureContainerInstancesOperator, self).__init__(*args, **kwargs)
+
+        self.ci_conn_id = ci_conn_id
+        self.resource_group = resource_group
+        self.name = name
+        self.image = image
+        self.region = region
+        self.registry_conn_id = registry_conn_id
+        self.environment_variables = environment_variables or DEFAULT_ENVIRONMENT_VARIABLES
+        self.volumes = volumes or DEFAULT_VOLUMES
+        self.memory_in_gb = memory_in_gb or DEFAULT_MEMORY_IN_GB
+        self.cpu = cpu or DEFAULT_CPU
+        self.command = command
+        self.remove_on_error = remove_on_error
+        self.fail_if_exists = fail_if_exists
+
+    def execute(self, context):
+        ci_hook = AzureContainerInstanceHook(self.ci_conn_id)
+
+        if self.fail_if_exists:
+            self.log.info("Testing if container group already exists")
+            if ci_hook.exists(self.resource_group, self.name):
+                raise AirflowException("Container group exists")
+
+        if self.registry_conn_id:
+            registry_hook = AzureContainerRegistryHook(self.registry_conn_id)
+            image_registry_credentials = [registry_hook.connection, ]
+        else:
+            image_registry_credentials = None
+
+        environment_variables = []
+        for key, value in self.environment_variables.items():
+            environment_variables.append(EnvironmentVariable(key, value))
+
+        volumes = []
+        volume_mounts = []
+        for conn_id, account_name, share_name, mount_path, read_only in self.volumes:
+            hook = AzureContainerVolumeHook(conn_id)
+
+            mount_name = "mount-%d" % len(volumes)
+            volumes.append(hook.get_file_volume(mount_name,
+                                                share_name,
+                                                account_name,
+                                                read_only))
+            volume_mounts.append(VolumeMount(mount_name, mount_path, read_only))
+
+        exit_code = 1
+        try:
+            self.log.info("Starting container group with %.1f cpu %.1f mem",
+                          self.cpu, self.memory_in_gb)
+
+            resources = ResourceRequirements(requests=ResourceRequests(
+                memory_in_gb=self.memory_in_gb,
+                cpu=self.cpu))
+
+            container = Container(
+                name=self.name,
+                image=self.image,
+                resources=resources,
+                command=self.command,
+                environment_variables=environment_variables,
+                volume_mounts=volume_mounts)
+
+            container_group = ContainerGroup(
+                location=self.region,
+                containers=[container, ],
+                image_registry_credentials=image_registry_credentials,
+                volumes=volumes,
+                restart_policy='Never',
+                os_type='Linux')
+
+            ci_hook.create_or_update(self.resource_group, self.name, container_group)
+
+            self.log.info("Container group started %s/%s", self.resource_group, self.name)
+
+            exit_code = self._monitor_logging(ci_hook, self.resource_group, self.name)
+
+            self.log.info("Container had exit code: %s", exit_code)
+            if exit_code != 0:
+                raise AirflowException("Container had a non-zero exit code, %s"
+                                       % exit_code)
+
+        except CloudError:
+            self.log.exception("Could not start container group")
+            raise AirflowException("Could not start container group")
+
+        finally:
+            if exit_code == 0 or self.remove_on_error:
+                self.log.info("Deleting container group")
+                try:
+                    ci_hook.delete(self.resource_group, self.name)
+                except Exception:
+                    self.log.exception("Could not delete container group")
+
+    def _monitor_logging(self, ci_hook, resource_group, name):
+        last_state = None
+        last_message_logged = None
+        last_line_logged = None
+        for _ in range(43200):  # roughly 12 hours
+            try:
+                state, exit_code, detail_status = ci_hook.get_state_exitcode_details(resource_group, name)
+                if state != last_state:
+                    self.log.info("Container group state changed to %s", state)
+                    last_state = state
+
+                messages = ci_hook.get_messages(resource_group, name)
+                last_message_logged = self._log_last(messages, last_message_logged)
+
+                if state in ["Running", "Terminated"]:
+                    try:
+                        logs = ci_hook.get_logs(resource_group, name)
+                        last_line_logged = self._log_last(logs, last_line_logged)
+                    except CloudError:
+                        self.log.exception("Exception while getting logs from "
+                                           "container instance, retrying...")
+
+                if state == "Terminated":
+                    self.log.info("Container exited with detail_status %s", detail_status)
+                    return exit_code
+
+            except CloudError as err:
+                if 'ResourceNotFound' in str(err):
+                    self.log.warning("ResourceNotFound, container is probably removed "
+                                     "by another process "
+                                     "(make sure that the name is unique).")
+                    return 1
+                else:
+                    self.log.exception("Exception while getting container groups")
+            except Exception:
+                self.log.exception("Exception while getting container groups")
+
+            sleep(1)
+
+        # no return -> hence still running
+        raise AirflowTaskTimeout("Did not complete on time")
+
+    def _log_last(self, logs, last_line_logged):
+        if logs:
+            # determine the last line which was logged before
+            last_line_index = 0
+            for i in range(len(logs) - 1, -1, -1):
+                if logs[i] == last_line_logged:
+                    # this line is the same, hence print from i+1
+                    last_line_index = i + 1
+                    break
+
+            # log all new ones
+            for line in logs[last_line_index:]:
+                self.log.info(line.rstrip())
+
+            return logs[-1]
diff --git a/airflow/contrib/operators/azure_cosmos_operator.py b/airflow/contrib/operators/azure_cosmos_operator.py
new file mode 100644
index 0000000000000..e0fbaf2dd2f72
--- /dev/null
+++ b/airflow/contrib/operators/azure_cosmos_operator.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.azure_cosmos_hook import AzureCosmosDBHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class AzureCosmosInsertDocumentOperator(BaseOperator):
+    """
+    Inserts a new document into the specified Cosmos database and collection
+    It will create both the database and collection if they do not already exist
+
+    :param database_name: The name of the database. (templated)
+    :type database_name: str
+    :param collection_name: The name of the collection. (templated)
+    :type collection_name: str
+    :param document: The document to insert
+    :type document: dict
+    :param azure_cosmos_conn_id: reference to a CosmosDB connection.
+    :type azure_cosmos_conn_id: str
+    """
+    template_fields = ('database_name', 'collection_name')
+    ui_color = '#e4f0e8'
+
+    @apply_defaults
+    def __init__(self,
+                 database_name,
+                 collection_name,
+                 document,
+                 azure_cosmos_conn_id='azure_cosmos_default',
+                 *args,
+                 **kwargs):
+        super(AzureCosmosInsertDocumentOperator, self).__init__(*args, **kwargs)
+        self.database_name = database_name
+        self.collection_name = collection_name
+        self.document = document
+        self.azure_cosmos_conn_id = azure_cosmos_conn_id
+
+    def execute(self, context):
+        # Create the hook
+        hook = AzureCosmosDBHook(azure_cosmos_conn_id=self.azure_cosmos_conn_id)
+
+        # Create the DB if it doesn't already exist
+        if not hook.does_database_exist(self.database_name):
+            hook.create_database(self.database_name)
+
+        # Create the collection as well
+        if not hook.does_collection_exist(self.collection_name, self.database_name):
+            hook.create_collection(self.collection_name, self.database_name)
+
+        # finally insert the document
+        hook.upsert_document(self.document, self.database_name, self.collection_name)
diff --git a/airflow/contrib/operators/bigquery_check_operator.py b/airflow/contrib/operators/bigquery_check_operator.py
index 59ef5d377d182..afb600a3d9120 100644
--- a/airflow/contrib/operators/bigquery_check_operator.py
+++ b/airflow/contrib/operators/bigquery_check_operator.py
@@ -48,28 +48,32 @@ class BigQueryCheckOperator(CheckOperator):
     This operator can be used as a data quality check in your pipeline, and
     depending on where you put it in your DAG, you have the choice to
     stop the critical path, preventing from
-    publishing dubious data, or on the side and receive email alterts
+    publishing dubious data, or on the side and receive email alerts
     without stopping the progress of the DAG.
 
     :param sql: the sql to be executed
-    :type sql: string
+    :type sql: str
     :param bigquery_conn_id: reference to the BigQuery database
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
+    :param use_legacy_sql: Whether to use legacy SQL (true)
+        or standard SQL (false).
+    :type use_legacy_sql: bool
     """
 
     @apply_defaults
-    def __init__(
-            self,
-            sql,
-            bigquery_conn_id='bigquery_default',
-            *args,
-            **kwargs):
+    def __init__(self,
+                 sql,
+                 bigquery_conn_id='bigquery_default',
+                 use_legacy_sql=True,
+                 *args, **kwargs):
         super(BigQueryCheckOperator, self).__init__(sql=sql, *args, **kwargs)
         self.bigquery_conn_id = bigquery_conn_id
         self.sql = sql
+        self.use_legacy_sql = use_legacy_sql
 
     def get_db_hook(self):
-        return BigQueryHook(bigquery_conn_id=self.bigquery_conn_id)
+        return BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
+                            use_legacy_sql=self.use_legacy_sql)
 
 
 class BigQueryValueCheckOperator(ValueCheckOperator):
@@ -77,21 +81,28 @@ class BigQueryValueCheckOperator(ValueCheckOperator):
     Performs a simple value check using sql code.
 
     :param sql: the sql to be executed
-    :type sql: string
+    :type sql: str
+    :param use_legacy_sql: Whether to use legacy SQL (true)
+        or standard SQL (false).
+    :type use_legacy_sql: bool
     """
 
     @apply_defaults
-    def __init__(
-            self, sql, pass_value, tolerance=None,
-            bigquery_conn_id='bigquery_default',
-            *args, **kwargs):
+    def __init__(self, sql,
+                 pass_value,
+                 tolerance=None,
+                 bigquery_conn_id='bigquery_default',
+                 use_legacy_sql=True,
+                 *args, **kwargs):
         super(BigQueryValueCheckOperator, self).__init__(
             sql=sql, pass_value=pass_value, tolerance=tolerance,
             *args, **kwargs)
         self.bigquery_conn_id = bigquery_conn_id
+        self.use_legacy_sql = use_legacy_sql
 
     def get_db_hook(self):
-        return BigQueryHook(bigquery_conn_id=self.bigquery_conn_id)
+        return BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
+                            use_legacy_sql=self.use_legacy_sql)
 
 
 class BigQueryIntervalCheckOperator(IntervalCheckOperator):
@@ -102,7 +113,7 @@ class BigQueryIntervalCheckOperator(IntervalCheckOperator):
     This method constructs a query like so ::
 
         SELECT {metrics_threshold_dict_key} FROM {table}
-            WHERE {date_filter_column}=<date>
+        WHERE {date_filter_column}=<date>
 
     :param table: the table name
     :type table: str
@@ -113,19 +124,22 @@ class BigQueryIntervalCheckOperator(IntervalCheckOperator):
         example 'COUNT(*)': 1.5 would require a 50 percent or less difference
         between the current day, and the prior days_back.
     :type metrics_threshold: dict
+    :param use_legacy_sql: Whether to use legacy SQL (true)
+        or standard SQL (false).
+    :type use_legacy_sql: bool
     """
 
     @apply_defaults
-    def __init__(
-            self, table, metrics_thresholds,
-            date_filter_column='ds', days_back=-7,
-            bigquery_conn_id='bigquery_default',
-            *args, **kwargs):
+    def __init__(self, table, metrics_thresholds, date_filter_column='ds',
+                 days_back=-7, bigquery_conn_id='bigquery_default',
+                 use_legacy_sql=True, *args, **kwargs):
         super(BigQueryIntervalCheckOperator, self).__init__(
             table=table, metrics_thresholds=metrics_thresholds,
             date_filter_column=date_filter_column, days_back=days_back,
             *args, **kwargs)
         self.bigquery_conn_id = bigquery_conn_id
+        self.use_legacy_sql = use_legacy_sql
 
     def get_db_hook(self):
-        return BigQueryHook(bigquery_conn_id=self.bigquery_conn_id)
+        return BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
+                            use_legacy_sql=self.use_legacy_sql)
diff --git a/airflow/contrib/operators/bigquery_get_data.py b/airflow/contrib/operators/bigquery_get_data.py
index ab8f71b717716..f5e6e50f066d5 100644
--- a/airflow/contrib/operators/bigquery_get_data.py
+++ b/airflow/contrib/operators/bigquery_get_data.py
@@ -51,21 +51,21 @@ class BigQueryGetDataOperator(BaseOperator):
         )
 
     :param dataset_id: The dataset ID of the requested table. (templated)
-    :type destination_dataset_table: string
+    :type dataset_id: str
     :param table_id: The table ID of the requested table. (templated)
-    :type table_id: string
+    :type table_id: str
     :param max_results: The maximum number of records (rows) to be fetched
         from the table. (templated)
-    :type max_results: string
+    :type max_results: str
     :param selected_fields: List of fields to return (comma-separated). If
         unspecified, all fields are returned.
-    :type selected_fields: string
+    :type selected_fields: str
     :param bigquery_conn_id: reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
     template_fields = ('dataset_id', 'table_id', 'max_results')
     ui_color = '#e4f0e8'
diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py
index b36efbd6bfd43..746abac5360ce 100644
--- a/airflow/contrib/operators/bigquery_operator.py
+++ b/airflow/contrib/operators/bigquery_operator.py
@@ -39,60 +39,80 @@ class BigQueryOperator(BaseOperator):
         a list of str (sql statements), or reference to a template file.
         Template reference are recognized by str ending in '.sql'.
     :param destination_dataset_table: A dotted
-        (<project>.|<project>:)<dataset>.<table> that, if set, will store the results
+        ``(<project>.|<project>:)<dataset>.<table>`` that, if set, will store the results
         of the query. (templated)
-    :type destination_dataset_table: string
+    :type destination_dataset_table: str
     :param write_disposition: Specifies the action that occurs if the destination table
         already exists. (default: 'WRITE_EMPTY')
-    :type write_disposition: string
+    :type write_disposition: str
     :param create_disposition: Specifies whether the job is allowed to create new tables.
         (default: 'CREATE_IF_NEEDED')
-    :type create_disposition: string
+    :type create_disposition: str
     :param allow_large_results: Whether to allow large results.
-    :type allow_large_results: boolean
+    :type allow_large_results: bool
     :param flatten_results: If true and query uses legacy SQL dialect, flattens
         all nested and repeated fields in the query results. ``allow_large_results``
         must be ``true`` if this is set to ``false``. For standard SQL queries, this
         flag is ignored and results are never flattened.
-    :type flatten_results: boolean
+    :type flatten_results: bool
     :param bigquery_conn_id: reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param udf_config: The User Defined Function configuration for the query.
         See https://cloud.google.com/bigquery/user-defined-functions for details.
     :type udf_config: list
     :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).
-    :type use_legacy_sql: boolean
+    :type use_legacy_sql: bool
     :param maximum_billing_tier: Positive integer that serves as a multiplier
         of the basic price.
         Defaults to None, in which case it uses the value set in the project.
-    :type maximum_billing_tier: integer
+    :type maximum_billing_tier: int
     :param maximum_bytes_billed: Limits the bytes billed for this job.
         Queries that will have bytes billed beyond this limit will fail
         (without incurring a charge). If unspecified, this will be
         set to your project default.
     :type maximum_bytes_billed: float
+    :param api_resource_configs: a dictionary that contain params
+        'configuration' applied for Google BigQuery Jobs API:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs
+        for example, {'query': {'useQueryCache': False}}. You could use it
+        if you need to provide some params that are not supported by BigQueryOperator
+        like args.
+    :type api_resource_configs: dict
     :param schema_update_options: Allows the schema of the destination
         table to be updated as a side effect of the load job.
     :type schema_update_options: tuple
-    :param query_params: a dictionary containing query parameter types and
-        values, passed to BigQuery.
-    :type query_params: dict
+    :param query_params: a list of dictionary containing query parameter types and
+        values, passed to BigQuery. The structure of dictionary should look like
+        'queryParameters' in Google BigQuery Jobs API:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs.
+        For example, [{ 'name': 'corpus', 'parameterType': { 'type': 'STRING' },
+        'parameterValue': { 'value': 'romeoandjuliet' } }].
+    :type query_params: list
+    :param labels: a dictionary containing labels for the job/query,
+        passed to BigQuery
+    :type labels: dict
     :param priority: Specifies a priority for the query.
         Possible values include INTERACTIVE and BATCH.
         The default value is INTERACTIVE.
-    :type priority: string
+    :type priority: str
     :param time_partitioning: configure optional time partitioning fields i.e.
-        partition by field, type and
-        expiration as per API specifications. Note that 'field' is not available in
-        conjunction with dataset.table$partition.
+        partition by field, type and expiration as per API specifications.
     :type time_partitioning: dict
+    :param cluster_fields: Request that the result of this query be stored sorted
+        by one or more columns. This is only available in conjunction with
+        time_partitioning. The order of columns given determines the sort order.
+    :type cluster_fields: list[str]
+    :param location: The geographic location of the job. Required except for
+        US and EU. See details at
+        https://cloud.google.com/bigquery/docs/locations#specifying_your_location
+    :type location: str
     """
 
-    template_fields = ('bql', 'sql', 'destination_dataset_table')
+    template_fields = ('bql', 'sql', 'destination_dataset_table', 'labels')
     template_ext = ('.sql', )
     ui_color = '#e4f0e8'
 
@@ -100,21 +120,25 @@ class BigQueryOperator(BaseOperator):
     def __init__(self,
                  bql=None,
                  sql=None,
-                 destination_dataset_table=False,
+                 destination_dataset_table=None,
                  write_disposition='WRITE_EMPTY',
                  allow_large_results=False,
-                 flatten_results=False,
+                 flatten_results=None,
                  bigquery_conn_id='bigquery_default',
                  delegate_to=None,
-                 udf_config=False,
+                 udf_config=None,
                  use_legacy_sql=True,
                  maximum_billing_tier=None,
                  maximum_bytes_billed=None,
                  create_disposition='CREATE_IF_NEEDED',
                  schema_update_options=(),
                  query_params=None,
+                 labels=None,
                  priority='INTERACTIVE',
-                 time_partitioning={},
+                 time_partitioning=None,
+                 api_resource_configs=None,
+                 cluster_fields=None,
+                 location=None,
                  *args,
                  **kwargs):
         super(BigQueryOperator, self).__init__(*args, **kwargs)
@@ -133,9 +157,13 @@ def __init__(self,
         self.maximum_bytes_billed = maximum_bytes_billed
         self.schema_update_options = schema_update_options
         self.query_params = query_params
+        self.labels = labels
         self.bq_cursor = None
         self.priority = priority
         self.time_partitioning = time_partitioning
+        self.api_resource_configs = api_resource_configs
+        self.cluster_fields = cluster_fields
+        self.location = location
 
         # TODO remove `bql` in Airflow 2.0
         if self.bql:
@@ -157,11 +185,13 @@ def execute(self, context):
             hook = BigQueryHook(
                 bigquery_conn_id=self.bigquery_conn_id,
                 use_legacy_sql=self.use_legacy_sql,
-                delegate_to=self.delegate_to)
+                delegate_to=self.delegate_to,
+                location=self.location,
+            )
             conn = hook.get_conn()
             self.bq_cursor = conn.cursor()
         self.bq_cursor.run_query(
-            self.sql,
+            sql=self.sql,
             destination_dataset_table=self.destination_dataset_table,
             write_disposition=self.write_disposition,
             allow_large_results=self.allow_large_results,
@@ -171,15 +201,18 @@ def execute(self, context):
             maximum_bytes_billed=self.maximum_bytes_billed,
             create_disposition=self.create_disposition,
             query_params=self.query_params,
+            labels=self.labels,
             schema_update_options=self.schema_update_options,
             priority=self.priority,
-            time_partitioning=self.time_partitioning
+            time_partitioning=self.time_partitioning,
+            api_resource_configs=self.api_resource_configs,
+            cluster_fields=self.cluster_fields,
         )
 
     def on_kill(self):
         super(BigQueryOperator, self).on_kill()
         if self.bq_cursor is not None:
-            self.log.info('Canceling running query due to execution timeout')
+            self.log.info('Cancelling running query')
             self.bq_cursor.cancel_query()
 
 
@@ -195,11 +228,11 @@ class BigQueryCreateEmptyTableOperator(BaseOperator):
     You can also create a table without schema.
 
     :param project_id: The project to create the table into. (templated)
-    :type project_id: string
+    :type project_id: str
     :param dataset_id: The dataset to create the table into. (templated)
-    :type dataset_id: string
+    :type dataset_id: str
     :param table_id: The Name of the table to be created. (templated)
-    :type table_id: string
+    :type table_id: str
     :param schema_fields: If set, the schema field list as defined here:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema
 
@@ -212,7 +245,7 @@ class BigQueryCreateEmptyTableOperator(BaseOperator):
     :param gcs_schema_object: Full path to the JSON file containing
         schema (templated). For
         example: ``gs://test-bucket/dir1/dir2/employee_schema.json``
-    :type gcs_schema_object: string
+    :type gcs_schema_object: str
     :param time_partitioning: configure optional time partitioning fields i.e.
         partition by field, type and  expiration as per API specifications.
 
@@ -220,57 +253,60 @@ class BigQueryCreateEmptyTableOperator(BaseOperator):
             https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning
     :type time_partitioning: dict
     :param bigquery_conn_id: Reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param google_cloud_storage_conn_id: Reference to a specific Google
         cloud storage hook.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any. For this to
         work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
-
-    **Example (with schema JSON in GCS)**: ::
-
-        CreateTable = BigQueryCreateEmptyTableOperator(
-            task_id='BigQueryCreateEmptyTableOperator_task',
-            dataset_id='ODS',
-            table_id='Employees',
-            project_id='internal-gcp-project',
-            gcs_schema_object='gs://schema-bucket/employee_schema.json',
-            bigquery_conn_id='airflow-service-account',
-            google_cloud_storage_conn_id='airflow-service-account'
-        )
-
-    **Corresponding Schema file** (``employee_schema.json``): ::
-
-        [
-          {
-            "mode": "NULLABLE",
-            "name": "emp_name",
-            "type": "STRING"
-          },
-          {
-            "mode": "REQUIRED",
-            "name": "salary",
-            "type": "INTEGER"
-          }
-        ]
-
-    **Example (with schema in the DAG)**: ::
-
-        CreateTable = BigQueryCreateEmptyTableOperator(
-            task_id='BigQueryCreateEmptyTableOperator_task',
-            dataset_id='ODS',
-            table_id='Employees',
-            project_id='internal-gcp-project',
-            schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
-                           {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}],
-            bigquery_conn_id='airflow-service-account',
-            google_cloud_storage_conn_id='airflow-service-account'
-        )
+    :type delegate_to: str
+    :param labels: a dictionary containing labels for the table, passed to BigQuery
+
+        **Example (with schema JSON in GCS)**: ::
+
+            CreateTable = BigQueryCreateEmptyTableOperator(
+                task_id='BigQueryCreateEmptyTableOperator_task',
+                dataset_id='ODS',
+                table_id='Employees',
+                project_id='internal-gcp-project',
+                gcs_schema_object='gs://schema-bucket/employee_schema.json',
+                bigquery_conn_id='airflow-service-account',
+                google_cloud_storage_conn_id='airflow-service-account'
+            )
+
+        **Corresponding Schema file** (``employee_schema.json``): ::
+
+            [
+              {
+                "mode": "NULLABLE",
+                "name": "emp_name",
+                "type": "STRING"
+              },
+              {
+                "mode": "REQUIRED",
+                "name": "salary",
+                "type": "INTEGER"
+              }
+            ]
+
+        **Example (with schema in the DAG)**: ::
+
+            CreateTable = BigQueryCreateEmptyTableOperator(
+                task_id='BigQueryCreateEmptyTableOperator_task',
+                dataset_id='ODS',
+                table_id='Employees',
+                project_id='internal-gcp-project',
+                schema_fields=[{"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+                               {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}],
+                bigquery_conn_id='airflow-service-account',
+                google_cloud_storage_conn_id='airflow-service-account'
+            )
+    :type labels: dict
 
     """
-    template_fields = ('dataset_id', 'table_id', 'project_id', 'gcs_schema_object')
+    template_fields = ('dataset_id', 'table_id', 'project_id',
+                       'gcs_schema_object', 'labels')
     ui_color = '#f0eee4'
 
     @apply_defaults
@@ -280,10 +316,11 @@ def __init__(self,
                  project_id=None,
                  schema_fields=None,
                  gcs_schema_object=None,
-                 time_partitioning={},
+                 time_partitioning=None,
                  bigquery_conn_id='bigquery_default',
                  google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
+                 labels=None,
                  *args, **kwargs):
 
         super(BigQueryCreateEmptyTableOperator, self).__init__(*args, **kwargs)
@@ -296,7 +333,8 @@ def __init__(self,
         self.bigquery_conn_id = bigquery_conn_id
         self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
         self.delegate_to = delegate_to
-        self.time_partitioning = time_partitioning
+        self.time_partitioning = {} if time_partitioning is None else time_partitioning
+        self.labels = labels
 
     def execute(self, context):
         bq_hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
@@ -323,7 +361,8 @@ def execute(self, context):
             dataset_id=self.dataset_id,
             table_id=self.table_id,
             schema_fields=schema_fields,
-            time_partitioning=self.time_partitioning
+            time_partitioning=self.time_partitioning,
+            labels=self.labels
         )
 
 
@@ -338,15 +377,15 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
     Google cloud storage must be a JSON file with the schema fields in it.
 
     :param bucket: The bucket to point the external table to. (templated)
-    :type bucket: string
+    :type bucket: str
     :param source_objects: List of Google cloud storage URIs to point
         table to. (templated)
         If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI.
-    :type object: list
-    :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table>
-        BigQuery table to load data into (templated). If <project> is not included,
+    :type source_objects: list
+    :param destination_project_dataset_table: The dotted ``(<project>.)<dataset>.<table>``
+        BigQuery table to load data into (templated). If ``<project>`` is not included,
         project will be the project defined in the connection json.
-    :type destination_project_dataset_table: string
+    :type destination_project_dataset_table: str
     :param schema_fields: If set, the schema field list as defined here:
         https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema
 
@@ -359,26 +398,26 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
     :type schema_fields: list
     :param schema_object: If set, a GCS object path pointing to a .json file that
         contains the schema for the table. (templated)
-    :param schema_object: string
+    :type schema_object: str
     :param source_format: File format of the data.
-    :type source_format: string
+    :type source_format: str
     :param compression: [Optional] The compression type of the data source.
         Possible values include GZIP and NONE.
         The default value is NONE.
         This setting is ignored for Google Cloud Bigtable,
         Google Cloud Datastore backups and Avro formats.
-    :type compression: string
+    :type compression: str
     :param skip_leading_rows: Number of rows to skip when loading from a CSV.
     :type skip_leading_rows: int
     :param field_delimiter: The delimiter to use for the CSV.
-    :type field_delimiter: string
+    :type field_delimiter: str
     :param max_bad_records: The maximum number of bad records that BigQuery can
         ignore when running the job.
     :type max_bad_records: int
     :param quote_character: The value that is used to quote data sections in a CSV file.
-    :type quote_character: string
+    :type quote_character: str
     :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).
-    :type allow_quoted_newlines: boolean
+    :type allow_quoted_newlines: bool
     :param allow_jagged_rows: Accept rows that are missing trailing optional columns.
         The missing values are treated as nulls. If false, records with missing trailing
         columns are treated as bad records, and if there are too many bad records, an
@@ -386,19 +425,21 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
         for other formats.
     :type allow_jagged_rows: bool
     :param bigquery_conn_id: Reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param google_cloud_storage_conn_id: Reference to a specific Google
         cloud storage hook.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any. For this to
         work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param src_fmt_configs: configure optional fields specific to the source format
     :type src_fmt_configs: dict
+    :param labels: a dictionary containing labels for the table, passed to BigQuery
+    :type labels: dict
     """
     template_fields = ('bucket', 'source_objects',
-                       'schema_object', 'destination_project_dataset_table')
+                       'schema_object', 'destination_project_dataset_table', 'labels')
     ui_color = '#f0eee4'
 
     @apply_defaults
@@ -420,6 +461,7 @@ def __init__(self,
                  google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  src_fmt_configs={},
+                 labels=None,
                  *args, **kwargs):
 
         super(BigQueryCreateExternalTableOperator, self).__init__(*args, **kwargs)
@@ -446,6 +488,7 @@ def __init__(self,
         self.delegate_to = delegate_to
 
         self.src_fmt_configs = src_fmt_configs
+        self.labels = labels
 
     def execute(self, context):
         bq_hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
@@ -479,5 +522,121 @@ def execute(self, context):
             quote_character=self.quote_character,
             allow_quoted_newlines=self.allow_quoted_newlines,
             allow_jagged_rows=self.allow_jagged_rows,
-            src_fmt_configs=self.src_fmt_configs
+            src_fmt_configs=self.src_fmt_configs,
+            labels=self.labels
+        )
+
+
+class BigQueryDeleteDatasetOperator(BaseOperator):
+    """
+    This operator deletes an existing dataset from your Project in Big query.
+    https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/delete
+
+    :param project_id: The project id of the dataset.
+    :type project_id: str
+    :param dataset_id: The dataset to be deleted.
+    :type dataset_id: str
+
+    **Example**: ::
+
+        delete_temp_data = BigQueryDeleteDatasetOperator(dataset_id = 'temp-dataset',
+                                                         project_id = 'temp-project',
+                                                         bigquery_conn_id='_my_gcp_conn_',
+                                                         task_id='Deletetemp',
+                                                         dag=dag)
+    """
+
+    template_fields = ('dataset_id', 'project_id')
+    ui_color = '#f00004'
+
+    @apply_defaults
+    def __init__(self,
+                 dataset_id,
+                 project_id=None,
+                 bigquery_conn_id='bigquery_default',
+                 delegate_to=None,
+                 *args, **kwargs):
+        self.dataset_id = dataset_id
+        self.project_id = project_id
+        self.bigquery_conn_id = bigquery_conn_id
+        self.delegate_to = delegate_to
+
+        self.log.info('Dataset id: %s', self.dataset_id)
+        self.log.info('Project id: %s', self.project_id)
+
+        super(BigQueryDeleteDatasetOperator, self).__init__(*args, **kwargs)
+
+    def execute(self, context):
+        bq_hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
+                               delegate_to=self.delegate_to)
+
+        conn = bq_hook.get_conn()
+        cursor = conn.cursor()
+
+        cursor.delete_dataset(
+            project_id=self.project_id,
+            dataset_id=self.dataset_id
         )
+
+
+class BigQueryCreateEmptyDatasetOperator(BaseOperator):
+    """
+    This operator is used to create new dataset for your Project in Big query.
+    https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource
+
+    :param project_id: The name of the project where we want to create the dataset.
+        Don't need to provide, if projectId in dataset_reference.
+    :type project_id: str
+    :param dataset_id: The id of dataset. Don't need to provide,
+        if datasetId in dataset_reference.
+    :type dataset_id: str
+    :param dataset_reference: Dataset reference that could be provided with request body.
+        More info:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource
+    :type dataset_reference: dict
+
+        **Example**: ::
+
+            create_new_dataset = BigQueryCreateEmptyDatasetOperator(
+                                    dataset_id = 'new-dataset',
+                                    project_id = 'my-project',
+                                    dataset_reference = {"friendlyName": "New Dataset"}
+                                    bigquery_conn_id='_my_gcp_conn_',
+                                    task_id='newDatasetCreator',
+                                    dag=dag)
+
+    """
+
+    template_fields = ('dataset_id', 'project_id')
+    ui_color = '#f0eee4'
+
+    @apply_defaults
+    def __init__(self,
+                 dataset_id,
+                 project_id=None,
+                 dataset_reference=None,
+                 bigquery_conn_id='bigquery_default',
+                 delegate_to=None,
+                 *args, **kwargs):
+        self.dataset_id = dataset_id
+        self.project_id = project_id
+        self.bigquery_conn_id = bigquery_conn_id
+        self.dataset_reference = dataset_reference if dataset_reference else {}
+        self.delegate_to = delegate_to
+
+        self.log.info('Dataset id: %s', self.dataset_id)
+        self.log.info('Project id: %s', self.project_id)
+
+        super(BigQueryCreateEmptyDatasetOperator, self).__init__(*args, **kwargs)
+
+    def execute(self, context):
+        bq_hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
+                               delegate_to=self.delegate_to)
+
+        conn = bq_hook.get_conn()
+        cursor = conn.cursor()
+
+        cursor.create_empty_dataset(
+            project_id=self.project_id,
+            dataset_id=self.dataset_id,
+            dataset_reference=self.dataset_reference)
diff --git a/airflow/contrib/operators/bigquery_table_delete_operator.py b/airflow/contrib/operators/bigquery_table_delete_operator.py
index a16107d8c41ec..106afd1e94f0e 100644
--- a/airflow/contrib/operators/bigquery_table_delete_operator.py
+++ b/airflow/contrib/operators/bigquery_table_delete_operator.py
@@ -27,18 +27,18 @@ class BigQueryTableDeleteOperator(BaseOperator):
     Deletes BigQuery tables
 
     :param deletion_dataset_table: A dotted
-        (<project>.|<project>:)<dataset>.<table> that indicates which table
+        ``(<project>.|<project>:)<dataset>.<table>`` that indicates which table
         will be deleted. (templated)
-    :type deletion_dataset_table: string
+    :type deletion_dataset_table: str
     :param bigquery_conn_id: reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param ignore_if_missing: if True, then return success even if the
         requested table does not exist.
-    :type ignore_if_missing: boolean
+    :type ignore_if_missing: bool
     """
     template_fields = ('deletion_dataset_table',)
     ui_color = '#ffd1dc'
diff --git a/airflow/contrib/operators/bigquery_to_bigquery.py b/airflow/contrib/operators/bigquery_to_bigquery.py
index 93a52b31020ce..288731e157de7 100644
--- a/airflow/contrib/operators/bigquery_to_bigquery.py
+++ b/airflow/contrib/operators/bigquery_to_bigquery.py
@@ -31,27 +31,30 @@ class BigQueryToBigQueryOperator(BaseOperator):
         https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy
 
     :param source_project_dataset_tables: One or more
-        dotted (project:|project.)<dataset>.<table> BigQuery tables to use as the
-        source data. If <project> is not included, project will be the
+        dotted ``(project:|project.)<dataset>.<table>`` BigQuery tables to use as the
+        source data. If ``<project>`` is not included, project will be the
         project defined in the connection json. Use a list if there are multiple
         source tables. (templated)
     :type source_project_dataset_tables: list|string
     :param destination_project_dataset_table: The destination BigQuery
-        table. Format is: (project:|project.)<dataset>.<table> (templated)
-    :type destination_project_dataset_table: string
+        table. Format is: ``(project:|project.)<dataset>.<table>`` (templated)
+    :type destination_project_dataset_table: str
     :param write_disposition: The write disposition if the table already exists.
-    :type write_disposition: string
+    :type write_disposition: str
     :param create_disposition: The create disposition if the table doesn't exist.
-    :type create_disposition: string
+    :type create_disposition: str
     :param bigquery_conn_id: reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
+    :param labels: a dictionary containing labels for the job/query,
+        passed to BigQuery
+    :type labels: dict
     """
     template_fields = ('source_project_dataset_tables',
-                       'destination_project_dataset_table')
+                       'destination_project_dataset_table', 'labels')
     template_ext = ('.sql',)
     ui_color = '#e6f0e4'
 
@@ -63,6 +66,7 @@ def __init__(self,
                  create_disposition='CREATE_IF_NEEDED',
                  bigquery_conn_id='bigquery_default',
                  delegate_to=None,
+                 labels=None,
                  *args,
                  **kwargs):
         super(BigQueryToBigQueryOperator, self).__init__(*args, **kwargs)
@@ -72,6 +76,7 @@ def __init__(self,
         self.create_disposition = create_disposition
         self.bigquery_conn_id = bigquery_conn_id
         self.delegate_to = delegate_to
+        self.labels = labels
 
     def execute(self, context):
         self.log.info(
@@ -86,4 +91,5 @@ def execute(self, context):
             self.source_project_dataset_tables,
             self.destination_project_dataset_table,
             self.write_disposition,
-            self.create_disposition)
+            self.create_disposition,
+            self.labels)
diff --git a/airflow/contrib/operators/bigquery_to_gcs.py b/airflow/contrib/operators/bigquery_to_gcs.py
index e2ce93068f433..19b004fdfe2fb 100644
--- a/airflow/contrib/operators/bigquery_to_gcs.py
+++ b/airflow/contrib/operators/bigquery_to_gcs.py
@@ -31,31 +31,35 @@ class BigQueryToCloudStorageOperator(BaseOperator):
         https://cloud.google.com/bigquery/docs/reference/v2/jobs
 
     :param source_project_dataset_table: The dotted
-        (<project>.|<project>:)<dataset>.<table> BigQuery table to use as the source
-        data. If <project> is not included, project will be the project
+        ``(<project>.|<project>:)<dataset>.<table>`` BigQuery table to use as the
+        source data. If ``<project>`` is not included, project will be the project
         defined in the connection json. (templated)
-    :type source_project_dataset_table: string
+    :type source_project_dataset_table: str
     :param destination_cloud_storage_uris: The destination Google Cloud
         Storage URI (e.g. gs://some-bucket/some-file.txt). (templated) Follows
         convention defined here:
         https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
     :type destination_cloud_storage_uris: list
     :param compression: Type of compression to use.
-    :type compression: string
+    :type compression: str
     :param export_format: File format to export.
-    :type field_delimiter: string
+    :type export_format: str
     :param field_delimiter: The delimiter to use when extracting to a CSV.
-    :type field_delimiter: string
+    :type field_delimiter: str
     :param print_header: Whether to print a header for a CSV file extract.
-    :type print_header: boolean
+    :type print_header: bool
     :param bigquery_conn_id: reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
+    :param labels: a dictionary containing labels for the job/query,
+        passed to BigQuery
+    :type labels: dict
     """
-    template_fields = ('source_project_dataset_table', 'destination_cloud_storage_uris')
+    template_fields = ('source_project_dataset_table',
+                       'destination_cloud_storage_uris', 'labels')
     template_ext = ('.sql',)
     ui_color = '#e4e6f0'
 
@@ -69,6 +73,7 @@ def __init__(self,
                  print_header=True,
                  bigquery_conn_id='bigquery_default',
                  delegate_to=None,
+                 labels=None,
                  *args,
                  **kwargs):
         super(BigQueryToCloudStorageOperator, self).__init__(*args, **kwargs)
@@ -80,6 +85,7 @@ def __init__(self,
         self.print_header = print_header
         self.bigquery_conn_id = bigquery_conn_id
         self.delegate_to = delegate_to
+        self.labels = labels
 
     def execute(self, context):
         self.log.info('Executing extract of %s into: %s',
@@ -95,4 +101,5 @@ def execute(self, context):
             self.compression,
             self.export_format,
             self.field_delimiter,
-            self.print_header)
+            self.print_header,
+            self.labels)
diff --git a/airflow/contrib/operators/cassandra_to_gcs.py b/airflow/contrib/operators/cassandra_to_gcs.py
index 211444b96d1b9..6819eca404ebb 100644
--- a/airflow/contrib/operators/cassandra_to_gcs.py
+++ b/airflow/contrib/operators/cassandra_to_gcs.py
@@ -60,18 +60,18 @@ def __init__(self,
                  **kwargs):
         """
         :param cql: The CQL to execute on the Cassandra table.
-        :type cql: string
+        :type cql: str
         :param bucket: The bucket to upload to.
-        :type bucket: string
+        :type bucket: str
         :param filename: The filename to use as the object name when uploading
             to Google cloud storage. A {} should be specified in the filename
             to allow the operator to inject file numbers in cases where the
             file is split due to size.
-        :type filename: string
+        :type filename: str
         :param schema_filename: If set, the filename to use as the object name
             when uploading a .json file containing the BigQuery schema fields
             for the table that was dumped from MySQL.
-        :type schema_filename: string
+        :type schema_filename: str
         :param approx_max_file_size_bytes: This operator supports the ability
             to split large table dumps into multiple files (see notes in the
             filenamed param docs above). Google cloud storage allows for files
@@ -79,14 +79,14 @@ def __init__(self,
             file size of the splits.
         :type approx_max_file_size_bytes: long
         :param cassandra_conn_id: Reference to a specific Cassandra hook.
-        :type cassandra_conn_id: string
+        :type cassandra_conn_id: str
         :param google_cloud_storage_conn_id: Reference to a specific Google
             cloud storage hook.
-        :type google_cloud_storage_conn_id: string
+        :type google_cloud_storage_conn_id: str
         :param delegate_to: The account to impersonate, if any. For this to
             work, the service account making the request must have domain-wide
             delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(CassandraToGoogleCloudStorageOperator, self).__init__(*args, **kwargs)
         self.cql = cql
@@ -266,7 +266,7 @@ def convert_tuple_type(cls, name, value):
         """
         Converts a tuple to RECORD that contains n fields, each will be converted
         to its corresponding data type in bq and will be named 'field_<index>', where
-        index is determined by the order of the tuple elments defined in cassandra.
+        index is determined by the order of the tuple elements defined in cassandra.
         """
         names = ['field_' + str(i) for i in range(len(value))]
         values = [cls.convert_value(name, value) for name, value in zip(names, value)]
@@ -276,7 +276,7 @@ def convert_tuple_type(cls, name, value):
     def convert_map_type(cls, name, value):
         """
         Converts a map to a repeated RECORD that contains two fields: 'key' and 'value',
-        each will be converted to its corresopnding data type in BQ.
+        each will be converted to its corresponding data type in BQ.
         """
         converted_map = []
         for k, v in zip(value.keys(), value.values()):
diff --git a/airflow/contrib/operators/databricks_operator.py b/airflow/contrib/operators/databricks_operator.py
index 7b8d522dba85b..df3f903725ef6 100644
--- a/airflow/contrib/operators/databricks_operator.py
+++ b/airflow/contrib/operators/databricks_operator.py
@@ -24,15 +24,76 @@
 from airflow.exceptions import AirflowException
 from airflow.contrib.hooks.databricks_hook import DatabricksHook
 from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
 
 
 XCOM_RUN_ID_KEY = 'run_id'
 XCOM_RUN_PAGE_URL_KEY = 'run_page_url'
 
 
+def _deep_string_coerce(content, json_path='json'):
+    """
+    Coerces content or all values of content if it is a dict to a string. The
+    function will throw if content contains non-string or non-numeric types.
+
+    The reason why we have this function is because the ``self.json`` field must be a
+    dict with only string values. This is because ``render_template`` will fail
+    for numerical values.
+    """
+    c = _deep_string_coerce
+    if isinstance(content, six.string_types):
+        return content
+    elif isinstance(content, six.integer_types + (float,)):
+        # Databricks can tolerate either numeric or string types in the API backend.
+        return str(content)
+    elif isinstance(content, (list, tuple)):
+        return [c(e, '{0}[{1}]'.format(json_path, i)) for i, e in enumerate(content)]
+    elif isinstance(content, dict):
+        return {k: c(v, '{0}[{1}]'.format(json_path, k))
+                for k, v in list(content.items())}
+    else:
+        param_type = type(content)
+        msg = 'Type {0} used for parameter {1} is not a number or a string' \
+            .format(param_type, json_path)
+        raise AirflowException(msg)
+
+
+def _handle_databricks_operator_execution(operator, hook, log, context):
+    """
+    Handles the Airflow + Databricks lifecycle logic for a Databricks operator
+    :param operator: Databricks operator being handled
+    :param context: Airflow context
+    """
+    if operator.do_xcom_push:
+        context['ti'].xcom_push(key=XCOM_RUN_ID_KEY, value=operator.run_id)
+    log.info('Run submitted with run_id: %s', operator.run_id)
+    run_page_url = hook.get_run_page_url(operator.run_id)
+    if operator.do_xcom_push:
+        context['ti'].xcom_push(key=XCOM_RUN_PAGE_URL_KEY, value=run_page_url)
+
+    log.info('View run status, Spark UI, and logs at %s', run_page_url)
+    while True:
+        run_state = hook.get_run_state(operator.run_id)
+        if run_state.is_terminal:
+            if run_state.is_successful:
+                log.info('%s completed successfully.', operator.task_id)
+                log.info('View run status, Spark UI, and logs at %s', run_page_url)
+                return
+            else:
+                error_message = '{t} failed with terminal state: {s}'.format(
+                    t=operator.task_id,
+                    s=run_state)
+                raise AirflowException(error_message)
+        else:
+            log.info('%s in run state: %s', operator.task_id, run_state)
+            log.info('View run status, Spark UI, and logs at %s', run_page_url)
+            log.info('Sleeping for %s seconds.', operator.polling_period_seconds)
+            time.sleep(operator.polling_period_seconds)
+
+
 class DatabricksSubmitRunOperator(BaseOperator):
     """
-    Submits an Spark job run to Databricks using the
+    Submits a Spark job run to Databricks using the
     `api/2.0/jobs/runs/submit
     <https://docs.databricks.com/api/latest/jobs.html#runs-submit>`_
     API endpoint.
@@ -43,6 +104,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     to call the ``api/2.0/jobs/runs/submit`` endpoint and pass it directly
     to our ``DatabricksSubmitRunOperator`` through the ``json`` parameter.
     For example ::
+
         json = {
           'new_cluster': {
             'spark_version': '2.1.0-db3-scala2.11',
@@ -58,6 +120,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     of the ``DatabricksSubmitRunOperator`` directly. Note that there is exactly
     one named parameter for each top level parameter in the ``runs/submit``
     endpoint. In this method, your code would look like this: ::
+
         new_cluster = {
           'spark_version': '2.1.0-db3-scala2.11',
           'num_workers': 2
@@ -119,7 +182,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     :param existing_cluster_id: ID for existing cluster on which to run this task.
         *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.
         This field will be templated.
-    :type existing_cluster_id: string
+    :type existing_cluster_id: str
     :param libraries: Libraries which this run will use.
         This field will be templated.
 
@@ -130,7 +193,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
         By default this will be set to the Airflow ``task_id``. This ``task_id`` is a
         required parameter of the superclass ``BaseOperator``.
         This field will be templated.
-    :type run_name: string
+    :type run_name: str
     :param timeout_seconds: The timeout for this run. By default a value of 0 is used
         which means to have no timeout.
         This field will be templated.
@@ -139,15 +202,18 @@ class DatabricksSubmitRunOperator(BaseOperator):
         By default and in the common case this will be ``databricks_default``. To use
         token based authentication, provide the key ``token`` in the extra field for the
         connection.
-    :type databricks_conn_id: string
+    :type databricks_conn_id: str
     :param polling_period_seconds: Controls the rate which we poll for the result of
         this run. By default the operator will poll every 30 seconds.
     :type polling_period_seconds: int
     :param databricks_retry_limit: Amount of times retry if the Databricks backend is
         unreachable. Its value must be greater than or equal to 1.
     :type databricks_retry_limit: int
+    :param databricks_retry_delay: Number of seconds to wait between retries (it
+            might be a floating point number).
+    :type databricks_retry_delay: float
     :param do_xcom_push: Whether we should push run_id and run_page_url to xcom.
-    :type do_xcom_push: boolean
+    :type do_xcom_push: bool
     """
     # Used in airflow.models.BaseOperator
     template_fields = ('json',)
@@ -155,6 +221,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     ui_color = '#1CB1C2'
     ui_fgcolor = '#fff'
 
+    @apply_defaults
     def __init__(
             self,
             json=None,
@@ -168,6 +235,7 @@ def __init__(
             databricks_conn_id='databricks_default',
             polling_period_seconds=30,
             databricks_retry_limit=3,
+            databricks_retry_delay=1,
             do_xcom_push=False,
             **kwargs):
         """
@@ -178,6 +246,7 @@ def __init__(
         self.databricks_conn_id = databricks_conn_id
         self.polling_period_seconds = polling_period_seconds
         self.databricks_retry_limit = databricks_retry_limit
+        self.databricks_retry_delay = databricks_retry_delay
         if spark_jar_task is not None:
             self.json['spark_jar_task'] = spark_jar_task
         if notebook_task is not None:
@@ -195,72 +264,217 @@ def __init__(
         if 'run_name' not in self.json:
             self.json['run_name'] = run_name or kwargs['task_id']
 
-        self.json = self._deep_string_coerce(self.json)
+        self.json = _deep_string_coerce(self.json)
         # This variable will be used in case our task gets killed.
         self.run_id = None
         self.do_xcom_push = do_xcom_push
 
-    def _deep_string_coerce(self, content, json_path='json'):
-        """
-        Coerces content or all values of content if it is a dict to a string. The
-        function will throw if content contains non-string or non-numeric types.
+    def get_hook(self):
+        return DatabricksHook(
+            self.databricks_conn_id,
+            retry_limit=self.databricks_retry_limit,
+            retry_delay=self.databricks_retry_delay)
+
+    def execute(self, context):
+        hook = self.get_hook()
+        self.run_id = hook.submit_run(self.json)
+        _handle_databricks_operator_execution(self, hook, self.log, context)
+
+    def on_kill(self):
+        hook = self.get_hook()
+        hook.cancel_run(self.run_id)
+        self.log.info(
+            'Task: %s with run_id: %s was requested to be cancelled.',
+            self.task_id, self.run_id
+        )
+
+
+class DatabricksRunNowOperator(BaseOperator):
+    """
+    Runs an existing Spark job run to Databricks using the
+    `api/2.0/jobs/run-now
+    <https://docs.databricks.com/api/latest/jobs.html#run-now>`_
+    API endpoint.
+
+    There are two ways to instantiate this operator.
+
+    In the first way, you can take the JSON payload that you typically use
+    to call the ``api/2.0/jobs/run-now`` endpoint and pass it directly
+    to our ``DatabricksRunNowOperator`` through the ``json`` parameter.
+    For example ::
+
+        json = {
+          "job_id": 42,
+          "notebook_params": {
+            "dry-run": "true",
+            "oldest-time-to-consider": "1457570074236"
+          }
+        }
+
+        notebook_run = DatabricksRunNowOperator(task_id='notebook_run', json=json)
+
+    Another way to accomplish the same thing is to use the named parameters
+    of the ``DatabricksRunNowOperator`` directly. Note that there is exactly
+    one named parameter for each top level parameter in the ``run-now``
+    endpoint. In this method, your code would look like this: ::
+
+        job_id=42
+
+        notebook_params = {
+            "dry-run": "true",
+            "oldest-time-to-consider": "1457570074236"
+        }
+
+        python_params = ["douglas adams", "42"]
+
+        spark_submit_params = ["--class", "org.apache.spark.examples.SparkPi"]
+
+        notebook_run = DatabricksRunNowOperator(
+            job_id=job_id,
+            notebook_params=notebook_params,
+            python_params=python_params,
+            spark_submit_params=spark_submit_params
+        )
+
+    In the case where both the json parameter **AND** the named parameters
+    are provided, they will be merged together. If there are conflicts during the merge,
+    the named parameters will take precedence and override the top level ``json`` keys.
+
+    Currently the named parameters that ``DatabricksRunNowOperator`` supports are
+        - ``job_id``
+        - ``json``
+        - ``notebook_params``
+        - ``python_params``
+        - ``spark_submit_params``
+
+
+    :param job_id: the job_id of the existing Databricks job.
+        This field will be templated.
+
+        .. seealso::
+            https://docs.databricks.com/api/latest/jobs.html#run-now
+    :type job_id: str
+    :param json: A JSON object containing API parameters which will be passed
+        directly to the ``api/2.0/jobs/run-now`` endpoint. The other named parameters
+        (i.e. ``notebook_params``, ``spark_submit_params``..) to this operator will
+        be merged with this json dictionary if they are provided.
+        If there are conflicts during the merge, the named parameters will
+        take precedence and override the top level json keys. (templated)
+
+        .. seealso::
+            For more information about templating see :ref:`jinja-templating`.
+            https://docs.databricks.com/api/latest/jobs.html#run-now
+    :type json: dict
+    :param notebook_params: A dict from keys to values for jobs with notebook task,
+        e.g. "notebook_params": {"name": "john doe", "age":  "35"}.
+        The map is passed to the notebook and will be accessible through the
+        dbutils.widgets.get function. See Widgets for more information.
+        If not specified upon run-now, the triggered run will use the
+        job’s base parameters. notebook_params cannot be
+        specified in conjunction with jar_params. The json representation
+        of this field (i.e. {"notebook_params":{"name":"john doe","age":"35"}})
+        cannot exceed 10,000 bytes.
+        This field will be templated.
+
+        .. seealso::
+            https://docs.databricks.com/user-guide/notebooks/widgets.html
+    :type notebook_params: dict
+    :param python_params: A list of parameters for jobs with python tasks,
+        e.g. "python_params": ["john doe", "35"].
+        The parameters will be passed to python file as command line parameters.
+        If specified upon run-now, it would overwrite the parameters specified in
+        job setting.
+        The json representation of this field (i.e. {"python_params":["john doe","35"]})
+        cannot exceed 10,000 bytes.
+        This field will be templated.
+
+        .. seealso::
+            https://docs.databricks.com/api/latest/jobs.html#run-now
+    :type python_params: list[str]
+    :param spark_submit_params: A list of parameters for jobs with spark submit task,
+        e.g. "spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"].
+        The parameters will be passed to spark-submit script as command line parameters.
+        If specified upon run-now, it would overwrite the parameters specified
+        in job setting.
+        The json representation of this field cannot exceed 10,000 bytes.
+        This field will be templated.
+
+        .. seealso::
+            https://docs.databricks.com/api/latest/jobs.html#run-now
+    :type spark_submit_params: list[str]
+    :param timeout_seconds: The timeout for this run. By default a value of 0 is used
+        which means to have no timeout.
+        This field will be templated.
+    :type timeout_seconds: int32
+    :param databricks_conn_id: The name of the Airflow connection to use.
+        By default and in the common case this will be ``databricks_default``. To use
+        token based authentication, provide the key ``token`` in the extra field for the
+        connection.
+    :type databricks_conn_id: str
+    :param polling_period_seconds: Controls the rate which we poll for the result of
+        this run. By default the operator will poll every 30 seconds.
+    :type polling_period_seconds: int
+    :param databricks_retry_limit: Amount of times retry if the Databricks backend is
+        unreachable. Its value must be greater than or equal to 1.
+    :type databricks_retry_limit: int
+    :param do_xcom_push: Whether we should push run_id and run_page_url to xcom.
+    :type do_xcom_push: bool
+    """
+    # Used in airflow.models.BaseOperator
+    template_fields = ('json',)
+    # Databricks brand color (blue) under white text
+    ui_color = '#1CB1C2'
+    ui_fgcolor = '#fff'
+
+    @apply_defaults
+    def __init__(
+            self,
+            job_id,
+            json=None,
+            notebook_params=None,
+            python_params=None,
+            spark_submit_params=None,
+            databricks_conn_id='databricks_default',
+            polling_period_seconds=30,
+            databricks_retry_limit=3,
+            databricks_retry_delay=1,
+            do_xcom_push=False,
+            **kwargs):
 
-        The reason why we have this function is because the ``self.json`` field must be a
-         dict with only string values. This is because ``render_template`` will fail
-        for numerical values.
         """
-        c = self._deep_string_coerce
-        if isinstance(content, six.string_types):
-            return content
-        elif isinstance(content, six.integer_types + (float,)):
-            # Databricks can tolerate either numeric or string types in the API backend.
-            return str(content)
-        elif isinstance(content, (list, tuple)):
-            return [c(e, '{0}[{1}]'.format(json_path, i)) for i, e in enumerate(content)]
-        elif isinstance(content, dict):
-            return {k: c(v, '{0}[{1}]'.format(json_path, k))
-                    for k, v in list(content.items())}
-        else:
-            param_type = type(content)
-            msg = 'Type {0} used for parameter {1} is not a number or a string'\
-                .format(param_type, json_path)
-            raise AirflowException(msg)
+        Creates a new ``DatabricksRunNowOperator``.
+        """
+        super(DatabricksRunNowOperator, self).__init__(**kwargs)
+        self.json = json or {}
+        self.databricks_conn_id = databricks_conn_id
+        self.polling_period_seconds = polling_period_seconds
+        self.databricks_retry_limit = databricks_retry_limit
+        self.databricks_retry_delay = databricks_retry_delay
 
-    def _log_run_page_url(self, url):
-        self.log.info('View run status, Spark UI, and logs at %s', url)
+        if job_id is not None:
+            self.json['job_id'] = job_id
+        if notebook_params is not None:
+            self.json['notebook_params'] = notebook_params
+        if python_params is not None:
+            self.json['python_params'] = python_params
+        if spark_submit_params is not None:
+            self.json['spark_submit_params'] = spark_submit_params
+
+        self.json = _deep_string_coerce(self.json)
+        # This variable will be used in case our task gets killed.
+        self.run_id = None
+        self.do_xcom_push = do_xcom_push
 
     def get_hook(self):
         return DatabricksHook(
             self.databricks_conn_id,
-            retry_limit=self.databricks_retry_limit)
+            retry_limit=self.databricks_retry_limit,
+            retry_delay=self.databricks_retry_delay)
 
     def execute(self, context):
         hook = self.get_hook()
-        self.run_id = hook.submit_run(self.json)
-        if self.do_xcom_push:
-            context['ti'].xcom_push(key=XCOM_RUN_ID_KEY, value=self.run_id)
-        self.log.info('Run submitted with run_id: %s', self.run_id)
-        run_page_url = hook.get_run_page_url(self.run_id)
-        if self.do_xcom_push:
-            context['ti'].xcom_push(key=XCOM_RUN_PAGE_URL_KEY, value=run_page_url)
-        self._log_run_page_url(run_page_url)
-        while True:
-            run_state = hook.get_run_state(self.run_id)
-            if run_state.is_terminal:
-                if run_state.is_successful:
-                    self.log.info('%s completed successfully.', self.task_id)
-                    self._log_run_page_url(run_page_url)
-                    return
-                else:
-                    error_message = '{t} failed with terminal state: {s}'.format(
-                        t=self.task_id,
-                        s=run_state)
-                    raise AirflowException(error_message)
-            else:
-                self.log.info('%s in run state: %s', self.task_id, run_state)
-                self._log_run_page_url(run_page_url)
-                self.log.info('Sleeping for %s seconds.', self.polling_period_seconds)
-                time.sleep(self.polling_period_seconds)
+        self.run_id = hook.run_now(self.json)
+        _handle_databricks_operator_execution(self, hook, self.log, context)
 
     def on_kill(self):
         hook = self.get_hook()
diff --git a/airflow/contrib/operators/dataflow_operator.py b/airflow/contrib/operators/dataflow_operator.py
index e3c8c1fff1572..e880642f6067c 100644
--- a/airflow/contrib/operators/dataflow_operator.py
+++ b/airflow/contrib/operators/dataflow_operator.py
@@ -16,7 +16,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+import os
 import re
 import uuid
 import copy
@@ -33,6 +33,77 @@ class DataFlowJavaOperator(BaseOperator):
     Start a Java Cloud DataFlow batch job. The parameters of the operation
     will be passed to the job.
 
+    **Example**: ::
+
+        default_args = {
+            'owner': 'airflow',
+            'depends_on_past': False,
+            'start_date':
+                (2016, 8, 1),
+            'email': ['alex@vanboxel.be'],
+            'email_on_failure': False,
+            'email_on_retry': False,
+            'retries': 1,
+            'retry_delay': timedelta(minutes=30),
+            'dataflow_default_options': {
+                'project': 'my-gcp-project',
+                'zone': 'us-central1-f',
+                'stagingLocation': 'gs://bucket/tmp/dataflow/staging/',
+            }
+        }
+
+        dag = DAG('test-dag', default_args=default_args)
+
+        task = DataFlowJavaOperator(
+            gcp_conn_id='gcp_default',
+            task_id='normalize-cal',
+            jar='{{var.value.gcp_dataflow_base}}pipeline-ingress-cal-normalize-1.0.jar',
+            options={
+                'autoscalingAlgorithm': 'BASIC',
+                'maxNumWorkers': '50',
+                'start': '{{ds}}',
+                'partitionType': 'DAY'
+
+            },
+            dag=dag)
+
+    .. seealso::
+        For more detail on job submission have a look at the reference:
+        https://cloud.google.com/dataflow/pipelines/specifying-exec-params
+
+    :param jar: The reference to a self executing DataFlow jar (templated).
+    :type jar: str
+    :param job_name: The 'jobName' to use when executing the DataFlow job
+        (templated). This ends up being set in the pipeline options, so any entry
+        with key ``'jobName'`` in ``options`` will be overwritten.
+    :type job_name: str
+    :param dataflow_default_options: Map of default job options.
+    :type dataflow_default_options: dict
+    :param options: Map of job specific options.
+    :type options: dict
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide delegation enabled.
+    :type delegate_to: str
+    :param poll_sleep: The time in seconds to sleep between polling Google
+        Cloud Platform for the dataflow job status while the job is in the
+        JOB_STATE_RUNNING state.
+    :type poll_sleep: int
+    :param job_class: The name of the dataflow job class to be executed, it
+        is often not the main class configured in the dataflow jar file.
+    :type job_class: str
+
+    ``jar``, ``options``, and ``job_name`` are templated so you can use variables in them.
+
+    Note that both
+    ``dataflow_default_options`` and ``options`` will be merged to specify pipeline
+    execution parameter, and ``dataflow_default_options`` is expected to save
+    high-level options, for instances, project and zone information, which
+    apply to all dataflow operators in the DAG.
+
     It's a good practice to define dataflow_* parameters in the default_args of the dag
     like the project, zone and staging location.
 
@@ -53,7 +124,7 @@ class DataFlowJavaOperator(BaseOperator):
 
     .. code-block:: python
 
-       t1 = DataFlowOperation(
+       t1 = DataFlowJavaOperator(
            task_id='datapflow_example',
            jar='{{var.value.gcp_dataflow_base}}pipeline/build/libs/pipeline-example-1.0.jar',
            options={
@@ -66,15 +137,15 @@ class DataFlowJavaOperator(BaseOperator):
            gcp_conn_id='gcp-airflow-service-account',
            dag=my-dag)
 
-    Both ``jar`` and ``options`` are templated so you can use variables in them.
     """
-    template_fields = ['options', 'jar']
+    template_fields = ['options', 'jar', 'job_name']
     ui_color = '#0273d4'
 
     @apply_defaults
     def __init__(
             self,
             jar,
+            job_name='{{task.task_id}}',
             dataflow_default_options=None,
             options=None,
             gcp_conn_id='google_cloud_default',
@@ -83,39 +154,6 @@ def __init__(
             job_class=None,
             *args,
             **kwargs):
-        """
-        Create a new DataFlowJavaOperator. Note that both
-        dataflow_default_options and options will be merged to specify pipeline
-        execution parameter, and dataflow_default_options is expected to save
-        high-level options, for instances, project and zone information, which
-        apply to all dataflow operators in the DAG.
-
-
-        .. seealso::
-            For more detail on job submission have a look at the reference:
-            https://cloud.google.com/dataflow/pipelines/specifying-exec-params
-
-        :param jar: The reference to a self executing DataFlow jar.
-        :type jar: string
-        :param dataflow_default_options: Map of default job options.
-        :type dataflow_default_options: dict
-        :param options: Map of job specific options.
-        :type options: dict
-        :param gcp_conn_id: The connection ID to use connecting to Google Cloud
-        Platform.
-        :type gcp_conn_id: string
-        :param delegate_to: The account to impersonate, if any.
-            For this to work, the service account making the request must have
-            domain-wide delegation enabled.
-        :type delegate_to: string
-        :param poll_sleep: The time in seconds to sleep between polling Google
-            Cloud Platform for the dataflow job status while the job is in the
-            JOB_STATE_RUNNING state.
-        :type poll_sleep: int
-        :param job_class: The name of the dataflow job class to be executued, it
-        is often not the main class configured in the dataflow jar file.
-        :type job_class: string
-        """
         super(DataFlowJavaOperator, self).__init__(*args, **kwargs)
 
         dataflow_default_options = dataflow_default_options or {}
@@ -125,6 +163,7 @@ def __init__(
         self.gcp_conn_id = gcp_conn_id
         self.delegate_to = delegate_to
         self.jar = jar
+        self.job_name = job_name
         self.dataflow_default_options = dataflow_default_options
         self.options = options
         self.poll_sleep = poll_sleep
@@ -141,7 +180,7 @@ def execute(self, context):
         dataflow_options = copy.copy(self.dataflow_default_options)
         dataflow_options.update(self.options)
 
-        hook.start_java_dataflow(self.task_id, dataflow_options,
+        hook.start_java_dataflow(self.job_name, dataflow_options,
                                  self.jar, self.job_class)
 
 
@@ -149,6 +188,27 @@ class DataflowTemplateOperator(BaseOperator):
     """
     Start a Templated Cloud DataFlow batch job. The parameters of the operation
     will be passed to the job.
+
+    :param template: The reference to the DataFlow template.
+    :type template: str
+    :param job_name: The 'jobName' to use when executing the DataFlow template
+        (templated).
+    :param dataflow_default_options: Map of default job environment options.
+    :type dataflow_default_options: dict
+    :param parameters: Map of job specific parameters for the template.
+    :type parameters: dict
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide delegation enabled.
+    :type delegate_to: str
+    :param poll_sleep: The time in seconds to sleep between polling Google
+        Cloud Platform for the dataflow job status while the job is in the
+        JOB_STATE_RUNNING state.
+    :type poll_sleep: int
+
     It's a good practice to define dataflow_* parameters in the default_args of the dag
     like the project, zone and staging location.
 
@@ -160,9 +220,10 @@ class DataflowTemplateOperator(BaseOperator):
 
        default_args = {
            'dataflow_default_options': {
-               'project': 'my-gcp-project'
+               'project': 'my-gcp-project',
+               'region': 'europe-west1',
                'zone': 'europe-west1-d',
-               'tempLocation': 'gs://my-staging-bucket/staging/'
+               'tempLocation': 'gs://my-staging-bucket/staging/',
                }
            }
        }
@@ -183,16 +244,27 @@ class DataflowTemplateOperator(BaseOperator):
            gcp_conn_id='gcp-airflow-service-account',
            dag=my-dag)
 
-    ``template``, ``dataflow_default_options`` and ``parameters`` are templated so you can
-    use variables in them.
+    ``template``, ``dataflow_default_options``, ``parameters``, and ``job_name`` are
+    templated so you can use variables in them.
+
+    Note that ``dataflow_default_options`` is expected to save high-level options
+    for project information, which apply to all dataflow operators in the DAG.
+
+        .. seealso::
+            https://cloud.google.com/dataflow/docs/reference/rest/v1b3
+            /LaunchTemplateParameters
+            https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment
+            For more detail on job template execution have a look at the reference:
+            https://cloud.google.com/dataflow/docs/templates/executing-templates
     """
-    template_fields = ['parameters', 'dataflow_default_options', 'template']
+    template_fields = ['parameters', 'dataflow_default_options', 'template', 'job_name']
     ui_color = '#0273d4'
 
     @apply_defaults
     def __init__(
             self,
             template,
+            job_name='{{task.task_id}}',
             dataflow_default_options=None,
             parameters=None,
             gcp_conn_id='google_cloud_default',
@@ -200,36 +272,6 @@ def __init__(
             poll_sleep=10,
             *args,
             **kwargs):
-        """
-        Create a new DataflowTemplateOperator. Note that
-        dataflow_default_options is expected to save high-level options
-        for project information, which apply to all dataflow operators in the DAG.
-
-        .. seealso::
-            https://cloud.google.com/dataflow/docs/reference/rest/v1b3
-            /LaunchTemplateParameters
-            https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment
-            For more detail on job template execution have a look at the reference:
-            https://cloud.google.com/dataflow/docs/templates/executing-templates
-
-        :param template: The reference to the DataFlow template.
-        :type template: string
-        :param dataflow_default_options: Map of default job environment options.
-        :type dataflow_default_options: dict
-        :param parameters: Map of job specific parameters for the template.
-        :type parameters: dict
-        :param gcp_conn_id: The connection ID to use connecting to Google Cloud
-        Platform.
-        :type gcp_conn_id: string
-        :param delegate_to: The account to impersonate, if any.
-            For this to work, the service account making the request must have
-            domain-wide delegation enabled.
-        :type delegate_to: string
-        :param poll_sleep: The time in seconds to sleep between polling Google
-            Cloud Platform for the dataflow job status while the job is in the
-            JOB_STATE_RUNNING state.
-        :type poll_sleep: int
-        """
         super(DataflowTemplateOperator, self).__init__(*args, **kwargs)
 
         dataflow_default_options = dataflow_default_options or {}
@@ -240,6 +282,7 @@ def __init__(
         self.dataflow_default_options = dataflow_default_options
         self.poll_sleep = poll_sleep
         self.template = template
+        self.job_name = job_name
         self.parameters = parameters
 
     def execute(self, context):
@@ -247,18 +290,54 @@ def execute(self, context):
                             delegate_to=self.delegate_to,
                             poll_sleep=self.poll_sleep)
 
-        hook.start_template_dataflow(self.task_id, self.dataflow_default_options,
+        hook.start_template_dataflow(self.job_name, self.dataflow_default_options,
                                      self.parameters, self.template)
 
 
 class DataFlowPythonOperator(BaseOperator):
+    """
+    Launching Cloud Dataflow jobs written in python. Note that both
+    dataflow_default_options and options will be merged to specify pipeline
+    execution parameter, and dataflow_default_options is expected to save
+    high-level options, for instances, project and zone information, which
+    apply to all dataflow operators in the DAG.
 
-    template_fields = ['options', 'dataflow_default_options']
+    .. seealso::
+        For more detail on job submission have a look at the reference:
+        https://cloud.google.com/dataflow/pipelines/specifying-exec-params
+
+    :param py_file: Reference to the python dataflow pipeline file.py, e.g.,
+        /some/local/file/path/to/your/python/pipeline/file.
+    :type py_file: str
+    :param job_name: The 'job_name' to use when executing the DataFlow job
+        (templated). This ends up being set in the pipeline options, so any entry
+        with key ``'jobName'`` or ``'job_name'`` in ``options`` will be overwritten.
+    :type job_name: str
+    :param py_options: Additional python options, e.g., ["-m", "-v"].
+    :type pyt_options: list[str]
+    :param dataflow_default_options: Map of default job options.
+    :type dataflow_default_options: dict
+    :param options: Map of job specific options.
+    :type options: dict
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide  delegation enabled.
+    :type delegate_to: str
+    :param poll_sleep: The time in seconds to sleep between polling Google
+        Cloud Platform for the dataflow job status while the job is in the
+        JOB_STATE_RUNNING state.
+    :type poll_sleep: int
+    """
+    template_fields = ['options', 'dataflow_default_options', 'job_name']
 
     @apply_defaults
     def __init__(
             self,
             py_file,
+            job_name='{{task.task_id}}',
             py_options=None,
             dataflow_default_options=None,
             options=None,
@@ -267,41 +346,11 @@ def __init__(
             poll_sleep=10,
             *args,
             **kwargs):
-        """
-        Create a new DataFlowPythonOperator. Note that both
-        dataflow_default_options and options will be merged to specify pipeline
-        execution parameter, and dataflow_default_options is expected to save
-        high-level options, for instances, project and zone information, which
-        apply to all dataflow operators in the DAG.
 
-        .. seealso::
-            For more detail on job submission have a look at the reference:
-            https://cloud.google.com/dataflow/pipelines/specifying-exec-params
-
-        :param py_file: Reference to the python dataflow pipleline file.py, e.g.,
-            /some/local/file/path/to/your/python/pipeline/file.
-        :type py_file: string
-        :param py_options: Additional python options.
-        :type pyt_options: list of strings, e.g., ["-m", "-v"].
-        :param dataflow_default_options: Map of default job options.
-        :type dataflow_default_options: dict
-        :param options: Map of job specific options.
-        :type options: dict
-        :param gcp_conn_id: The connection ID to use connecting to Google Cloud
-            Platform.
-        :type gcp_conn_id: string
-        :param delegate_to: The account to impersonate, if any.
-            For this to work, the service account making the request must have
-            domain-wide  delegation enabled.
-        :type delegate_to: string
-        :param poll_sleep: The time in seconds to sleep between polling Google
-            Cloud Platform for the dataflow job status while the job is in the
-            JOB_STATE_RUNNING state.
-        :type poll_sleep: int
-        """
         super(DataFlowPythonOperator, self).__init__(*args, **kwargs)
 
         self.py_file = py_file
+        self.job_name = job_name
         self.py_options = py_options or []
         self.dataflow_default_options = dataflow_default_options or {}
         self.options = options or {}
@@ -327,11 +376,11 @@ def execute(self, context):
         formatted_options = {camel_to_snake(key): dataflow_options[key]
                              for key in dataflow_options}
         hook.start_python_dataflow(
-            self.task_id, formatted_options,
+            self.job_name, formatted_options,
             self.py_file, self.py_options)
 
 
-class GoogleCloudBucketHelper():
+class GoogleCloudBucketHelper(object):
     """GoogleCloudStorageHook helper class to download GCS object."""
     GCS_PREFIX_LENGTH = 5
 
@@ -348,9 +397,9 @@ def google_cloud_to_local(self, file_name):
         will be returned immediately.
 
         :param file_name: The full path of input file.
-        :type file_name: string
+        :type file_name: str
         :return: The full path of local file.
-        :type: string
+        :rtype: str
         """
         if not file_name.startswith('gs://'):
             return file_name
@@ -358,19 +407,19 @@ def google_cloud_to_local(self, file_name):
         # Extracts bucket_id and object_id by first removing 'gs://' prefix and
         # then split the remaining by path delimiter '/'.
         path_components = file_name[self.GCS_PREFIX_LENGTH:].split('/')
-        if path_components < 2:
+        if len(path_components) < 2:
             raise Exception(
-                'Invalid Google Cloud Storage (GCS) object path: {}.'
+                'Invalid Google Cloud Storage (GCS) object path: {}'
                 .format(file_name))
 
         bucket_id = path_components[0]
         object_id = '/'.join(path_components[1:])
-        local_file = '/tmp/dataflow{}-{}'.format(str(uuid.uuid1())[:8],
+        local_file = '/tmp/dataflow{}-{}'.format(str(uuid.uuid4())[:8],
                                                  path_components[-1])
-        file_size = self._gcs_hook.download(bucket_id, object_id, local_file)
+        self._gcs_hook.download(bucket_id, object_id, local_file)
 
-        if file_size > 0:
+        if os.stat(local_file).st_size > 0:
             return local_file
         raise Exception(
-            'Failed to download Google Cloud Storage GCS object: {}'
+            'Failed to download Google Cloud Storage (GCS) object: {}'
             .format(file_name))
diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py
index 5d59f7fb6eb58..f525db96836e6 100644
--- a/airflow/contrib/operators/dataproc_operator.py
+++ b/airflow/contrib/operators/dataproc_operator.py
@@ -48,37 +48,50 @@ class DataprocClusterCreateOperator(BaseOperator):
     parameters detailed in the link are available as a parameter to this operator.
 
     :param cluster_name: The name of the DataProc cluster to create. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param project_id: The ID of the google cloud project in which
         to create the cluster. (templated)
-    :type project_id: string
-    :param num_workers: The # of workers to spin up
+    :type project_id: str
+    :param num_workers: The # of workers to spin up. If set to zero will
+        spin up cluster in a single node mode
     :type num_workers: int
     :param storage_bucket: The storage bucket to use, setting to None lets dataproc
         generate a custom one for you
-    :type storage_bucket: string
+    :type storage_bucket: str
     :param init_actions_uris: List of GCS uri's containing
         dataproc initialization scripts
-    :type init_actions_uris: list[string]
+    :type init_actions_uris: list[str]
     :param init_action_timeout: Amount of time executable scripts in
         init_actions_uris has to complete
-    :type init_action_timeout: string
+    :type init_action_timeout: str
     :param metadata: dict of key-value google compute engine metadata entries
         to add to all instances
     :type metadata: dict
     :param image_version: the version of software inside the Dataproc cluster
-    :type image_version: string
+    :type image_version: str
+    :param custom_image: custom Dataproc image for more info see
+        https://cloud.google.com/dataproc/docs/guides/dataproc-images
+    :type: custom_image: str
     :param properties: dict of properties to set on
         config files (e.g. spark-defaults.conf), see
-        https://cloud.google.com/dataproc/docs/reference/rest/v1/ \
-        projects.regions.clusters#SoftwareConfig
+        https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters#SoftwareConfig
     :type properties: dict
     :param master_machine_type: Compute engine machine type to use for the master node
-    :type master_machine_type: string
+    :type master_machine_type: str
+    :param master_disk_type: Type of the boot disk for the master node
+        (default is ``pd-standard``).
+        Valid values: ``pd-ssd`` (Persistent Disk Solid State Drive) or
+        ``pd-standard`` (Persistent Disk Hard Disk Drive).
+    :type master_disk_type: str
     :param master_disk_size: Disk size for the master node
     :type master_disk_size: int
     :param worker_machine_type: Compute engine machine type to use for the worker nodes
-    :type worker_machine_type: string
+    :type worker_machine_type: str
+    :param worker_disk_type: Type of the boot disk for the worker node
+        (default is ``pd-standard``).
+        Valid values: ``pd-ssd`` (Persistent Disk Solid State Drive) or
+        ``pd-standard`` (Persistent Disk Hard Disk Drive).
+    :type worker_disk_type: str
     :param worker_disk_size: Disk size for the worker nodes
     :type worker_disk_size: int
     :param num_preemptible_workers: The # of preemptible worker nodes to spin up
@@ -86,30 +99,31 @@ class DataprocClusterCreateOperator(BaseOperator):
     :param labels: dict of labels to add to the cluster
     :type labels: dict
     :param zone: The zone where the cluster will be located. (templated)
-    :type zone: string
+    :type zone: str
     :param network_uri: The network uri to be used for machine communication, cannot be
         specified with subnetwork_uri
-    :type network_uri: string
+    :type network_uri: str
     :param subnetwork_uri: The subnetwork uri to be used for machine communication,
         cannot be specified with network_uri
-    :type subnetwork_uri: string
+    :type subnetwork_uri: str
     :param internal_ip_only: If true, all instances in the cluster will only
         have internal IP addresses. This can only be enabled for subnetwork
         enabled networks
     :type internal_ip_only: bool
     :param tags: The GCE tags to add to all instances
-    :type tags: list[string]
+    :type tags: list[str]
     :param region: leave as 'global', might become relevant in the future. (templated)
+    :type region: str
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param service_account: The service account of the dataproc instances.
-    :type service_account: string
+    :type service_account: str
     :param service_account_scopes: The URIs of service account scopes to be included.
-    :type service_account_scopes: list[string]
+    :type service_account_scopes: list[str]
     :param idle_delete_ttl: The longest duration that cluster would keep alive while
         staying idle. Passing this threshold will cause cluster to be auto-deleted.
         A duration in seconds.
@@ -138,11 +152,14 @@ def __init__(self,
                  init_actions_uris=None,
                  init_action_timeout="10m",
                  metadata=None,
+                 custom_image=None,
                  image_version=None,
                  properties=None,
                  master_machine_type='n1-standard-4',
+                 master_disk_type='pd-standard',
                  master_disk_size=500,
                  worker_machine_type='n1-standard-4',
+                 worker_disk_type='pd-standard',
                  worker_disk_size=500,
                  num_preemptible_workers=0,
                  labels=None,
@@ -168,11 +185,14 @@ def __init__(self,
         self.init_actions_uris = init_actions_uris
         self.init_action_timeout = init_action_timeout
         self.metadata = metadata
+        self.custom_image = custom_image
         self.image_version = image_version
-        self.properties = properties
+        self.properties = properties or dict()
         self.master_machine_type = master_machine_type
+        self.master_disk_type = master_disk_type
         self.master_disk_size = master_disk_size
         self.worker_machine_type = worker_machine_type
+        self.worker_disk_type = worker_disk_type
         self.worker_disk_size = worker_disk_size
         self.labels = labels
         self.zone = zone
@@ -186,6 +206,19 @@ def __init__(self,
         self.idle_delete_ttl = idle_delete_ttl
         self.auto_delete_time = auto_delete_time
         self.auto_delete_ttl = auto_delete_ttl
+        self.single_node = num_workers == 0
+
+        assert not (self.custom_image and self.image_version), \
+            "custom_image and image_version can't be both set"
+
+        assert (
+            not self.single_node or (
+                self.single_node and self.num_preemptible_workers == 0
+            )
+        ), "num_workers == 0 means single node mode - no preemptibles allowed"
+
+        assert not (self.custom_image and self.image_version), \
+            "custom_image and image_version can't be both set"
 
     def _get_cluster_list_for_project(self, service):
         result = service.projects().regions().clusters().list(
@@ -272,6 +305,7 @@ def _build_cluster_data(self):
                     'numInstances': 1,
                     'machineTypeUri': master_type_uri,
                     'diskConfig': {
+                        'bootDiskType': self.master_disk_type,
                         'bootDiskSizeGb': self.master_disk_size
                     }
                 },
@@ -279,6 +313,7 @@ def _build_cluster_data(self):
                     'numInstances': self.num_workers,
                     'machineTypeUri': worker_type_uri,
                     'diskConfig': {
+                        'bootDiskType': self.worker_disk_type,
                         'bootDiskSizeGb': self.worker_disk_size
                     }
                 },
@@ -292,6 +327,7 @@ def _build_cluster_data(self):
                 'numInstances': self.num_preemptible_workers,
                 'machineTypeUri': worker_type_uri,
                 'diskConfig': {
+                    'bootDiskType': self.worker_disk_type,
                     'bootDiskSizeGb': self.worker_disk_size
                 },
                 'isPreemptible': True
@@ -321,6 +357,17 @@ def _build_cluster_data(self):
             cluster_data['config']['gceClusterConfig']['tags'] = self.tags
         if self.image_version:
             cluster_data['config']['softwareConfig']['imageVersion'] = self.image_version
+        elif self.custom_image:
+            custom_image_url = 'https://www.googleapis.com/compute/beta/projects/' \
+                               '{}/global/images/{}'.format(self.project_id,
+                                                            self.custom_image)
+            cluster_data['config']['masterConfig']['imageUri'] = custom_image_url
+            if not self.single_node:
+                cluster_data['config']['workerConfig']['imageUri'] = custom_image_url
+
+        if self.single_node:
+            self.properties["dataproc:dataproc.allow.zero.workers"] = "true"
+
         if self.properties:
             cluster_data['config']['softwareConfig']['properties'] = self.properties
         if self.idle_delete_ttl:
@@ -395,39 +442,39 @@ class DataprocClusterScaleOperator(BaseOperator):
 
     **Example**: ::
 
-    t1 = DataprocClusterScaleOperator(
-            task_id='dataproc_scale',
-            project_id='my-project',
-            cluster_name='cluster-1',
-            num_workers=10,
-            num_preemptible_workers=10,
-            graceful_decommission_timeout='1h'
-            dag=dag)
+        t1 = DataprocClusterScaleOperator(
+                task_id='dataproc_scale',
+                project_id='my-project',
+                cluster_name='cluster-1',
+                num_workers=10,
+                num_preemptible_workers=10,
+                graceful_decommission_timeout='1h',
+                dag=dag)
 
     .. seealso::
         For more detail on about scaling clusters have a look at the reference:
         https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/scaling-clusters
 
     :param cluster_name: The name of the cluster to scale. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param project_id: The ID of the google cloud project in which
         the cluster runs. (templated)
-    :type project_id: string
+    :type project_id: str
     :param region: The region for the dataproc cluster. (templated)
-    :type region: string
+    :type region: str
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param num_workers: The new number of workers
     :type num_workers: int
     :param num_preemptible_workers: The new number of preemptible workers
     :type num_preemptible_workers: int
     :param graceful_decommission_timeout: Timeout for graceful YARN decomissioning.
         Maximum value is 1d
-    :type graceful_decommission_timeout: string
+    :type graceful_decommission_timeout: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
 
     template_fields = ['cluster_name', 'project_id', 'region']
@@ -491,7 +538,8 @@ def _build_scale_cluster_data(self):
         }
         return scale_data
 
-    def _get_graceful_decommission_timeout(self, timeout):
+    @staticmethod
+    def _get_graceful_decommission_timeout(timeout):
         match = re.match(r"^(\d+)(s|m|h|d)$", timeout)
         if match:
             if match.group(2) == "s":
@@ -542,18 +590,18 @@ class DataprocClusterDeleteOperator(BaseOperator):
     cluster is destroyed.
 
     :param cluster_name: The name of the cluster to create. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param project_id: The ID of the google cloud project in which
         the cluster runs. (templated)
-    :type project_id: string
+    :type project_id: str
     :param region: leave as 'global', might become relevant in the future. (templated)
-    :type region: string
+    :type region: str
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
 
     template_fields = ['cluster_name', 'project_id', 'region']
@@ -575,7 +623,8 @@ def __init__(self,
         self.project_id = project_id
         self.region = region
 
-    def _wait_for_done(self, service, operation_name):
+    @staticmethod
+    def _wait_for_done(service, operation_name):
         time.sleep(15)
         while True:
             response = service.projects().regions().operations().get(
@@ -643,18 +692,18 @@ class DataProcPigOperator(BaseOperator):
 
     :param query: The query or reference to the query
         file (pg or pig extension). (templated)
-    :type query: string
+    :type query: str
     :param query_uri: The uri of a pig script on Cloud Storage.
-    :type query_uri: string
+    :type query_uri: str
     :param variables: Map of named parameters for the query. (templated)
     :type variables: dict
     :param job_name: The job name used in the DataProc cluster. This
         name by default is the task_id appended with the execution data, but can
         be templated. The name will always be appended with a random number to
         avoid name clashes. (templated)
-    :type job_name: string
+    :type job_name: str
     :param cluster_name: The name of the DataProc cluster. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param dataproc_pig_properties: Map for the Pig properties. Ideal to put in
         default arguments
     :type dataproc_pig_properties: dict
@@ -662,15 +711,27 @@ class DataProcPigOperator(BaseOperator):
         UDFs and libs) and are ideal to put in default arguments.
     :type dataproc_pig_jars: list
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param region: The specified region where the dataproc cluster is created.
-    :type region: string
+    :type region: str
+    :param job_error_states: Job states that should be considered error states.
+        Any states in this list will result in an error being raised and failure of the
+        task. Eg, if the ``CANCELLED`` state should also be considered a task failure,
+        pass in ``['ERROR', 'CANCELLED']``. Possible values are currently only
+        ``'ERROR'`` and ``'CANCELLED'``, but could change in the future. Defaults to
+        ``['ERROR']``.
+    :type job_error_states: list
+    :var dataproc_job_id: The actual "jobId" as submitted to the Dataproc API.
+        This is useful for identifying or linking to the job in the Google Cloud Console
+        Dataproc UI, as the actual "jobId" submitted to the Dataproc API is appended with
+        an 8 character random string.
+    :vartype dataproc_job_id: str
     """
-    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
+    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'region', 'dataproc_jars']
     template_ext = ('.pg', '.pig',)
     ui_color = '#0273d4'
 
@@ -687,6 +748,7 @@ def __init__(
             gcp_conn_id='google_cloud_default',
             delegate_to=None,
             region='global',
+            job_error_states=['ERROR'],
             *args,
             **kwargs):
 
@@ -701,6 +763,7 @@ def __init__(
         self.dataproc_properties = dataproc_pig_properties
         self.dataproc_jars = dataproc_pig_jars
         self.region = region
+        self.job_error_states = job_error_states
 
     def execute(self, context):
         hook = DataProcHook(gcp_conn_id=self.gcp_conn_id,
@@ -716,7 +779,10 @@ def execute(self, context):
         job.add_jar_file_uris(self.dataproc_jars)
         job.set_job_name(self.job_name)
 
-        hook.submit(hook.project_id, job.build(), self.region)
+        job_to_submit = job.build()
+        self.dataproc_job_id = job_to_submit["job"]["reference"]["jobId"]
+
+        hook.submit(hook.project_id, job_to_submit, self.region, self.job_error_states)
 
 
 class DataProcHiveOperator(BaseOperator):
@@ -724,17 +790,17 @@ class DataProcHiveOperator(BaseOperator):
     Start a Hive query Job on a Cloud DataProc cluster.
 
     :param query: The query or reference to the query file (q extension).
-    :type query: string
+    :type query: str
     :param query_uri: The uri of a hive script on Cloud Storage.
-    :type query_uri: string
+    :type query_uri: str
     :param variables: Map of named parameters for the query.
     :type variables: dict
     :param job_name: The job name used in the DataProc cluster. This name by default
         is the task_id appended with the execution data, but can be templated. The
         name will always be appended with a random number to avoid name clashes.
-    :type job_name: string
+    :type job_name: str
     :param cluster_name: The name of the DataProc cluster.
-    :type cluster_name: string
+    :type cluster_name: str
     :param dataproc_hive_properties: Map for the Pig properties. Ideal to put in
         default arguments
     :type dataproc_hive_properties: dict
@@ -742,15 +808,27 @@ class DataProcHiveOperator(BaseOperator):
         UDFs and libs) and are ideal to put in default arguments.
     :type dataproc_hive_jars: list
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param region: The specified region where the dataproc cluster is created.
-    :type region: string
+    :type region: str
+    :param job_error_states: Job states that should be considered error states.
+        Any states in this list will result in an error being raised and failure of the
+        task. Eg, if the ``CANCELLED`` state should also be considered a task failure,
+        pass in ``['ERROR', 'CANCELLED']``. Possible values are currently only
+        ``'ERROR'`` and ``'CANCELLED'``, but could change in the future. Defaults to
+        ``['ERROR']``.
+    :type job_error_states: list
+    :var dataproc_job_id: The actual "jobId" as submitted to the Dataproc API.
+        This is useful for identifying or linking to the job in the Google Cloud Console
+        Dataproc UI, as the actual "jobId" submitted to the Dataproc API is appended with
+        an 8 character random string.
+    :vartype dataproc_job_id: str
     """
-    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
+    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'region', 'dataproc_jars']
     template_ext = ('.q',)
     ui_color = '#0273d4'
 
@@ -767,6 +845,7 @@ def __init__(
             gcp_conn_id='google_cloud_default',
             delegate_to=None,
             region='global',
+            job_error_states=['ERROR'],
             *args,
             **kwargs):
 
@@ -781,6 +860,7 @@ def __init__(
         self.dataproc_properties = dataproc_hive_properties
         self.dataproc_jars = dataproc_hive_jars
         self.region = region
+        self.job_error_states = job_error_states
 
     def execute(self, context):
         hook = DataProcHook(gcp_conn_id=self.gcp_conn_id,
@@ -797,7 +877,10 @@ def execute(self, context):
         job.add_jar_file_uris(self.dataproc_jars)
         job.set_job_name(self.job_name)
 
-        hook.submit(hook.project_id, job.build(), self.region)
+        job_to_submit = job.build()
+        self.dataproc_job_id = job_to_submit["job"]["reference"]["jobId"]
+
+        hook.submit(hook.project_id, job_to_submit, self.region, self.job_error_states)
 
 
 class DataProcSparkSqlOperator(BaseOperator):
@@ -805,18 +888,18 @@ class DataProcSparkSqlOperator(BaseOperator):
     Start a Spark SQL query Job on a Cloud DataProc cluster.
 
     :param query: The query or reference to the query file (q extension). (templated)
-    :type query: string
+    :type query: str
     :param query_uri: The uri of a spark sql script on Cloud Storage.
-    :type query_uri: string
+    :type query_uri: str
     :param variables: Map of named parameters for the query. (templated)
     :type variables: dict
     :param job_name: The job name used in the DataProc cluster. This
         name by default is the task_id appended with the execution data, but can
         be templated. The name will always be appended with a random number to
         avoid name clashes. (templated)
-    :type job_name: string
+    :type job_name: str
     :param cluster_name: The name of the DataProc cluster. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in
         default arguments
     :type dataproc_spark_properties: dict
@@ -824,15 +907,27 @@ class DataProcSparkSqlOperator(BaseOperator):
         for UDFs and libs) and are ideal to put in default arguments.
     :type dataproc_spark_jars: list
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param region: The specified region where the dataproc cluster is created.
-    :type region: string
+    :type region: str
+    :param job_error_states: Job states that should be considered error states.
+        Any states in this list will result in an error being raised and failure of the
+        task. Eg, if the ``CANCELLED`` state should also be considered a task failure,
+        pass in ``['ERROR', 'CANCELLED']``. Possible values are currently only
+        ``'ERROR'`` and ``'CANCELLED'``, but could change in the future. Defaults to
+        ``['ERROR']``.
+    :type job_error_states: list
+    :var dataproc_job_id: The actual "jobId" as submitted to the Dataproc API.
+        This is useful for identifying or linking to the job in the Google Cloud Console
+        Dataproc UI, as the actual "jobId" submitted to the Dataproc API is appended with
+        an 8 character random string.
+    :vartype dataproc_job_id: str
     """
-    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
+    template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'region', 'dataproc_jars']
     template_ext = ('.q',)
     ui_color = '#0273d4'
 
@@ -849,6 +944,7 @@ def __init__(
             gcp_conn_id='google_cloud_default',
             delegate_to=None,
             region='global',
+            job_error_states=['ERROR'],
             *args,
             **kwargs):
 
@@ -863,6 +959,7 @@ def __init__(
         self.dataproc_properties = dataproc_spark_properties
         self.dataproc_jars = dataproc_spark_jars
         self.region = region
+        self.job_error_states = job_error_states
 
     def execute(self, context):
         hook = DataProcHook(gcp_conn_id=self.gcp_conn_id,
@@ -879,7 +976,10 @@ def execute(self, context):
         job.add_jar_file_uris(self.dataproc_jars)
         job.set_job_name(self.job_name)
 
-        hook.submit(hook.project_id, job.build(), self.region)
+        job_to_submit = job.build()
+        self.dataproc_job_id = job_to_submit["job"]["reference"]["jobId"]
+
+        hook.submit(hook.project_id, job_to_submit, self.region, self.job_error_states)
 
 
 class DataProcSparkOperator(BaseOperator):
@@ -888,10 +988,10 @@ class DataProcSparkOperator(BaseOperator):
 
     :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or
             the main_class, not both together).
-    :type main_jar: string
+    :type main_jar: str
     :param main_class: Name of the job class. (use this or the main_jar, not both
         together).
-    :type main_class: string
+    :type main_class: str
     :param arguments: Arguments for the job. (templated)
     :type arguments: list
     :param archives: List of archived files that will be unpacked in the work
@@ -903,9 +1003,9 @@ class DataProcSparkOperator(BaseOperator):
         name by default is the task_id appended with the execution data, but can
         be templated. The name will always be appended with a random number to
         avoid name clashes. (templated)
-    :type job_name: string
+    :type job_name: str
     :param cluster_name: The name of the DataProc cluster. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in
         default arguments
     :type dataproc_spark_properties: dict
@@ -913,16 +1013,28 @@ class DataProcSparkOperator(BaseOperator):
         for UDFs and libs) and are ideal to put in default arguments.
     :type dataproc_spark_jars: list
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param region: The specified region where the dataproc cluster is created.
-    :type region: string
+    :type region: str
+    :param job_error_states: Job states that should be considered error states.
+        Any states in this list will result in an error being raised and failure of the
+        task. Eg, if the ``CANCELLED`` state should also be considered a task failure,
+        pass in ``['ERROR', 'CANCELLED']``. Possible values are currently only
+        ``'ERROR'`` and ``'CANCELLED'``, but could change in the future. Defaults to
+        ``['ERROR']``.
+    :type job_error_states: list
+    :var dataproc_job_id: The actual "jobId" as submitted to the Dataproc API.
+        This is useful for identifying or linking to the job in the Google Cloud Console
+        Dataproc UI, as the actual "jobId" submitted to the Dataproc API is appended with
+        an 8 character random string.
+    :vartype dataproc_job_id: str
     """
 
-    template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
+    template_fields = ['arguments', 'job_name', 'cluster_name', 'region', 'dataproc_jars']
     ui_color = '#0273d4'
 
     @apply_defaults
@@ -940,6 +1052,7 @@ def __init__(
             gcp_conn_id='google_cloud_default',
             delegate_to=None,
             region='global',
+            job_error_states=['ERROR'],
             *args,
             **kwargs):
 
@@ -956,6 +1069,7 @@ def __init__(
         self.dataproc_properties = dataproc_spark_properties
         self.dataproc_jars = dataproc_spark_jars
         self.region = region
+        self.job_error_states = job_error_states
 
     def execute(self, context):
         hook = DataProcHook(gcp_conn_id=self.gcp_conn_id,
@@ -970,7 +1084,10 @@ def execute(self, context):
         job.add_file_uris(self.files)
         job.set_job_name(self.job_name)
 
-        hook.submit(hook.project_id, job.build(), self.region)
+        job_to_submit = job.build()
+        self.dataproc_job_id = job_to_submit["job"]["reference"]["jobId"]
+
+        hook.submit(hook.project_id, job_to_submit, self.region, self.job_error_states)
 
 
 class DataProcHadoopOperator(BaseOperator):
@@ -979,10 +1096,10 @@ class DataProcHadoopOperator(BaseOperator):
 
     :param main_jar: URI of the job jar provisioned on Cloud Storage. (use this or
             the main_class, not both together).
-    :type main_jar: string
+    :type main_jar: str
     :param main_class: Name of the job class. (use this or the main_jar, not both
         together).
-    :type main_class: string
+    :type main_class: str
     :param arguments: Arguments for the job. (templated)
     :type arguments: list
     :param archives: List of archived files that will be unpacked in the work
@@ -994,9 +1111,9 @@ class DataProcHadoopOperator(BaseOperator):
         name by default is the task_id appended with the execution data, but can
         be templated. The name will always be appended with a random number to
         avoid name clashes. (templated)
-    :type job_name: string
+    :type job_name: str
     :param cluster_name: The name of the DataProc cluster. (templated)
-    :type cluster_name: string
+    :type cluster_name: str
     :param dataproc_hadoop_properties: Map for the Pig properties. Ideal to put in
         default arguments
     :type dataproc_hadoop_properties: dict
@@ -1004,16 +1121,28 @@ class DataProcHadoopOperator(BaseOperator):
         for UDFs and libs) and are ideal to put in default arguments.
     :type dataproc_hadoop_jars: list
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param region: The specified region where the dataproc cluster is created.
-    :type region: string
+    :type region: str
+    :param job_error_states: Job states that should be considered error states.
+        Any states in this list will result in an error being raised and failure of the
+        task. Eg, if the ``CANCELLED`` state should also be considered a task failure,
+        pass in ``['ERROR', 'CANCELLED']``. Possible values are currently only
+        ``'ERROR'`` and ``'CANCELLED'``, but could change in the future. Defaults to
+        ``['ERROR']``.
+    :type job_error_states: list
+    :var dataproc_job_id: The actual "jobId" as submitted to the Dataproc API.
+        This is useful for identifying or linking to the job in the Google Cloud Console
+        Dataproc UI, as the actual "jobId" submitted to the Dataproc API is appended with
+        an 8 character random string.
+    :vartype dataproc_job_id: str
     """
 
-    template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
+    template_fields = ['arguments', 'job_name', 'cluster_name', 'region', 'dataproc_jars']
     ui_color = '#0273d4'
 
     @apply_defaults
@@ -1031,6 +1160,7 @@ def __init__(
             gcp_conn_id='google_cloud_default',
             delegate_to=None,
             region='global',
+            job_error_states=['ERROR'],
             *args,
             **kwargs):
 
@@ -1047,6 +1177,7 @@ def __init__(
         self.dataproc_properties = dataproc_hadoop_properties
         self.dataproc_jars = dataproc_hadoop_jars
         self.region = region
+        self.job_error_states = job_error_states
 
     def execute(self, context):
         hook = DataProcHook(gcp_conn_id=self.gcp_conn_id,
@@ -1061,7 +1192,10 @@ def execute(self, context):
         job.add_file_uris(self.files)
         job.set_job_name(self.job_name)
 
-        hook.submit(hook.project_id, job.build(), self.region)
+        job_to_submit = job.build()
+        self.dataproc_job_id = job_to_submit["job"]["reference"]["jobId"]
+
+        hook.submit(hook.project_id, job_to_submit, self.region, self.job_error_states)
 
 
 class DataProcPySparkOperator(BaseOperator):
@@ -1070,7 +1204,7 @@ class DataProcPySparkOperator(BaseOperator):
 
     :param main: [Required] The Hadoop Compatible Filesystem (HCFS) URI of the main
             Python file to use as the driver. Must be a .py file.
-    :type main: string
+    :type main: str
     :param arguments: Arguments for the job. (templated)
     :type arguments: list
     :param archives: List of archived files that will be unpacked in the work
@@ -1085,9 +1219,9 @@ class DataProcPySparkOperator(BaseOperator):
         name by default is the task_id appended with the execution data, but can
         be templated. The name will always be appended with a random number to
         avoid name clashes. (templated)
-    :type job_name: string
+    :type job_name: str
     :param cluster_name: The name of the DataProc cluster.
-    :type cluster_name: string
+    :type cluster_name: str
     :param dataproc_pyspark_properties: Map for the Pig properties. Ideal to put in
         default arguments
     :type dataproc_pyspark_properties: dict
@@ -1095,22 +1229,34 @@ class DataProcPySparkOperator(BaseOperator):
         for UDFs and libs) and are ideal to put in default arguments.
     :type dataproc_pyspark_jars: list
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param region: The specified region where the dataproc cluster is created.
-    :type region: string
+    :type region: str
+    :param job_error_states: Job states that should be considered error states.
+        Any states in this list will result in an error being raised and failure of the
+        task. Eg, if the ``CANCELLED`` state should also be considered a task failure,
+        pass in ``['ERROR', 'CANCELLED']``. Possible values are currently only
+        ``'ERROR'`` and ``'CANCELLED'``, but could change in the future. Defaults to
+        ``['ERROR']``.
+    :type job_error_states: list
+    :var dataproc_job_id: The actual "jobId" as submitted to the Dataproc API.
+        This is useful for identifying or linking to the job in the Google Cloud Console
+        Dataproc UI, as the actual "jobId" submitted to the Dataproc API is appended with
+        an 8 character random string.
+    :vartype dataproc_job_id: str
     """
 
-    template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
+    template_fields = ['arguments', 'job_name', 'cluster_name', 'region', 'dataproc_jars']
     ui_color = '#0273d4'
 
     @staticmethod
     def _generate_temp_filename(filename):
         dt = time.strftime('%Y%m%d%H%M%S')
-        return "{}_{}_{}".format(dt, str(uuid.uuid1())[:8], ntpath.basename(filename))
+        return "{}_{}_{}".format(dt, str(uuid.uuid4())[:8], ntpath.basename(filename))
 
     """
     Upload a local file to a Google Cloud Storage bucket
@@ -1149,6 +1295,7 @@ def __init__(
             gcp_conn_id='google_cloud_default',
             delegate_to=None,
             region='global',
+            job_error_states=['ERROR'],
             *args,
             **kwargs):
 
@@ -1165,6 +1312,7 @@ def __init__(
         self.dataproc_properties = dataproc_pyspark_properties
         self.dataproc_jars = dataproc_pyspark_jars
         self.region = region
+        self.job_error_states = job_error_states
 
     def execute(self, context):
         hook = DataProcHook(
@@ -1192,7 +1340,10 @@ def execute(self, context):
         job.add_python_file_uris(self.pyfiles)
         job.set_job_name(self.job_name)
 
-        hook.submit(hook.project_id, job.build(), self.region)
+        job_to_submit = job.build()
+        self.dataproc_job_id = job_to_submit["job"]["reference"]["jobId"]
+
+        hook.submit(hook.project_id, job_to_submit, self.region, self.job_error_states)
 
 
 class DataprocWorkflowTemplateBaseOperator(BaseOperator):
@@ -1216,10 +1367,10 @@ def __init__(self,
         )
 
     def execute(self, context):
-        self.hook.await(self.start())
+        self.hook.wait(self.start())
 
     def start(self, context):
-        raise AirflowException('plese start a workflow operation')
+        raise AirflowException('Please start a workflow operation')
 
 
 class DataprocWorkflowTemplateInstantiateOperator(DataprocWorkflowTemplateBaseOperator):
@@ -1232,18 +1383,18 @@ class DataprocWorkflowTemplateInstantiateOperator(DataprocWorkflowTemplateBaseOp
         https://cloud.google.com/dataproc/docs/reference/rest/v1beta2/projects.regions.workflowTemplates/instantiate
 
     :param template_id: The id of the template. (templated)
-    :type template_id: string
+    :type template_id: str
     :param project_id: The ID of the google cloud project in which
         the template runs
-    :type project_id: string
+    :type project_id: str
     :param region: leave as 'global', might become relevant in the future
-    :type region: string
+    :type region: str
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
 
     template_fields = ['template_id']
@@ -1261,7 +1412,7 @@ def start(self):
             .instantiate(
                 name=('projects/%s/regions/%s/workflowTemplates/%s' %
                       (self.project_id, self.region, self.template_id)),
-                body={'instanceId': str(uuid.uuid1())})
+                body={'instanceId': str(uuid.uuid4())})
             .execute())
 
 
@@ -1279,15 +1430,15 @@ class DataprocWorkflowTemplateInstantiateInlineOperator(
     :type template: map
     :param project_id: The ID of the google cloud project in which
         the template runs
-    :type project_id: string
+    :type project_id: str
     :param region: leave as 'global', might become relevant in the future
-    :type region: string
+    :type region: str
     :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
 
     template_fields = ['template']
@@ -1304,6 +1455,6 @@ def start(self):
             self.hook.get_conn().projects().regions().workflowTemplates()
             .instantiateInline(
                 parent='projects/%s/regions/%s' % (self.project_id, self.region),
-                instanceId=str(uuid.uuid1()),
+                instanceId=str(uuid.uuid4()),
                 body=self.template)
             .execute())
diff --git a/airflow/contrib/operators/datastore_export_operator.py b/airflow/contrib/operators/datastore_export_operator.py
index f6dc7cc571b41..9d95eadc74822 100644
--- a/airflow/contrib/operators/datastore_export_operator.py
+++ b/airflow/contrib/operators/datastore_export_operator.py
@@ -29,19 +29,19 @@ class DatastoreExportOperator(BaseOperator):
     Export entities from Google Cloud Datastore to Cloud Storage
 
     :param bucket: name of the cloud storage bucket to backup data
-    :type bucket: string
+    :type bucket: str
     :param namespace: optional namespace path in the specified Cloud Storage bucket
         to backup data. If this namespace does not exist in GCS, it will be created.
     :type namespace: str
     :param datastore_conn_id: the name of the Datastore connection id to use
-    :type datastore_conn_id: string
+    :type datastore_conn_id: str
     :param cloud_storage_conn_id: the name of the cloud storage connection id to
         force-write backup
-    :type cloud_storage_conn_id: string
+    :type cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param entity_filter: description of what data from the project is included in the
         export, refer to
         https://cloud.google.com/datastore/docs/reference/rest/Shared.Types/EntityFilter
diff --git a/airflow/contrib/operators/datastore_import_operator.py b/airflow/contrib/operators/datastore_import_operator.py
index 401d36e05b7a8..c79767f35e9cd 100644
--- a/airflow/contrib/operators/datastore_import_operator.py
+++ b/airflow/contrib/operators/datastore_import_operator.py
@@ -28,10 +28,10 @@ class DatastoreImportOperator(BaseOperator):
     Import entities from Cloud Storage to Google Cloud Datastore
 
     :param bucket: container in Cloud Storage to store data
-    :type bucket: string
+    :type bucket: str
     :param file: path of the backup metadata file in the specified Cloud Storage bucket.
         It should have the extension .overall_export_metadata
-    :type file: string
+    :type file: str
     :param namespace: optional namespace of the backup metadata file in
         the specified Cloud Storage bucket.
     :type namespace: str
@@ -42,11 +42,11 @@ class DatastoreImportOperator(BaseOperator):
     :param labels: client-assigned labels for cloud storage
     :type labels: dict
     :param datastore_conn_id: the name of the connection id to use
-    :type datastore_conn_id: string
+    :type datastore_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param polling_interval_in_seconds: number of seconds to wait before polling for
         execution status again
     :type polling_interval_in_seconds: int
diff --git a/airflow/contrib/operators/dingding_operator.py b/airflow/contrib/operators/dingding_operator.py
new file mode 100644
index 0000000000000..2d7ea2225e90e
--- /dev/null
+++ b/airflow/contrib/operators/dingding_operator.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.dingding_hook import DingdingHook
+from airflow.operators.bash_operator import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class DingdingOperator(BaseOperator):
+    """
+    This operator allows you send Dingding message using Dingding custom bot.
+    Get Dingding token from conn_id.password. And prefer set domain to
+    conn_id.host, if not will use default ``https://oapi.dingtalk.com``.
+
+    For more detail message in
+    `Dingding custom bot <https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq>`_
+
+    :param dingding_conn_id: The name of the Dingding connection to use
+    :type dingding_conn_id: str
+    :param message_type: Message type you want to send to Dingding, support five type so far
+        including text, link, markdown, actionCard, feedCard
+    :type message_type: str
+    :param message: The message send to Dingding chat group
+    :type message: str or dict
+    :param at_mobiles: Remind specific users with this message
+    :type at_mobiles: list[str]
+    :param at_all: Remind all people in group or not. If True, will overwrite ``at_mobiles``
+    :type at_all: bool
+    """
+    template_fields = ('message',)
+    ui_color = '#4ea4d4'  # Dingding icon color
+
+    @apply_defaults
+    def __init__(self,
+                 dingding_conn_id='dingding_default',
+                 message_type='text',
+                 message=None,
+                 at_mobiles=None,
+                 at_all=False,
+                 *args,
+                 **kwargs):
+        super(DingdingOperator, self).__init__(*args, **kwargs)
+        self.dingding_conn_id = dingding_conn_id
+        self.message_type = message_type
+        self.message = message
+        self.at_mobiles = at_mobiles
+        self.at_all = at_all
+
+    def execute(self, context):
+        self.log.info('Sending Dingding message.')
+        hook = DingdingHook(
+            self.dingding_conn_id,
+            self.message_type,
+            self.message,
+            self.at_mobiles,
+            self.at_all
+        )
+        hook.send()
diff --git a/airflow/contrib/operators/druid_operator.py b/airflow/contrib/operators/druid_operator.py
index 426393deeb315..75d552fec5a5b 100644
--- a/airflow/contrib/operators/druid_operator.py
+++ b/airflow/contrib/operators/druid_operator.py
@@ -21,6 +21,7 @@
 
 from airflow.hooks.druid_hook import DruidHook
 from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
 
 
 class DruidOperator(BaseOperator):
@@ -36,6 +37,7 @@ class DruidOperator(BaseOperator):
     template_fields = ('index_spec_str',)
     template_ext = ('.json',)
 
+    @apply_defaults
     def __init__(self, json_index_file,
                  druid_ingest_conn_id='druid_ingest_default',
                  max_ingestion_time=None,
@@ -58,5 +60,5 @@ def execute(self, context):
             druid_ingest_conn_id=self.conn_id,
             max_ingestion_time=self.max_ingestion_time
         )
-        self.log.info("Sumitting %s", self.index_spec_str)
+        self.log.info("Submitting %s", self.index_spec_str)
         hook.submit_indexing_job(self.index_spec_str)
diff --git a/airflow/contrib/operators/ecs_operator.py b/airflow/contrib/operators/ecs_operator.py
index 60540f5e289d6..2f7d8c20013e4 100644
--- a/airflow/contrib/operators/ecs_operator.py
+++ b/airflow/contrib/operators/ecs_operator.py
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import sys
+import re
 
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
@@ -33,17 +34,27 @@ class ECSOperator(BaseOperator):
     :type task_definition: str
     :param cluster: the cluster name on EC2 Container Service
     :type cluster: str
-    :param: overrides: the same parameter that boto3 will receive (templated):
-            http://boto3.readthedocs.org/en/latest/reference/services/ecs.html#ECS.Client.run_task
-    :type: overrides: dict
+    :param overrides: the same parameter that boto3 will receive (templated):
+        http://boto3.readthedocs.org/en/latest/reference/services/ecs.html#ECS.Client.run_task
+    :type overrides: dict
     :param aws_conn_id: connection id of AWS credentials / region name. If None,
-            credential boto3 strategy will be used
-            (http://boto3.readthedocs.io/en/latest/guide/configuration.html).
+        credential boto3 strategy will be used
+        (http://boto3.readthedocs.io/en/latest/guide/configuration.html).
     :type aws_conn_id: str
     :param region_name: region name to use in AWS Hook.
         Override the region_name in connection (if provided)
+    :type region_name: str
     :param launch_type: the launch type on which to run your task ('EC2' or 'FARGATE')
-    :type: launch_type: str
+    :type launch_type: str
+    :param group: the name of the task group associated with the task
+    :type group: str
+    :param placement_constraints: an array of placement constraint objects to use for
+        the task
+    :type placement_constraints: list
+    :param platform_version: the platform version on which your task is running
+    :type platform_version: str
+    :param network_configuration: the network configuration for the task
+    :type network_configuration: dict
     """
 
     ui_color = '#f0ede4'
@@ -53,7 +64,9 @@ class ECSOperator(BaseOperator):
 
     @apply_defaults
     def __init__(self, task_definition, cluster, overrides,
-                 aws_conn_id=None, region_name=None, launch_type='EC2', **kwargs):
+                 aws_conn_id=None, region_name=None, launch_type='EC2',
+                 group=None, placement_constraints=None, platform_version='LATEST',
+                 network_configuration=None, **kwargs):
         super(ECSOperator, self).__init__(**kwargs)
 
         self.aws_conn_id = aws_conn_id
@@ -62,6 +75,10 @@ def __init__(self, task_definition, cluster, overrides,
         self.cluster = cluster
         self.overrides = overrides
         self.launch_type = launch_type
+        self.group = group
+        self.placement_constraints = placement_constraints
+        self.platform_version = platform_version
+        self.network_configuration = network_configuration
 
         self.hook = self.get_hook()
 
@@ -77,13 +94,23 @@ def execute(self, context):
             region_name=self.region_name
         )
 
-        response = self.client.run_task(
-            cluster=self.cluster,
-            taskDefinition=self.task_definition,
-            overrides=self.overrides,
-            startedBy=self.owner,
-            launchType=self.launch_type
-        )
+        run_opts = {
+            'cluster': self.cluster,
+            'taskDefinition': self.task_definition,
+            'overrides': self.overrides,
+            'startedBy': self.owner,
+            'launchType': self.launch_type,
+        }
+
+        if self.launch_type == 'FARGATE':
+            run_opts['platformVersion'] = self.platform_version
+        if self.group is not None:
+            run_opts['group'] = self.group
+        if self.placement_constraints is not None:
+            run_opts['placementConstraints'] = self.placement_constraints
+        if self.network_configuration is not None:
+            run_opts['networkConfiguration'] = self.network_configuration
+        response = self.client.run_task(**run_opts)
 
         failures = response['failures']
         if len(failures) > 0:
@@ -115,6 +142,15 @@ def _check_success_task(self):
             raise AirflowException(response)
 
         for task in response['tasks']:
+            # This is a `stoppedReason` that indicates a task has not
+            # successfully finished, but there is no other indication of failure
+            # in the response.
+            # See, https://docs.aws.amazon.com/AmazonECS/latest/developerguide/stopped-task-errors.html # noqa E501
+            if re.match(r'Host EC2 \(instance .+?\) (stopped|terminated)\.',
+                        task.get('stoppedReason', '')):
+                raise AirflowException(
+                    'The task was stopped because the host instance terminated: {}'.
+                    format(task.get('stoppedReason', '')))
             containers = task['containers']
             for container in containers:
                 if container.get('lastStatus') == 'STOPPED' and \
diff --git a/airflow/contrib/operators/emr_add_steps_operator.py b/airflow/contrib/operators/emr_add_steps_operator.py
index 643ffe9c1b94c..44e2e674524f0 100644
--- a/airflow/contrib/operators/emr_add_steps_operator.py
+++ b/airflow/contrib/operators/emr_add_steps_operator.py
@@ -27,7 +27,7 @@ class EmrAddStepsOperator(BaseOperator):
     An operator that adds steps to an existing EMR job_flow.
 
     :param job_flow_id: id of the JobFlow to add steps to. (templated)
-    :type job_flow_name: str
+    :type job_flow_id: str
     :param aws_conn_id: aws connection to uses
     :type aws_conn_id: str
     :param steps: boto3 style steps to be added to the jobflow. (templated)
diff --git a/airflow/contrib/operators/emr_create_job_flow_operator.py b/airflow/contrib/operators/emr_create_job_flow_operator.py
index 89be12f0657e4..62c21a7533fe1 100644
--- a/airflow/contrib/operators/emr_create_job_flow_operator.py
+++ b/airflow/contrib/operators/emr_create_job_flow_operator.py
@@ -33,8 +33,8 @@ class EmrCreateJobFlowOperator(BaseOperator):
     :param emr_conn_id: emr connection to use
     :type emr_conn_id: str
     :param job_flow_overrides: boto3 style arguments to override
-           emr_connection extra. (templated)
-    :type steps: dict
+       emr_connection extra. (templated)
+    :type job_flow_overrides: dict
     """
     template_fields = ['job_flow_overrides']
     template_ext = ()
@@ -46,6 +46,7 @@ def __init__(
             aws_conn_id='s3_default',
             emr_conn_id='emr_default',
             job_flow_overrides=None,
+            region_name=None,
             *args, **kwargs):
         super(EmrCreateJobFlowOperator, self).__init__(*args, **kwargs)
         self.aws_conn_id = aws_conn_id
@@ -53,9 +54,12 @@ def __init__(
         if job_flow_overrides is None:
             job_flow_overrides = {}
         self.job_flow_overrides = job_flow_overrides
+        self.region_name = region_name
 
     def execute(self, context):
-        emr = EmrHook(aws_conn_id=self.aws_conn_id, emr_conn_id=self.emr_conn_id)
+        emr = EmrHook(aws_conn_id=self.aws_conn_id,
+                      emr_conn_id=self.emr_conn_id,
+                      region_name=self.region_name)
 
         self.log.info(
             'Creating JobFlow using aws-conn-id: %s, emr-conn-id: %s',
diff --git a/airflow/contrib/operators/emr_terminate_job_flow_operator.py b/airflow/contrib/operators/emr_terminate_job_flow_operator.py
index 50407a14acad9..0a72f4ac73e81 100644
--- a/airflow/contrib/operators/emr_terminate_job_flow_operator.py
+++ b/airflow/contrib/operators/emr_terminate_job_flow_operator.py
@@ -27,7 +27,7 @@ class EmrTerminateJobFlowOperator(BaseOperator):
     Operator to terminate EMR JobFlows.
 
     :param job_flow_id: id of the JobFlow to terminate. (templated)
-    :type job_flow_name: str
+    :type job_flow_id: str
     :param aws_conn_id: aws connection to uses
     :type aws_conn_id: str
     """
diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py
index 807385b43c3a7..de40abd473e86 100644
--- a/airflow/contrib/operators/file_to_gcs.py
+++ b/airflow/contrib/operators/file_to_gcs.py
@@ -25,20 +25,23 @@
 
 class FileToGoogleCloudStorageOperator(BaseOperator):
     """
-    Uploads a file to Google Cloud Storage
+    Uploads a file to Google Cloud Storage.
+    Optionally can compress the file for upload.
 
     :param src: Path to the local file. (templated)
-    :type src: string
+    :type src: str
     :param dst: Destination path within the specified bucket. (templated)
-    :type dst: string
+    :type dst: str
     :param bucket: The bucket to upload to. (templated)
-    :type bucket: string
+    :type bucket: str
     :param google_cloud_storage_conn_id: The Airflow connection ID to upload with
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param mime_type: The mime-type string
-    :type mime_type: string
+    :type mime_type: str
     :param delegate_to: The account to impersonate, if any
-    :type delegate_to: string
+    :type delegate_to: str
+    :param gzip: Allows for file to be compressed and uploaded as gzip
+    :type gzip: bool
     """
     template_fields = ('src', 'dst', 'bucket')
 
@@ -50,6 +53,7 @@ def __init__(self,
                  google_cloud_storage_conn_id='google_cloud_default',
                  mime_type='application/octet-stream',
                  delegate_to=None,
+                 gzip=False,
                  *args,
                  **kwargs):
         super(FileToGoogleCloudStorageOperator, self).__init__(*args, **kwargs)
@@ -59,6 +63,7 @@ def __init__(self,
         self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
         self.mime_type = mime_type
         self.delegate_to = delegate_to
+        self.gzip = gzip
 
     def execute(self, context):
         """
@@ -72,4 +77,6 @@ def execute(self, context):
             bucket=self.bucket,
             object=self.dst,
             mime_type=self.mime_type,
-            filename=self.src)
+            filename=self.src,
+            gzip=self.gzip,
+        )
diff --git a/airflow/contrib/operators/gcp_bigtable_operator.py b/airflow/contrib/operators/gcp_bigtable_operator.py
new file mode 100644
index 0000000000000..d99746798dc28
--- /dev/null
+++ b/airflow/contrib/operators/gcp_bigtable_operator.py
@@ -0,0 +1,485 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Iterable
+import google.api_core.exceptions
+
+from airflow import AirflowException
+from airflow.models import BaseOperator
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.contrib.hooks.gcp_bigtable_hook import BigtableHook
+from airflow.utils.decorators import apply_defaults
+from google.cloud.bigtable_admin_v2 import enums
+from google.cloud.bigtable.table import ClusterState
+
+
+class BigtableValidationMixin(object):
+    """
+    Common class for Cloud Bigtable operators for validating required fields.
+    """
+
+    REQUIRED_ATTRIBUTES = []  # type: Iterable[str]
+
+    def _validate_inputs(self):
+        for attr_name in self.REQUIRED_ATTRIBUTES:
+            if not getattr(self, attr_name):
+                raise AirflowException('Empty parameter: {}'.format(attr_name))
+
+
+class BigtableInstanceCreateOperator(BaseOperator, BigtableValidationMixin):
+    """
+    Creates a new Cloud Bigtable instance.
+    If the Cloud Bigtable instance with the given ID exists, the operator does not
+    compare its configuration
+    and immediately succeeds. No changes are made to the existing instance.
+
+    For more details about instance creation have a look at the reference:
+    https://googleapis.github.io/google-cloud-python/latest/bigtable/instance.html#google.cloud.bigtable.instance.Instance.create
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BigtableInstanceCreateOperator`
+
+    :type instance_id: str
+    :param instance_id: The ID of the Cloud Bigtable instance to create.
+    :type main_cluster_id: str
+    :param main_cluster_id: The ID for main cluster for the new instance.
+    :type main_cluster_zone: str
+    :param main_cluster_zone: The zone for main cluster
+        See https://cloud.google.com/bigtable/docs/locations for more details.
+    :type project_id: str
+    :param project_id: Optional, the ID of the GCP project.  If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type replica_cluster_id: str
+    :param replica_cluster_id: (optional) The ID for replica cluster for the new instance.
+    :type replica_cluster_zone: str
+    :param replica_cluster_zone: (optional)  The zone for replica cluster.
+    :type instance_type: enums.IntEnum
+    :param instance_type: (optional) The type of the instance.
+    :type instance_display_name: str
+    :param instance_display_name: (optional) Human-readable name of the instance. Defaults
+        to ``instance_id``.
+    :type instance_labels: dict
+    :param instance_labels: (optional) Dictionary of labels to associate
+        with the instance.
+    :type cluster_nodes: int
+    :param cluster_nodes: (optional) Number of nodes for cluster.
+    :type cluster_storage_type: enums.IntEnum
+    :param cluster_storage_type: (optional) The type of storage.
+    :type timeout: int
+    :param timeout: (optional) timeout (in seconds) for instance creation.
+                    If None is not specified, Operator will wait indefinitely.
+    """
+
+    REQUIRED_ATTRIBUTES = ('instance_id', 'main_cluster_id',
+                           'main_cluster_zone')
+    template_fields = ['project_id', 'instance_id', 'main_cluster_id',
+                       'main_cluster_zone']
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 main_cluster_id,
+                 main_cluster_zone,
+                 project_id=None,
+                 replica_cluster_id=None,
+                 replica_cluster_zone=None,
+                 instance_display_name=None,
+                 instance_type=None,
+                 instance_labels=None,
+                 cluster_nodes=None,
+                 cluster_storage_type=None,
+                 timeout=None,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance_id = instance_id
+        self.main_cluster_id = main_cluster_id
+        self.main_cluster_zone = main_cluster_zone
+        self.replica_cluster_id = replica_cluster_id
+        self.replica_cluster_zone = replica_cluster_zone
+        self.instance_display_name = instance_display_name
+        self.instance_type = instance_type
+        self.instance_labels = instance_labels
+        self.cluster_nodes = cluster_nodes
+        self.cluster_storage_type = cluster_storage_type
+        self.timeout = timeout
+        self._validate_inputs()
+        self.hook = BigtableHook()
+        super(BigtableInstanceCreateOperator, self).__init__(*args, **kwargs)
+
+    def execute(self, context):
+        instance = self.hook.get_instance(project_id=self.project_id,
+                                          instance_id=self.instance_id)
+        if instance:
+            # Based on Instance.__eq__ instance with the same ID and client is
+            # considered as equal.
+            self.log.info(
+                "The instance '%s' already exists in this project. "
+                "Consider it as created",
+                self.instance_id
+            )
+            return
+        try:
+            self.hook.create_instance(
+                project_id=self.project_id,
+                instance_id=self.instance_id,
+                main_cluster_id=self.main_cluster_id,
+                main_cluster_zone=self.main_cluster_zone,
+                replica_cluster_id=self.replica_cluster_id,
+                replica_cluster_zone=self.replica_cluster_zone,
+                instance_display_name=self.instance_display_name,
+                instance_type=self.instance_type,
+                instance_labels=self.instance_labels,
+                cluster_nodes=self.cluster_nodes,
+                cluster_storage_type=self.cluster_storage_type,
+                timeout=self.timeout,
+            )
+        except google.api_core.exceptions.GoogleAPICallError as e:
+            self.log.error('An error occurred. Exiting.')
+            raise e
+
+
+class BigtableInstanceDeleteOperator(BaseOperator, BigtableValidationMixin):
+    """
+    Deletes the Cloud Bigtable instance, including its clusters and all related tables.
+
+    For more details about deleting instance have a look at the reference:
+    https://googleapis.github.io/google-cloud-python/latest/bigtable/instance.html#google.cloud.bigtable.instance.Instance.delete
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BigtableInstanceDeleteOperator`
+
+    :type instance_id: str
+    :param instance_id: The ID of the Cloud Bigtable instance to delete.
+    :param project_id: Optional, the ID of the GCP project.  If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    """
+    REQUIRED_ATTRIBUTES = ('instance_id',)
+    template_fields = ['project_id', 'instance_id']
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 project_id=None,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance_id = instance_id
+        self._validate_inputs()
+        self.hook = BigtableHook()
+        super(BigtableInstanceDeleteOperator, self).__init__(*args, **kwargs)
+
+    def execute(self, context):
+        try:
+            self.hook.delete_instance(project_id=self.project_id,
+                                      instance_id=self.instance_id)
+        except google.api_core.exceptions.NotFound:
+            self.log.info(
+                "The instance '%s' does not exist in project '%s'. "
+                "Consider it as deleted",
+                self.instance_id, self.project_id
+            )
+        except google.api_core.exceptions.GoogleAPICallError as e:
+            self.log.error('An error occurred. Exiting.')
+            raise e
+
+
+class BigtableTableCreateOperator(BaseOperator, BigtableValidationMixin):
+    """
+    Creates the table in the Cloud Bigtable instance.
+
+    For more details about creating table have a look at the reference:
+    https://googleapis.github.io/google-cloud-python/latest/bigtable/table.html#google.cloud.bigtable.table.Table.create
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BigtableTableCreateOperator`
+
+    :type instance_id: str
+    :param instance_id: The ID of the Cloud Bigtable instance that will
+        hold the new table.
+    :type table_id: str
+    :param table_id: The ID of the table to be created.
+    :type project_id: str
+    :param project_id: Optional, the ID of the GCP project. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type initial_split_keys: list
+    :param initial_split_keys: (Optional) list of row keys in bytes that will be used to
+        initially split the table into several tablets.
+    :type column_families: dict
+    :param column_families: (Optional) A map columns to create.
+                            The key is the column_id str and the value is a
+                            :class:`google.cloud.bigtable.column_family.GarbageCollectionRule`
+    """
+    REQUIRED_ATTRIBUTES = ('instance_id', 'table_id')
+    template_fields = ['project_id', 'instance_id', 'table_id']
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 table_id,
+                 project_id=None,
+                 initial_split_keys=None,
+                 column_families=None,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance_id = instance_id
+        self.table_id = table_id
+        self.initial_split_keys = initial_split_keys or list()
+        self.column_families = column_families or dict()
+        self._validate_inputs()
+        self.hook = BigtableHook()
+        self.instance = None
+        super(BigtableTableCreateOperator, self).__init__(*args, **kwargs)
+
+    def _compare_column_families(self):
+        table_column_families = self.hook.get_column_families_for_table(self.instance,
+                                                                        self.table_id)
+        if set(table_column_families.keys()) != set(self.column_families.keys()):
+            self.log.error("Table '%s' has different set of Column Families",
+                           self.table_id)
+            self.log.error("Expected: %s", self.column_families.keys())
+            self.log.error("Actual: %s", table_column_families.keys())
+            return False
+
+        for key in table_column_families.keys():
+            # There is difference in structure between local Column Families
+            # and remote ones
+            # Local `self.column_families` is dict with column_id as key
+            # and GarbageCollectionRule as value.
+            # Remote `table_column_families` is list of ColumnFamily objects.
+            # For more information about ColumnFamily please refer to the documentation:
+            # https://googleapis.github.io/google-cloud-python/latest/bigtable/column-family.html#google.cloud.bigtable.column_family.ColumnFamily
+            if table_column_families[key].gc_rule != self.column_families[key]:
+                self.log.error("Column Family '%s' differs for table '%s'.", key,
+                               self.table_id)
+                return False
+        return True
+
+    def execute(self, context):
+        self.instance = self.hook.get_instance(project_id=self.project_id,
+                                               instance_id=self.instance_id)
+        if not self.instance:
+            raise AirflowException(
+                "Dependency: instance '{}' does not exist in project '{}'.".
+                format(self.instance_id, self.project_id))
+        try:
+            self.hook.create_table(
+                instance=self.instance,
+                table_id=self.table_id,
+                initial_split_keys=self.initial_split_keys,
+                column_families=self.column_families
+            )
+        except google.api_core.exceptions.AlreadyExists:
+            if not self._compare_column_families():
+                raise AirflowException(
+                    "Table '{}' already exists with different Column Families.".
+                    format(self.table_id))
+            self.log.info("The table '%s' already exists. Consider it as created",
+                          self.table_id)
+
+
+class BigtableTableDeleteOperator(BaseOperator, BigtableValidationMixin):
+    """
+    Deletes the Cloud Bigtable table.
+
+    For more details about deleting table have a look at the reference:
+    https://googleapis.github.io/google-cloud-python/latest/bigtable/table.html#google.cloud.bigtable.table.Table.delete
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BigtableTableDeleteOperator`
+
+    :type instance_id: str
+    :param instance_id: The ID of the Cloud Bigtable instance.
+    :type table_id: str
+    :param table_id: The ID of the table to be deleted.
+    :type project_id: str
+    :param project_id: Optional, the ID of the GCP project. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type app_profile_id: str
+    :parm app_profile_id: Application profile.
+    """
+    REQUIRED_ATTRIBUTES = ('instance_id', 'table_id')
+    template_fields = ['project_id', 'instance_id', 'table_id']
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 table_id,
+                 project_id=None,
+                 app_profile_id=None,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance_id = instance_id
+        self.table_id = table_id
+        self.app_profile_id = app_profile_id
+        self._validate_inputs()
+        self.hook = BigtableHook()
+        super(BigtableTableDeleteOperator, self).__init__(*args, **kwargs)
+
+    def execute(self, context):
+        instance = self.hook.get_instance(project_id=self.project_id,
+                                          instance_id=self.instance_id)
+        if not instance:
+            raise AirflowException("Dependency: instance '{}' does not exist.".format(
+                self.instance_id))
+
+        try:
+            self.hook.delete_table(
+                project_id=self.project_id,
+                instance_id=self.instance_id,
+                table_id=self.table_id,
+            )
+        except google.api_core.exceptions.NotFound:
+            # It's OK if table doesn't exists.
+            self.log.info("The table '%s' no longer exists. Consider it as deleted",
+                          self.table_id)
+        except google.api_core.exceptions.GoogleAPICallError as e:
+            self.log.error('An error occurred. Exiting.')
+            raise e
+
+
+class BigtableClusterUpdateOperator(BaseOperator, BigtableValidationMixin):
+    """
+    Updates a Cloud Bigtable cluster.
+
+    For more details about updating a Cloud Bigtable cluster,
+    have a look at the reference:
+    https://googleapis.github.io/google-cloud-python/latest/bigtable/cluster.html#google.cloud.bigtable.cluster.Cluster.update
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BigtableClusterUpdateOperator`
+
+    :type instance_id: str
+    :param instance_id: The ID of the Cloud Bigtable instance.
+    :type cluster_id: str
+    :param cluster_id: The ID of the Cloud Bigtable cluster to update.
+    :type nodes: int
+    :param nodes: The desired number of nodes for the Cloud Bigtable cluster.
+    :type project_id: str
+    :param project_id: Optional, the ID of the GCP project.
+    """
+    REQUIRED_ATTRIBUTES = ('instance_id', 'cluster_id', 'nodes')
+    template_fields = ['project_id', 'instance_id', 'cluster_id', 'nodes']
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 cluster_id,
+                 nodes,
+                 project_id=None,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance_id = instance_id
+        self.cluster_id = cluster_id
+        self.nodes = nodes
+        self._validate_inputs()
+        self.hook = BigtableHook()
+        super(BigtableClusterUpdateOperator, self).__init__(*args, **kwargs)
+
+    def execute(self, context):
+        instance = self.hook.get_instance(project_id=self.project_id,
+                                          instance_id=self.instance_id)
+        if not instance:
+            raise AirflowException("Dependency: instance '{}' does not exist.".format(
+                self.instance_id))
+
+        try:
+            self.hook.update_cluster(
+                instance=instance,
+                cluster_id=self.cluster_id,
+                nodes=self.nodes
+            )
+        except google.api_core.exceptions.NotFound:
+            raise AirflowException(
+                "Dependency: cluster '{}' does not exist for instance '{}'.".
+                format(self.cluster_id, self.instance_id))
+        except google.api_core.exceptions.GoogleAPICallError as e:
+            self.log.error('An error occurred. Exiting.')
+            raise e
+
+
+class BigtableTableWaitForReplicationSensor(BaseSensorOperator, BigtableValidationMixin):
+    """
+    Sensor that waits for Cloud Bigtable table to be fully replicated to its clusters.
+    No exception will be raised if the instance or the table does not exist.
+
+    For more details about cluster states for a table, have a look at the reference:
+    https://googleapis.github.io/google-cloud-python/latest/bigtable/table.html#google.cloud.bigtable.table.Table.get_cluster_states
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BigtableTableWaitForReplicationSensor`
+
+    :type instance_id: str
+    :param instance_id: The ID of the Cloud Bigtable instance.
+    :type table_id: str
+    :param table_id: The ID of the table to check replication status.
+    :type project_id: str
+    :param project_id: Optional, the ID of the GCP project.
+    """
+    REQUIRED_ATTRIBUTES = ('instance_id', 'table_id')
+    template_fields = ['project_id', 'instance_id', 'table_id']
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 table_id,
+                 project_id=None,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance_id = instance_id
+        self.table_id = table_id
+        self._validate_inputs()
+        self.hook = BigtableHook()
+        super(BigtableTableWaitForReplicationSensor, self).__init__(*args, **kwargs)
+
+    def poke(self, context):
+        instance = self.hook.get_instance(project_id=self.project_id,
+                                          instance_id=self.instance_id)
+        if not instance:
+            self.log.info("Dependency: instance '%s' does not exist.", self.instance_id)
+            return False
+
+        try:
+            cluster_states = self.hook.get_cluster_states_for_table(instance=instance,
+                                                                    table_id=self.table_id)
+        except google.api_core.exceptions.NotFound:
+            self.log.info(
+                "Dependency: table '%s' does not exist in instance '%s'.",
+                self.table_id, self.instance_id)
+            return False
+
+        ready_state = ClusterState(enums.Table.ClusterState.ReplicationState.READY)
+
+        is_table_replicated = True
+        for cluster_id in cluster_states.keys():
+            if cluster_states[cluster_id] != ready_state:
+                self.log.info("Table '%s' is not yet replicated on cluster '%s'.",
+                              self.table_id, cluster_id)
+                is_table_replicated = False
+
+        if not is_table_replicated:
+            return False
+
+        self.log.info("Table '%s' is replicated.", self.table_id)
+        return True
diff --git a/airflow/contrib/operators/gcp_compute_operator.py b/airflow/contrib/operators/gcp_compute_operator.py
new file mode 100644
index 0000000000000..23c339bf32842
--- /dev/null
+++ b/airflow/contrib/operators/gcp_compute_operator.py
@@ -0,0 +1,484 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from copy import deepcopy
+
+from googleapiclient.errors import HttpError
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_compute_hook import GceHook
+from airflow.contrib.utils.gcp_field_sanitizer import GcpBodyFieldSanitizer
+from airflow.contrib.utils.gcp_field_validator import GcpBodyFieldValidator
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+from json_merge_patch import merge
+
+
+class GceBaseOperator(BaseOperator):
+    """
+    Abstract base operator for Google Compute Engine operators to inherit from.
+    """
+
+    @apply_defaults
+    def __init__(self,
+                 zone,
+                 resource_id,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.zone = zone
+        self.resource_id = resource_id
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+        self._hook = GceHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version)
+        super(GceBaseOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is missing")
+        if not self.zone:
+            raise AirflowException("The required parameter 'zone' is missing")
+        if not self.resource_id:
+            raise AirflowException("The required parameter 'resource_id' is missing")
+
+    def execute(self, context):
+        pass
+
+
+class GceInstanceStartOperator(GceBaseOperator):
+    """
+    Starts an instance in Google Compute Engine.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GceInstanceStartOperator`
+
+    :param zone: Google Cloud Platform zone where the instance exists.
+    :type zone: str
+    :param resource_id: Name of the Compute Engine instance resource.
+    :type resource_id: str
+    :param project_id: Optional, Google Cloud Platform Project ID where the Compute
+        Engine Instance exists.  If set to None or missing, the default project_id from the GCP connection is
+        used.
+    :type project_id: str
+    :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud
+        Platform. Defaults to 'google_cloud_default'.
+    :type gcp_conn_id: str
+    :param api_version: Optional, API version used (for example v1 - or beta). Defaults
+        to v1.
+    :type api_version: str
+    :param validate_body: Optional, If set to False, body validation is not performed.
+        Defaults to False.
+    """
+    # [START gce_instance_start_template_fields]
+    template_fields = ('project_id', 'zone', 'resource_id', 'gcp_conn_id', 'api_version')
+    # [END gce_instance_start_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 zone,
+                 resource_id,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 *args, **kwargs):
+        super(GceInstanceStartOperator, self).__init__(
+            project_id=project_id, zone=zone, resource_id=resource_id,
+            gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
+
+    def execute(self, context):
+        return self._hook.start_instance(zone=self.zone,
+                                         resource_id=self.resource_id,
+                                         project_id=self.project_id)
+
+
+class GceInstanceStopOperator(GceBaseOperator):
+    """
+    Stops an instance in Google Compute Engine.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GceInstanceStopOperator`
+
+    :param zone: Google Cloud Platform zone where the instance exists.
+    :type zone: str
+    :param resource_id: Name of the Compute Engine instance resource.
+    :type resource_id: str
+    :param project_id: Optional, Google Cloud Platform Project ID where the Compute
+        Engine Instance exists. If set to None or missing, the default project_id from the GCP connection is
+        used.
+    :type project_id: str
+    :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud
+        Platform. Defaults to 'google_cloud_default'.
+    :type gcp_conn_id: str
+    :param api_version: Optional, API version used (for example v1 - or beta). Defaults
+        to v1.
+    :type api_version: str
+    :param validate_body: Optional, If set to False, body validation is not performed.
+        Defaults to False.
+    """
+    # [START gce_instance_stop_template_fields]
+    template_fields = ('project_id', 'zone', 'resource_id', 'gcp_conn_id', 'api_version')
+    # [END gce_instance_stop_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 zone,
+                 resource_id,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 *args, **kwargs):
+        super(GceInstanceStopOperator, self).__init__(
+            project_id=project_id, zone=zone, resource_id=resource_id,
+            gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
+
+    def execute(self, context):
+        self._hook.stop_instance(zone=self.zone,
+                                 resource_id=self.resource_id,
+                                 project_id=self.project_id)
+
+
+SET_MACHINE_TYPE_VALIDATION_SPECIFICATION = [
+    dict(name="machineType", regexp="^.+$"),
+]
+
+
+class GceSetMachineTypeOperator(GceBaseOperator):
+    """
+    Changes the machine type for a stopped instance to the machine type specified in
+        the request.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GceSetMachineTypeOperator`
+
+    :param zone: Google Cloud Platform zone where the instance exists.
+    :type zone: str
+    :param resource_id: Name of the Compute Engine instance resource.
+    :type resource_id: str
+    :param body: Body required by the Compute Engine setMachineType API, as described in
+        https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType#request-body
+    :type body: dict
+    :param project_id: Optional, Google Cloud Platform Project ID where the Compute
+        Engine Instance exists. If set to None or missing, the default project_id from the GCP connection
+        is used.
+    :type project_id: str
+    :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud
+        Platform. Defaults to 'google_cloud_default'.
+    :type gcp_conn_id: str
+    :param api_version: Optional, API version used (for example v1 - or beta). Defaults
+        to v1.
+    :type api_version: str
+    :param validate_body: Optional, If set to False, body validation is not performed.
+        Defaults to False.
+    :type validate_body: bool
+    """
+    # [START gce_instance_set_machine_type_template_fields]
+    template_fields = ('project_id', 'zone', 'resource_id', 'gcp_conn_id', 'api_version')
+    # [END gce_instance_set_machine_type_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 zone,
+                 resource_id,
+                 body,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.body = body
+        self._field_validator = None
+        if validate_body:
+            self._field_validator = GcpBodyFieldValidator(
+                SET_MACHINE_TYPE_VALIDATION_SPECIFICATION, api_version=api_version)
+        super(GceSetMachineTypeOperator, self).__init__(
+            project_id=project_id, zone=zone, resource_id=resource_id,
+            gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
+
+    def _validate_all_body_fields(self):
+        if self._field_validator:
+            self._field_validator.validate(self.body)
+
+    def execute(self, context):
+        self._validate_all_body_fields()
+        return self._hook.set_machine_type(zone=self.zone,
+                                           resource_id=self.resource_id,
+                                           body=self.body,
+                                           project_id=self.project_id)
+
+
+GCE_INSTANCE_TEMPLATE_VALIDATION_PATCH_SPECIFICATION = [
+    dict(name="name", regexp="^.+$"),
+    dict(name="description", optional=True),
+    dict(name="properties", type='dict', optional=True, fields=[
+        dict(name="description", optional=True),
+        dict(name="tags", optional=True, fields=[
+            dict(name="items", optional=True)
+        ]),
+        dict(name="machineType", optional=True),
+        dict(name="canIpForward", optional=True),
+        dict(name="networkInterfaces", optional=True),  # not validating deeper
+        dict(name="disks", optional=True),  # not validating the array deeper
+        dict(name="metadata", optional=True, fields=[
+            dict(name="fingerprint", optional=True),
+            dict(name="items", optional=True),
+            dict(name="kind", optional=True),
+        ]),
+        dict(name="serviceAccounts", optional=True),  # not validating deeper
+        dict(name="scheduling", optional=True, fields=[
+            dict(name="onHostMaintenance", optional=True),
+            dict(name="automaticRestart", optional=True),
+            dict(name="preemptible", optional=True),
+            dict(name="nodeAffinitites", optional=True),  # not validating deeper
+        ]),
+        dict(name="labels", optional=True),
+        dict(name="guestAccelerators", optional=True),  # not validating deeper
+        dict(name="minCpuPlatform", optional=True),
+    ]),
+]
+
+GCE_INSTANCE_TEMPLATE_FIELDS_TO_SANITIZE = [
+    "kind",
+    "id",
+    "name",
+    "creationTimestamp",
+    "properties.disks.sha256",
+    "properties.disks.kind",
+    "properties.disks.sourceImageEncryptionKey.sha256",
+    "properties.disks.index",
+    "properties.disks.licenses",
+    "properties.networkInterfaces.kind",
+    "properties.networkInterfaces.accessConfigs.kind",
+    "properties.networkInterfaces.name",
+    "properties.metadata.kind",
+    "selfLink"
+]
+
+
+class GceInstanceTemplateCopyOperator(GceBaseOperator):
+    """
+    Copies the instance template, applying specified changes.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GceInstanceTemplateCopyOperator`
+
+    :param resource_id: Name of the Instance Template
+    :type resource_id: str
+    :param body_patch: Patch to the body of instanceTemplates object following rfc7386
+        PATCH semantics. The body_patch content follows
+        https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates
+        Name field is required as we need to rename the template,
+        all the other fields are optional. It is important to follow PATCH semantics
+        - arrays are replaced fully, so if you need to update an array you should
+        provide the whole target array as patch element.
+    :type body_patch: dict
+    :param project_id: Optional, Google Cloud Platform Project ID where the Compute
+        Engine Instance exists.  If set to None or missing, the default project_id from the GCP connection
+        is used.
+    :type project_id: str
+    :param request_id: Optional, unique request_id that you might add to achieve
+        full idempotence (for example when client call times out repeating the request
+        with the same request id will not create a new instance template again).
+        It should be in UUID format as defined in RFC 4122.
+    :type request_id: str
+    :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud
+        Platform. Defaults to 'google_cloud_default'.
+    :type gcp_conn_id: str
+    :param api_version: Optional, API version used (for example v1 - or beta). Defaults
+        to v1.
+    :type api_version: str
+    :param validate_body: Optional, If set to False, body validation is not performed.
+        Defaults to False.
+    :type validate_body: bool
+    """
+    # [START gce_instance_template_copy_operator_template_fields]
+    template_fields = ('project_id', 'resource_id', 'request_id',
+                       'gcp_conn_id', 'api_version')
+    # [END gce_instance_template_copy_operator_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 resource_id,
+                 body_patch,
+                 project_id=None,
+                 request_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.body_patch = body_patch
+        self.request_id = request_id
+        self._field_validator = None
+        if 'name' not in self.body_patch:
+            raise AirflowException("The body '{}' should contain at least "
+                                   "name for the new operator in the 'name' field".
+                                   format(body_patch))
+        if validate_body:
+            self._field_validator = GcpBodyFieldValidator(
+                GCE_INSTANCE_TEMPLATE_VALIDATION_PATCH_SPECIFICATION, api_version=api_version)
+        self._field_sanitizer = GcpBodyFieldSanitizer(
+            GCE_INSTANCE_TEMPLATE_FIELDS_TO_SANITIZE)
+        super(GceInstanceTemplateCopyOperator, self).__init__(
+            project_id=project_id, zone='global', resource_id=resource_id,
+            gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
+
+    def _validate_all_body_fields(self):
+        if self._field_validator:
+            self._field_validator.validate(self.body_patch)
+
+    def execute(self, context):
+        self._validate_all_body_fields()
+        try:
+            # Idempotence check (sort of) - we want to check if the new template
+            # is already created and if is, then we assume it was created by previous run
+            # of CopyTemplate operator - we do not check if content of the template
+            # is as expected. Templates are immutable so we cannot update it anyway
+            # and deleting/recreating is not worth the hassle especially
+            # that we cannot delete template if it is already used in some Instance
+            # Group Manager. We assume success if the template is simply present
+            existing_template = self._hook.get_instance_template(
+                resource_id=self.body_patch['name'], project_id=self.project_id)
+            self.log.info(
+                "The %s template already existed. It was likely created by previous run of the operator. "
+                "Assuming success.",
+                existing_template
+            )
+            return existing_template
+        except HttpError as e:
+            # We actually expect to get 404 / Not Found here as the template should
+            # not yet exist
+            if not e.resp.status == 404:
+                raise e
+        old_body = self._hook.get_instance_template(resource_id=self.resource_id,
+                                                    project_id=self.project_id)
+        new_body = deepcopy(old_body)
+        self._field_sanitizer.sanitize(new_body)
+        new_body = merge(new_body, self.body_patch)
+        self.log.info("Calling insert instance template with updated body: %s", new_body)
+        self._hook.insert_instance_template(body=new_body,
+                                            request_id=self.request_id,
+                                            project_id=self.project_id)
+        return self._hook.get_instance_template(resource_id=self.body_patch['name'],
+                                                project_id=self.project_id)
+
+
+class GceInstanceGroupManagerUpdateTemplateOperator(GceBaseOperator):
+    """
+    Patches the Instance Group Manager, replacing source template URL with the
+    destination one. API V1 does not have update/patch operations for Instance
+    Group Manager, so you must use beta or newer API version. Beta is the default.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GceInstanceGroupManagerUpdateTemplateOperator`
+
+    :param resource_id: Name of the Instance Group Manager
+    :type resource_id: str
+    :param zone: Google Cloud Platform zone where the Instance Group Manager exists.
+    :type zone: str
+    :param source_template: URL of the template to replace.
+    :type source_template: str
+    :param destination_template: URL of the target template.
+    :type destination_template: str
+    :param project_id: Optional, Google Cloud Platform Project ID where the Compute
+        Engine Instance exists.  If set to None or missing, the default project_id from the GCP connection is
+        used.
+    :type project_id: str
+    :param request_id: Optional, unique request_id that you might add to achieve
+        full idempotence (for example when client call times out repeating the request
+        with the same request id will not create a new instance template again).
+        It should be in UUID format as defined in RFC 4122.
+    :type request_id: str
+    :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud
+        Platform. Defaults to 'google_cloud_default'.
+    :type gcp_conn_id: str
+    :param api_version: Optional, API version used (for example v1 - or beta). Defaults
+        to v1.
+    :type api_version: str
+    :param validate_body: Optional, If set to False, body validation is not performed.
+        Defaults to False.
+    :type validate_body: bool
+    """
+    # [START gce_igm_update_template_operator_template_fields]
+    template_fields = ('project_id', 'resource_id', 'zone', 'request_id',
+                       'source_template', 'destination_template',
+                       'gcp_conn_id', 'api_version')
+    # [END gce_igm_update_template_operator_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 resource_id,
+                 zone,
+                 source_template,
+                 destination_template,
+                 project_id=None,
+                 update_policy=None,
+                 request_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='beta',
+                 *args, **kwargs):
+        self.zone = zone
+        self.source_template = source_template
+        self.destination_template = destination_template
+        self.request_id = request_id
+        self.update_policy = update_policy
+        self._change_performed = False
+        if api_version == 'v1':
+            raise AirflowException("Api version v1 does not have update/patch "
+                                   "operations for Instance Group Managers. Use beta"
+                                   " api version or above")
+        super(GceInstanceGroupManagerUpdateTemplateOperator, self).__init__(
+            project_id=project_id, zone=self.zone, resource_id=resource_id,
+            gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
+
+    def _possibly_replace_template(self, dictionary):
+        # type: (dict) -> None
+        if dictionary.get('instanceTemplate') == self.source_template:
+            dictionary['instanceTemplate'] = self.destination_template
+            self._change_performed = True
+
+    def execute(self, context):
+        old_instance_group_manager = self._hook.get_instance_group_manager(
+            zone=self.zone, resource_id=self.resource_id, project_id=self.project_id)
+        patch_body = {}
+        if 'versions' in old_instance_group_manager:
+            patch_body['versions'] = old_instance_group_manager['versions']
+        if 'instanceTemplate' in old_instance_group_manager:
+            patch_body['instanceTemplate'] = old_instance_group_manager['instanceTemplate']
+        if self.update_policy:
+            patch_body['updatePolicy'] = self.update_policy
+        self._possibly_replace_template(patch_body)
+        if 'versions' in patch_body:
+            for version in patch_body['versions']:
+                self._possibly_replace_template(version)
+        if self._change_performed or self.update_policy:
+            self.log.info("Calling patch instance template with updated body: {}".
+                          format(patch_body))
+            return self._hook.patch_instance_group_manager(
+                zone=self.zone, resource_id=self.resource_id,
+                body=patch_body, request_id=self.request_id,
+                project_id=self.project_id)
+        else:
+            # Idempotence achieved
+            return True
diff --git a/airflow/contrib/operators/gcp_container_operator.py b/airflow/contrib/operators/gcp_container_operator.py
index 5648b4d8a01b3..c5325193f6ff0 100644
--- a/airflow/contrib/operators/gcp_container_operator.py
+++ b/airflow/contrib/operators/gcp_container_operator.py
@@ -17,13 +17,48 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+import os
+import subprocess
+import tempfile
+
 from airflow import AirflowException
 from airflow.contrib.hooks.gcp_container_hook import GKEClusterHook
+from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
 
 
 class GKEClusterDeleteOperator(BaseOperator):
+    """
+    Deletes the cluster, including the Kubernetes endpoint and all worker nodes.
+
+    To delete a certain cluster, you must specify the ``project_id``, the ``name``
+    of the cluster, the ``location`` that the cluster is in, and the ``task_id``.
+
+    **Operator Creation**: ::
+
+        operator = GKEClusterDeleteOperator(
+                    task_id='cluster_delete',
+                    project_id='my-project',
+                    location='cluster-location'
+                    name='cluster-name')
+
+    .. seealso::
+        For more detail about deleting clusters have a look at the reference:
+        https://google-cloud-python.readthedocs.io/en/latest/container/gapic/v1/api.html#google.cloud.container_v1.ClusterManagerClient.delete_cluster
+
+    :param project_id: The Google Developers Console [project ID or project number]
+    :type project_id: str
+    :param name: The name of the resource to delete, in this case cluster name
+    :type name: str
+    :param location: The name of the Google Compute Engine zone in which the cluster
+        resides.
+    :type location: str
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: The api version to use
+    :type api_version: str
+    """
     template_fields = ['project_id', 'gcp_conn_id', 'name', 'location', 'api_version']
 
     @apply_defaults
@@ -35,37 +70,6 @@ def __init__(self,
                  api_version='v2',
                  *args,
                  **kwargs):
-        """
-        Deletes the cluster, including the Kubernetes endpoint and all worker nodes.
-
-
-        To delete a certain cluster, you must specify the ``project_id``, the ``name``
-        of the cluster, the ``location`` that the cluster is in, and the ``task_id``.
-
-        **Operator Creation**: ::
-
-            operator = GKEClusterDeleteOperator(
-                        task_id='cluster_delete',
-                        project_id='my-project',
-                        location='cluster-location'
-                        name='cluster-name')
-
-        .. seealso::
-            For more detail about deleting clusters have a look at the reference:
-            https://google-cloud-python.readthedocs.io/en/latest/container/gapic/v1/api.html#google.cloud.container_v1.ClusterManagerClient.delete_cluster
-
-        :param project_id: The Google Developers Console [project ID or project number]
-        :type project_id: str
-        :param name: The name of the resource to delete, in this case cluster name
-        :type name: str
-        :param location: The name of the Google Compute Engine zone in which the cluster
-            resides.
-        :type location: str
-        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-        :type gcp_conn_id: str
-        :param api_version: The api version to use
-        :type api_version: str
-        """
         super(GKEClusterDeleteOperator, self).__init__(*args, **kwargs)
 
         self.project_id = project_id
@@ -88,61 +92,63 @@ def execute(self, context):
 
 
 class GKEClusterCreateOperator(BaseOperator):
+    """
+    Create a Google Kubernetes Engine Cluster of specified dimensions
+    The operator will wait until the cluster is created.
+
+    The **minimum** required to define a cluster to create is:
+
+    ``dict()`` ::
+        cluster_def = {'name': 'my-cluster-name',
+                       'initial_node_count': 1}
+
+    or
+
+    ``Cluster`` proto ::
+        from google.cloud.container_v1.types import Cluster
+
+        cluster_def = Cluster(name='my-cluster-name', initial_node_count=1)
+
+    **Operator Creation**: ::
+
+        operator = GKEClusterCreateOperator(
+                    task_id='cluster_create',
+                    project_id='my-project',
+                    location='my-location'
+                    body=cluster_def)
+
+    .. seealso::
+        For more detail on about creating clusters have a look at the reference:
+        :class:`google.cloud.container_v1.types.Cluster`
+
+    :param project_id: The Google Developers Console [project ID or project number]
+    :type project_id: str
+    :param location: The name of the Google Compute Engine zone in which the cluster
+        resides.
+    :type location: str
+    :param body: The Cluster definition to create, can be protobuf or python dict, if
+        dict it must match protobuf message Cluster
+    :type body: dict or google.cloud.container_v1.types.Cluster
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: The api version to use
+    :type api_version: str
+    """
     template_fields = ['project_id', 'gcp_conn_id', 'location', 'api_version', 'body']
 
     @apply_defaults
     def __init__(self,
                  project_id,
                  location,
-                 body={},
+                 body=None,
                  gcp_conn_id='google_cloud_default',
                  api_version='v2',
                  *args,
                  **kwargs):
-        """
-        Create a Google Kubernetes Engine Cluster of specified dimensions
-        The operator will wait until the cluster is created.
-
-        The **minimum** required to define a cluster to create is:
-
-        ``dict()`` ::
-            cluster_def = {'name': 'my-cluster-name',
-                           'initial_node_count': 1}
-
-        or
-
-        ``Cluster`` proto ::
-            from google.cloud.container_v1.types import Cluster
-
-            cluster_def = Cluster(name='my-cluster-name', initial_node_count=1)
-
-        **Operator Creation**: ::
-
-            operator = GKEClusterCreateOperator(
-                        task_id='cluster_create',
-                        project_id='my-project',
-                        location='my-location'
-                        body=cluster_def)
-
-        .. seealso::
-            For more detail on about creating clusters have a look at the reference:
-            https://google-cloud-python.readthedocs.io/en/latest/container/gapic/v1/types.html#google.cloud.container_v1.types.Cluster
-
-        :param project_id: The Google Developers Console [project ID or project number]
-        :type project_id: str
-        :param location: The name of the Google Compute Engine zone in which the cluster
-            resides.
-        :type location: str
-        :param body: The Cluster definition to create, can be protobuf or python dict, if
-            dict it must match protobuf message Cluster
-        :type body: dict or google.cloud.container_v1.types.Cluster
-        :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
-        :type gcp_conn_id: str
-        :param api_version: The api version to use
-        :type api_version: str
-        """
         super(GKEClusterCreateOperator, self).__init__(*args, **kwargs)
 
+        if body is None:
+            body = {}
         self.project_id = project_id
         self.gcp_conn_id = gcp_conn_id
         self.location = location
@@ -170,3 +176,147 @@ def execute(self, context):
         hook = GKEClusterHook(self.project_id, self.location)
         create_op = hook.create_cluster(cluster=self.body)
         return create_op
+
+
+KUBE_CONFIG_ENV_VAR = "KUBECONFIG"
+G_APP_CRED = "GOOGLE_APPLICATION_CREDENTIALS"
+
+
+class GKEPodOperator(KubernetesPodOperator):
+    """
+    Executes a task in a Kubernetes pod in the specified Google Kubernetes
+    Engine cluster
+
+    This Operator assumes that the system has gcloud installed and either
+    has working default application credentials or has configured a
+    connection id with a service account.
+
+    The **minimum** required to define a cluster to create are the variables
+    ``task_id``, ``project_id``, ``location``, ``cluster_name``, ``name``,
+    ``namespace``, and ``image``
+
+    **Operator Creation**: ::
+
+        operator = GKEPodOperator(task_id='pod_op',
+                                  project_id='my-project',
+                                  location='us-central1-a',
+                                  cluster_name='my-cluster-name',
+                                  name='task-name',
+                                  namespace='default',
+                                  image='perl')
+
+    .. seealso::
+        For more detail about application authentication have a look at the reference:
+        https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application
+
+    :param project_id: The Google Developers Console project id
+    :type project_id: str
+    :param location: The name of the Google Kubernetes Engine zone in which the
+        cluster resides, e.g. 'us-central1-a'
+    :type location: str
+    :param cluster_name: The name of the Google Kubernetes Engine cluster the pod
+        should be spawned in
+    :type cluster_name: str
+    :param gcp_conn_id: The google cloud connection id to use. This allows for
+        users to specify a service account.
+    :type gcp_conn_id: str
+    """
+    template_fields = ('project_id', 'location',
+                       'cluster_name') + KubernetesPodOperator.template_fields
+
+    @apply_defaults
+    def __init__(self,
+                 project_id,
+                 location,
+                 cluster_name,
+                 gcp_conn_id='google_cloud_default',
+                 *args,
+                 **kwargs):
+        super(GKEPodOperator, self).__init__(*args, **kwargs)
+        self.project_id = project_id
+        self.location = location
+        self.cluster_name = cluster_name
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        # Specifying a service account file allows the user to using non default
+        # authentication for creating a Kubernetes Pod. This is done by setting the
+        # environment variable `GOOGLE_APPLICATION_CREDENTIALS` that gcloud looks at.
+        key_file = None
+
+        # If gcp_conn_id is not specified gcloud will use the default
+        # service account credentials.
+        if self.gcp_conn_id:
+            from airflow.hooks.base_hook import BaseHook
+            # extras is a deserialized json object
+            extras = BaseHook.get_connection(self.gcp_conn_id).extra_dejson
+            # key_file only gets set if a json file is created from a JSON string in
+            # the web ui, else none
+            key_file = self._set_env_from_extras(extras=extras)
+
+        # Write config to a temp file and set the environment variable to point to it.
+        # This is to avoid race conditions of reading/writing a single file
+        with tempfile.NamedTemporaryFile() as conf_file:
+            os.environ[KUBE_CONFIG_ENV_VAR] = conf_file.name
+            # Attempt to get/update credentials
+            # We call gcloud directly instead of using google-cloud-python api
+            # because there is no way to write kubernetes config to a file, which is
+            # required by KubernetesPodOperator.
+            # The gcloud command looks at the env variable `KUBECONFIG` for where to save
+            # the kubernetes config file.
+            subprocess.check_call(
+                ["gcloud", "container", "clusters", "get-credentials",
+                 self.cluster_name,
+                 "--zone", self.location,
+                 "--project", self.project_id])
+
+            # Since the key file is of type mkstemp() closing the file will delete it from
+            # the file system so it cannot be accessed after we don't need it anymore
+            if key_file:
+                key_file.close()
+
+            # Tell `KubernetesPodOperator` where the config file is located
+            self.config_file = os.environ[KUBE_CONFIG_ENV_VAR]
+            super(GKEPodOperator, self).execute(context)
+
+    def _set_env_from_extras(self, extras):
+        """
+        Sets the environment variable `GOOGLE_APPLICATION_CREDENTIALS` with either:
+
+        - The path to the keyfile from the specified connection id
+        - A generated file's path if the user specified JSON in the connection id. The
+            file is assumed to be deleted after the process dies due to how mkstemp()
+            works.
+
+        The environment variable is used inside the gcloud command to determine correct
+        service account to use.
+        """
+        key_path = self._get_field(extras, 'key_path', False)
+        keyfile_json_str = self._get_field(extras, 'keyfile_dict', False)
+
+        if not key_path and not keyfile_json_str:
+            self.log.info('Using gcloud with application default credentials.')
+        elif key_path:
+            os.environ[G_APP_CRED] = key_path
+        else:
+            # Write service account JSON to secure file for gcloud to reference
+            service_key = tempfile.NamedTemporaryFile(delete=False)
+            service_key.write(keyfile_json_str)
+            os.environ[G_APP_CRED] = service_key.name
+            # Return file object to have a pointer to close after use,
+            # thus deleting from file system.
+            return service_key
+
+    def _get_field(self, extras, field, default=None):
+        """
+        Fetches a field from extras, and returns it. This is some Airflow
+        magic. The google_cloud_platform hook type adds custom UI elements
+        to the hook page, which allow admins to specify service_account,
+        key_path, etc. They get formatted as shown below.
+        """
+        long_f = 'extra__google_cloud_platform__{}'.format(field)
+        if long_f in extras:
+            return extras[long_f]
+        else:
+            self.log.info('Field %s not found in extras.', field)
+            return default
diff --git a/airflow/contrib/operators/gcp_function_operator.py b/airflow/contrib/operators/gcp_function_operator.py
new file mode 100644
index 0000000000000..5fc3b82794b80
--- /dev/null
+++ b/airflow/contrib/operators/gcp_function_operator.py
@@ -0,0 +1,328 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import re
+
+from googleapiclient.errors import HttpError
+
+from airflow import AirflowException
+from airflow.contrib.utils.gcp_field_validator import GcpBodyFieldValidator, \
+    GcpFieldValidationException
+from airflow.version import version
+from airflow.models import BaseOperator
+from airflow.contrib.hooks.gcp_function_hook import GcfHook
+from airflow.utils.decorators import apply_defaults
+
+
+def _validate_available_memory_in_mb(value):
+    if int(value) <= 0:
+        raise GcpFieldValidationException("The available memory has to be greater than 0")
+
+
+def _validate_max_instances(value):
+    if int(value) <= 0:
+        raise GcpFieldValidationException(
+            "The max instances parameter has to be greater than 0")
+
+
+CLOUD_FUNCTION_VALIDATION = [
+    dict(name="name", regexp="^.+$"),
+    dict(name="description", regexp="^.+$", optional=True),
+    dict(name="entryPoint", regexp=r'^.+$', optional=True),
+    dict(name="runtime", regexp=r'^.+$', optional=True),
+    dict(name="timeout", regexp=r'^.+$', optional=True),
+    dict(name="availableMemoryMb", custom_validation=_validate_available_memory_in_mb,
+         optional=True),
+    dict(name="labels", optional=True),
+    dict(name="environmentVariables", optional=True),
+    dict(name="network", regexp=r'^.+$', optional=True),
+    dict(name="maxInstances", optional=True, custom_validation=_validate_max_instances),
+
+    dict(name="source_code", type="union", fields=[
+        dict(name="sourceArchiveUrl", regexp=r'^.+$'),
+        dict(name="sourceRepositoryUrl", regexp=r'^.+$', api_version='v1beta2'),
+        dict(name="sourceRepository", type="dict", fields=[
+            dict(name="url", regexp=r'^.+$')
+        ]),
+        dict(name="sourceUploadUrl")
+    ]),
+
+    dict(name="trigger", type="union", fields=[
+        dict(name="httpsTrigger", type="dict", fields=[
+            # This dict should be empty at input (url is added at output)
+        ]),
+        dict(name="eventTrigger", type="dict", fields=[
+            dict(name="eventType", regexp=r'^.+$'),
+            dict(name="resource", regexp=r'^.+$'),
+            dict(name="service", regexp=r'^.+$', optional=True),
+            dict(name="failurePolicy", type="dict", optional=True, fields=[
+                dict(name="retry", type="dict", optional=True)
+            ])
+        ])
+    ]),
+]
+
+
+class GcfFunctionDeployOperator(BaseOperator):
+    """
+    Creates a function in Google Cloud Functions.
+    If a function with this name already exists, it will be updated.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcfFunctionDeployOperator`
+
+    :param location: Google Cloud Platform region where the function should be created.
+    :type location: str
+    :param body: Body of the Cloud Functions definition. The body must be a
+        Cloud Functions dictionary as described in:
+        https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions
+        . Different API versions require different variants of the Cloud Functions
+        dictionary.
+    :type body: dict or google.cloud.functions.v1.CloudFunction
+    :param project_id: (Optional) Google Cloud Platform project ID where the function
+        should be created.
+    :type project_id: str
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud
+         Platform - default 'google_cloud_default'.
+    :type gcp_conn_id: str
+    :param api_version: (Optional) API version used (for example v1 - default -  or
+        v1beta1).
+    :type api_version: str
+    :param zip_path: Path to zip file containing source code of the function. If the path
+        is set, the sourceUploadUrl should not be specified in the body or it should
+        be empty. Then the zip file will be uploaded using the upload URL generated
+        via generateUploadUrl from the Cloud Functions API.
+    :type zip_path: str
+    :param validate_body: If set to False, body validation is not performed.
+    :type validate_body: bool
+    """
+    # [START gcf_function_deploy_template_fields]
+    template_fields = ('project_id', 'location', 'gcp_conn_id', 'api_version')
+    # [END gcf_function_deploy_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 location,
+                 body,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 zip_path=None,
+                 validate_body=True,
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.location = location
+        self.body = body
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self.zip_path = zip_path
+        self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path)
+        self._field_validator = None
+        if validate_body:
+            self._field_validator = GcpBodyFieldValidator(CLOUD_FUNCTION_VALIDATION,
+                                                          api_version=api_version)
+        self._hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version)
+        self._validate_inputs()
+        super(GcfFunctionDeployOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if not self.location:
+            raise AirflowException("The required parameter 'location' is missing")
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is missing")
+        self.zip_path_preprocessor.preprocess_body()
+
+    def _validate_all_body_fields(self):
+        if self._field_validator:
+            self._field_validator.validate(self.body)
+
+    def _create_new_function(self):
+        self._hook.create_new_function(
+            project_id=self.project_id,
+            location=self.location,
+            body=self.body)
+
+    def _update_function(self):
+        self._hook.update_function(self.body['name'], self.body, self.body.keys())
+
+    def _check_if_function_exists(self):
+        name = self.body.get('name')
+        if not name:
+            raise GcpFieldValidationException("The 'name' field should be present in "
+                                              "body: '{}'.".format(self.body))
+        try:
+            self._hook.get_function(name)
+        except HttpError as e:
+            status = e.resp.status
+            if status == 404:
+                return False
+            raise e
+        return True
+
+    def _upload_source_code(self):
+        return self._hook.upload_function_zip(project_id=self.project_id,
+                                              location=self.location,
+                                              zip_path=self.zip_path)
+
+    def _set_airflow_version_label(self):
+        if 'labels' not in self.body.keys():
+            self.body['labels'] = {}
+        self.body['labels'].update(
+            {'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')})
+
+    def execute(self, context):
+        if self.zip_path_preprocessor.should_upload_function():
+            self.body[GCF_SOURCE_UPLOAD_URL] = self._upload_source_code()
+        self._validate_all_body_fields()
+        self._set_airflow_version_label()
+        if not self._check_if_function_exists():
+            self._create_new_function()
+        else:
+            self._update_function()
+
+
+GCF_SOURCE_ARCHIVE_URL = 'sourceArchiveUrl'
+GCF_SOURCE_UPLOAD_URL = 'sourceUploadUrl'
+SOURCE_REPOSITORY = 'sourceRepository'
+GCF_ZIP_PATH = 'zip_path'
+
+
+class ZipPathPreprocessor:
+    """
+    Pre-processes zip path parameter.
+
+    Responsible for checking if the zip path parameter is correctly specified in
+    relation with source_code body fields. Non empty zip path parameter is special because
+    it is mutually exclusive with sourceArchiveUrl and sourceRepository body fields.
+    It is also mutually exclusive with non-empty sourceUploadUrl.
+    The pre-process modifies sourceUploadUrl body field in special way when zip_path
+    is not empty. An extra step is run when execute method is called and sourceUploadUrl
+    field value is set in the body with the value returned by generateUploadUrl Cloud
+    Function API method.
+
+    :param body: Body passed to the create/update method calls.
+    :type body: dict
+    :param zip_path: path to the zip file containing source code.
+    :type body: dict
+
+    """
+    upload_function = None
+
+    def __init__(self, body, zip_path):
+        self.body = body
+        self.zip_path = zip_path
+
+    @staticmethod
+    def _is_present_and_empty(dictionary, field):
+        return field in dictionary and not dictionary[field]
+
+    def _verify_upload_url_and_no_zip_path(self):
+        if self._is_present_and_empty(self.body, GCF_SOURCE_UPLOAD_URL):
+            if not self.zip_path:
+                raise AirflowException(
+                    "Parameter '{}' is empty in the body and argument '{}' "
+                    "is missing or empty. You need to have non empty '{}' "
+                    "when '{}' is present and empty.".
+                    format(GCF_SOURCE_UPLOAD_URL, GCF_ZIP_PATH, GCF_ZIP_PATH, GCF_SOURCE_UPLOAD_URL))
+
+    def _verify_upload_url_and_zip_path(self):
+        if GCF_SOURCE_UPLOAD_URL in self.body and self.zip_path:
+            if not self.body[GCF_SOURCE_UPLOAD_URL]:
+                self.upload_function = True
+            else:
+                raise AirflowException("Only one of '{}' in body or '{}' argument "
+                                       "allowed. Found both."
+                                       .format(GCF_SOURCE_UPLOAD_URL, GCF_ZIP_PATH))
+
+    def _verify_archive_url_and_zip_path(self):
+        if GCF_SOURCE_ARCHIVE_URL in self.body and self.zip_path:
+            raise AirflowException("Only one of '{}' in body or '{}' argument "
+                                   "allowed. Found both."
+                                   .format(GCF_SOURCE_ARCHIVE_URL, GCF_ZIP_PATH))
+
+    def should_upload_function(self):
+        if self.upload_function is None:
+            raise AirflowException('validate() method has to be invoked before '
+                                   'should_upload_function')
+        return self.upload_function
+
+    def preprocess_body(self):
+        self._verify_archive_url_and_zip_path()
+        self._verify_upload_url_and_zip_path()
+        self._verify_upload_url_and_no_zip_path()
+        if self.upload_function is None:
+            self.upload_function = False
+
+
+FUNCTION_NAME_PATTERN = '^projects/[^/]+/locations/[^/]+/functions/[^/]+$'
+FUNCTION_NAME_COMPILED_PATTERN = re.compile(FUNCTION_NAME_PATTERN)
+
+
+class GcfFunctionDeleteOperator(BaseOperator):
+    """
+    Deletes the specified function from Google Cloud Functions.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcfFunctionDeleteOperator`
+
+    :param name: A fully-qualified function name, matching
+        the pattern: `^projects/[^/]+/locations/[^/]+/functions/[^/]+$`
+    :type name: str
+    :param gcp_conn_id: The connection ID to use to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (for example v1 or v1beta1).
+    :type api_version: str
+    """
+    # [START gcf_function_delete_template_fields]
+    template_fields = ('name', 'gcp_conn_id', 'api_version')
+    # [END gcf_function_delete_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 name,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1',
+                 *args, **kwargs):
+        self.name = name
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+        self.hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version)
+        super(GcfFunctionDeleteOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if not self.name:
+            raise AttributeError('Empty parameter: name')
+        else:
+            pattern = FUNCTION_NAME_COMPILED_PATTERN
+            if not pattern.match(self.name):
+                raise AttributeError(
+                    'Parameter name must match pattern: {}'.format(FUNCTION_NAME_PATTERN))
+
+    def execute(self, context):
+        try:
+            return self.hook.delete_function(self.name)
+        except HttpError as e:
+            status = e.resp.status
+            if status == 404:
+                self.log.info('The function does not exist in this project')
+            else:
+                self.log.error('An error occurred. Exiting.')
+                raise e
diff --git a/airflow/contrib/operators/gcp_natural_language_operator.py b/airflow/contrib/operators/gcp_natural_language_operator.py
new file mode 100644
index 0000000000000..0d47ea461ab10
--- /dev/null
+++ b/airflow/contrib/operators/gcp_natural_language_operator.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from google.protobuf.json_format import MessageToDict
+
+from airflow.contrib.hooks.gcp_natural_language_hook import CloudNaturalLanguageHook
+from airflow.models import BaseOperator
+
+
+class CloudLanguageAnalyzeEntitiesOperator(BaseOperator):
+    """
+    Finds named entities in the text along with entity types,
+    salience, mentions for each entity, and other properties.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudLanguageAnalyzeEntitiesOperator`
+
+    :param document: Input document.
+        If a dict is provided, it must be of the same form as the protobuf message Document
+    :type document: dict or google.cloud.language_v1.types.Document
+    :param encoding_type: The encoding type used by the API to calculate offsets.
+    :type encoding_type: google.cloud.language_v1.types.EncodingType
+    :param retry: A retry object used to retry requests. If None is specified, requests will not be
+        retried.
+    :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+        retry is specified, the timeout applies to each individual attempt.
+    :type timeout: float
+    :param metadata: Additional metadata that is provided to the method.
+    :type metadata: seq[tuple[str, str]]]
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START natural_langauge_analyze_entities_template_fields]
+    template_fields = ("document", "gcp_conn_id")
+    # [END natural_langauge_analyze_entities_template_fields]
+
+    def __init__(
+        self,
+        document,
+        encoding_type=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudLanguageAnalyzeEntitiesOperator, self).__init__(*args, **kwargs)
+        self.document = document
+        self.encoding_type = encoding_type
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudNaturalLanguageHook(gcp_conn_id=self.gcp_conn_id)
+
+        self.log.info("Start analyzing entities")
+        response = hook.analyze_entities(
+            document=self.document, retry=self.retry, timeout=self.timeout, metadata=self.metadata
+        )
+        self.log.info("Finished analyzing entities")
+
+        return MessageToDict(response)
+
+
+class CloudLanguageAnalyzeEntitySentimentOperator(BaseOperator):
+    """
+    Finds entities, similar to AnalyzeEntities in the text and analyzes sentiment associated with each
+    entity and its mentions.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudLanguageAnalyzeEntitySentimentOperator`
+
+    :param document: Input document.
+        If a dict is provided, it must be of the same form as the protobuf message Document
+    :type document: dict or google.cloud.language_v1.types.Document
+    :param encoding_type: The encoding type used by the API to calculate offsets.
+    :type encoding_type: google.cloud.language_v1.types.EncodingType
+    :param retry: A retry object used to retry requests. If None is specified, requests will not be
+        retried.
+    :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+        retry is specified, the timeout applies to each individual attempt.
+    :type timeout: float
+    :param metadata: Additional metadata that is provided to the method.
+    :type metadata: seq[tuple[str, str]]]
+    :rtype: google.cloud.language_v1.types.AnalyzeEntitiesResponse
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START natural_langauge_analyze_entity_sentiment_template_fields]
+    template_fields = ("document", "gcp_conn_id")
+    # [END natural_langauge_analyze_entity_sentiment_template_fields]
+
+    def __init__(
+        self,
+        document,
+        encoding_type=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudLanguageAnalyzeEntitySentimentOperator, self).__init__(*args, **kwargs)
+        self.document = document
+        self.encoding_type = encoding_type
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudNaturalLanguageHook(gcp_conn_id=self.gcp_conn_id)
+
+        self.log.info("Start entity sentiment analyze")
+        response = hook.analyze_entity_sentiment(
+            document=self.document,
+            encoding_type=self.encoding_type,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+        self.log.info("Finished entity sentiment analyze")
+
+        return MessageToDict(response)
+
+
+class CloudLanguageAnalyzeSentimentOperator(BaseOperator):
+    """
+    Analyzes the sentiment of the provided text.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudLanguageAnalyzeSentimentOperator`
+
+    :param document: Input document.
+        If a dict is provided, it must be of the same form as the protobuf message Document
+    :type document: dict or google.cloud.language_v1.types.Document
+    :param encoding_type: The encoding type used by the API to calculate offsets.
+    :type encoding_type: google.cloud.language_v1.types.EncodingType
+    :param retry: A retry object used to retry requests. If None is specified, requests will not be
+        retried.
+    :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+        retry is specified, the timeout applies to each individual attempt.
+    :type timeout: float
+    :param metadata: Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]]
+    :rtype: google.cloud.language_v1.types.AnalyzeEntitiesResponse
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START natural_langauge_analyze_sentiment_template_fields]
+    template_fields = ("document", "gcp_conn_id")
+    # [END natural_langauge_analyze_sentiment_template_fields]
+
+    def __init__(
+        self,
+        document,
+        encoding_type=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudLanguageAnalyzeSentimentOperator, self).__init__(*args, **kwargs)
+        self.document = document
+        self.encoding_type = encoding_type
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudNaturalLanguageHook(gcp_conn_id=self.gcp_conn_id)
+
+        self.log.info("Start sentiment analyze")
+        response = hook.analyze_sentiment(
+            document=self.document, retry=self.retry, timeout=self.timeout, metadata=self.metadata
+        )
+        self.log.info("Finished sentiment analyze")
+
+        return MessageToDict(response)
+
+
+class CloudLanguageClassifyTextOperator(BaseOperator):
+    """
+    Classifies a document into categories.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudLanguageClassifyTextOperator`
+
+    :param document: Input document.
+        If a dict is provided, it must be of the same form as the protobuf message Document
+    :type document: dict or google.cloud.language_v1.types.Document
+    :param retry: A retry object used to retry requests. If None is specified, requests will not be
+        retried.
+    :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
+        retry is specified, the timeout applies to each individual attempt.
+    :type timeout: float
+    :param metadata: Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]]
+    :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START natural_langauge_classify_text_template_fields]
+    template_fields = ("document", "gcp_conn_id")
+    # [END natural_langauge_classify_text_template_fields]
+
+    def __init__(
+        self,
+        document,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudLanguageClassifyTextOperator, self).__init__(*args, **kwargs)
+        self.document = document
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudNaturalLanguageHook(gcp_conn_id=self.gcp_conn_id)
+
+        self.log.info("Start text classify")
+        response = hook.classify_text(
+            document=self.document, retry=self.retry, timeout=self.timeout, metadata=self.metadata
+        )
+        self.log.info("Finished text classify")
+
+        return MessageToDict(response)
diff --git a/airflow/contrib/operators/gcp_spanner_operator.py b/airflow/contrib/operators/gcp_spanner_operator.py
new file mode 100644
index 0000000000000..bea4f676e3ccc
--- /dev/null
+++ b/airflow/contrib/operators/gcp_spanner_operator.py
@@ -0,0 +1,432 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import six
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_spanner_hook import CloudSpannerHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class CloudSpannerInstanceDeployOperator(BaseOperator):
+    """
+    Creates a new Cloud Spanner instance, or if an instance with the same instance_id
+    exists in the specified project, updates the Cloud Spanner instance.
+
+    :param instance_id: Cloud Spanner instance ID.
+    :type instance_id: str
+    :param configuration_name:  The name of the Cloud Spanner instance configuration
+      defining how the instance will be created. Required for
+      instances that do not yet exist.
+    :type configuration_name: str
+    :param node_count: (Optional) The number of nodes allocated to the Cloud Spanner
+      instance.
+    :type node_count: int
+    :param display_name: (Optional) The display name for the Cloud Spanner  instance in
+      the GCP Console. (Must be between 4 and 30 characters.) If this value is not set
+      in the constructor, the name is the same as the instance ID.
+    :type display_name: str
+    :param project_id: Optional, the ID of the project which owns the Cloud Spanner
+        Database.  If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+    # [START gcp_spanner_deploy_template_fields]
+    template_fields = ('project_id', 'instance_id', 'configuration_name', 'display_name',
+                       'gcp_conn_id')
+    # [END gcp_spanner_deploy_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 configuration_name,
+                 node_count,
+                 display_name,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        self.instance_id = instance_id
+        self.project_id = project_id
+        self.configuration_name = configuration_name
+        self.node_count = node_count
+        self.display_name = display_name
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+        self._hook = CloudSpannerHook(gcp_conn_id=gcp_conn_id)
+        super(CloudSpannerInstanceDeployOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance_id:
+            raise AirflowException("The required parameter 'instance_id' "
+                                   "is empty or None")
+
+    def execute(self, context):
+        if not self._hook.get_instance(project_id=self.project_id, instance_id=self.instance_id):
+            self.log.info("Creating Cloud Spanner instance '%s'", self.instance_id)
+            func = self._hook.create_instance
+        else:
+            self.log.info("Updating Cloud Spanner instance '%s'", self.instance_id)
+            func = self._hook.update_instance
+        func(project_id=self.project_id,
+             instance_id=self.instance_id,
+             configuration_name=self.configuration_name,
+             node_count=self.node_count,
+             display_name=self.display_name)
+
+
+class CloudSpannerInstanceDeleteOperator(BaseOperator):
+    """
+    Deletes a Cloud Spanner instance. If an instance does not exist,
+    no action is taken and the operator succeeds.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSpannerInstanceDeleteOperator`
+
+    :param instance_id: The Cloud Spanner instance ID.
+    :type instance_id: str
+    :param project_id: Optional, the ID of the project that owns the Cloud Spanner
+        Database.  If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+    # [START gcp_spanner_delete_template_fields]
+    template_fields = ('project_id', 'instance_id', 'gcp_conn_id')
+    # [END gcp_spanner_delete_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        self.instance_id = instance_id
+        self.project_id = project_id
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+        self._hook = CloudSpannerHook(gcp_conn_id=gcp_conn_id)
+        super(CloudSpannerInstanceDeleteOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance_id:
+            raise AirflowException("The required parameter 'instance_id' "
+                                   "is empty or None")
+
+    def execute(self, context):
+        if self._hook.get_instance(project_id=self.project_id, instance_id=self.instance_id):
+            return self._hook.delete_instance(project_id=self.project_id,
+                                              instance_id=self.instance_id)
+        else:
+            self.log.info("Instance '%s' does not exist in project '%s'. "
+                          "Aborting delete.", self.instance_id, self.project_id)
+            return True
+
+
+class CloudSpannerInstanceDatabaseQueryOperator(BaseOperator):
+    """
+    Executes an arbitrary DML query (INSERT, UPDATE, DELETE).
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSpannerInstanceDatabaseQueryOperator`
+
+    :param instance_id: The Cloud Spanner instance ID.
+    :type instance_id: str
+    :param database_id: The Cloud Spanner database ID.
+    :type database_id: str
+    :param query: The query or list of queries to be executed. Can be a path to a SQL
+       file.
+    :type query: str or list
+    :param project_id: Optional, the ID of the project that owns the Cloud Spanner
+        Database.  If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+    # [START gcp_spanner_query_template_fields]
+    template_fields = ('project_id', 'instance_id', 'database_id', 'query', 'gcp_conn_id')
+    template_ext = ('.sql',)
+    # [END gcp_spanner_query_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 database_id,
+                 query,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        self.instance_id = instance_id
+        self.project_id = project_id
+        self.database_id = database_id
+        self.query = query
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+        self._hook = CloudSpannerHook(gcp_conn_id=gcp_conn_id)
+        super(CloudSpannerInstanceDatabaseQueryOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance_id:
+            raise AirflowException("The required parameter 'instance_id' "
+                                   "is empty or None")
+        if not self.database_id:
+            raise AirflowException("The required parameter 'database_id' "
+                                   "is empty or None")
+        if not self.query:
+            raise AirflowException("The required parameter 'query' is empty")
+
+    def execute(self, context):
+        queries = self.query
+        if isinstance(self.query, six.string_types):
+            queries = [x.strip() for x in self.query.split(';')]
+            self.sanitize_queries(queries)
+        self.log.info("Executing DML query(-ies) on "
+                      "projects/%s/instances/%s/databases/%s",
+                      self.project_id, self.instance_id, self.database_id)
+        self.log.info(queries)
+        self._hook.execute_dml(project_id=self.project_id,
+                               instance_id=self.instance_id,
+                               database_id=self.database_id,
+                               queries=queries)
+
+    @staticmethod
+    def sanitize_queries(queries):
+        if len(queries) and queries[-1] == '':
+            del queries[-1]
+
+
+class CloudSpannerInstanceDatabaseDeployOperator(BaseOperator):
+    """
+    Creates a new Cloud Spanner database, or if database exists,
+    the operator does nothing.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSpannerInstanceDatabaseDeployOperator`
+
+    :param instance_id: The Cloud Spanner instance ID.
+    :type instance_id: str
+    :param database_id: The Cloud Spanner database ID.
+    :type database_id: str
+    :param ddl_statements: The string list containing DDL for the new database.
+    :type ddl_statements: list[str]
+    :param project_id: Optional, the ID of the project that owns the Cloud Spanner
+        Database.  If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+    # [START gcp_spanner_database_deploy_template_fields]
+    template_fields = ('project_id', 'instance_id', 'database_id', 'ddl_statements',
+                       'gcp_conn_id')
+    template_ext = ('.sql', )
+    # [END gcp_spanner_database_deploy_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 database_id,
+                 ddl_statements,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        self.instance_id = instance_id
+        self.project_id = project_id
+        self.database_id = database_id
+        self.ddl_statements = ddl_statements
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+        self._hook = CloudSpannerHook(gcp_conn_id=gcp_conn_id)
+        super(CloudSpannerInstanceDatabaseDeployOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance_id:
+            raise AirflowException("The required parameter 'instance_id' is empty "
+                                   "or None")
+        if not self.database_id:
+            raise AirflowException("The required parameter 'database_id' is empty"
+                                   " or None")
+
+    def execute(self, context):
+        if not self._hook.get_database(project_id=self.project_id,
+                                       instance_id=self.instance_id,
+                                       database_id=self.database_id):
+            self.log.info("Creating Cloud Spanner database "
+                          "'%s' in project '%s' and instance '%s'",
+                          self.database_id, self.project_id, self.instance_id)
+            return self._hook.create_database(project_id=self.project_id,
+                                              instance_id=self.instance_id,
+                                              database_id=self.database_id,
+                                              ddl_statements=self.ddl_statements)
+        else:
+            self.log.info("The database '%s' in project '%s' and instance '%s'"
+                          " already exists. Nothing to do. Exiting.",
+                          self.database_id, self.project_id, self.instance_id)
+        return True
+
+
+class CloudSpannerInstanceDatabaseUpdateOperator(BaseOperator):
+    """
+    Updates a Cloud Spanner database with the specified DDL statement.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSpannerInstanceDatabaseUpdateOperator`
+
+    :param instance_id: The Cloud Spanner instance ID.
+    :type instance_id: str
+    :param database_id: The Cloud Spanner database ID.
+    :type database_id: str
+    :param ddl_statements: The string list containing DDL to apply to the database.
+    :type ddl_statements: list[str]
+    :param project_id: Optional, the ID of the project that owns the the Cloud Spanner
+        Database.  If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param operation_id: (Optional) Unique per database operation id that can
+           be specified to implement idempotency check.
+    :type operation_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+    # [START gcp_spanner_database_update_template_fields]
+    template_fields = ('project_id', 'instance_id', 'database_id', 'ddl_statements',
+                       'gcp_conn_id')
+    template_ext = ('.sql', )
+    # [END gcp_spanner_database_update_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 database_id,
+                 ddl_statements,
+                 project_id=None,
+                 operation_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        self.instance_id = instance_id
+        self.project_id = project_id
+        self.database_id = database_id
+        self.ddl_statements = ddl_statements
+        self.operation_id = operation_id
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+        self._hook = CloudSpannerHook(gcp_conn_id=gcp_conn_id)
+        super(CloudSpannerInstanceDatabaseUpdateOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance_id:
+            raise AirflowException("The required parameter 'instance_id' is empty"
+                                   " or None")
+        if not self.database_id:
+            raise AirflowException("The required parameter 'database_id' is empty"
+                                   " or None")
+        if not self.ddl_statements:
+            raise AirflowException("The required parameter 'ddl_statements' is empty"
+                                   " or None")
+
+    def execute(self, context):
+        if not self._hook.get_database(project_id=self.project_id,
+                                       instance_id=self.instance_id,
+                                       database_id=self.database_id):
+            raise AirflowException("The Cloud Spanner database '{}' in project '{}' and "
+                                   "instance '{}' is missing. Create the database first "
+                                   "before you can update it.".format(self.database_id,
+                                                                      self.project_id,
+                                                                      self.instance_id))
+        else:
+            return self._hook.update_database(project_id=self.project_id,
+                                              instance_id=self.instance_id,
+                                              database_id=self.database_id,
+                                              ddl_statements=self.ddl_statements,
+                                              operation_id=self.operation_id)
+
+
+class CloudSpannerInstanceDatabaseDeleteOperator(BaseOperator):
+    """
+    Deletes a Cloud Spanner database.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSpannerInstanceDatabaseDeleteOperator`
+
+    :param instance_id: Cloud Spanner instance ID.
+    :type instance_id: str
+    :param database_id: Cloud Spanner database ID.
+    :type database_id: str
+    :param project_id: Optional, the ID of the project that owns the Cloud Spanner
+        Database.  If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+    # [START gcp_spanner_database_delete_template_fields]
+    template_fields = ('project_id', 'instance_id', 'database_id',
+                       'gcp_conn_id')
+    # [END gcp_spanner_database_delete_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance_id,
+                 database_id,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        self.instance_id = instance_id
+        self.project_id = project_id
+        self.database_id = database_id
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+        self._hook = CloudSpannerHook(gcp_conn_id=gcp_conn_id)
+        super(CloudSpannerInstanceDatabaseDeleteOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance_id:
+            raise AirflowException("The required parameter 'instance_id' is empty"
+                                   " or None")
+        if not self.database_id:
+            raise AirflowException("The required parameter 'database_id' is empty"
+                                   " or None")
+
+    def execute(self, context):
+        db = self._hook.get_database(project_id=self.project_id,
+                                     instance_id=self.instance_id,
+                                     database_id=self.database_id)
+        if not db:
+            self.log.info("The Cloud Spanner database was missing: "
+                          "'%s' in project '%s' and instance '%s'. Assuming success.",
+                          self.database_id, self.project_id, self.instance_id)
+            return True
+        else:
+            return self._hook.delete_database(project_id=self.project_id,
+                                              instance_id=self.instance_id,
+                                              database_id=self.database_id)
diff --git a/airflow/contrib/operators/gcp_sql_operator.py b/airflow/contrib/operators/gcp_sql_operator.py
new file mode 100644
index 0000000000000..cdce4baa7591b
--- /dev/null
+++ b/airflow/contrib/operators/gcp_sql_operator.py
@@ -0,0 +1,805 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from googleapiclient.errors import HttpError
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_sql_hook import CloudSqlHook, CloudSqlDatabaseHook
+from airflow.contrib.utils.gcp_field_validator import GcpBodyFieldValidator
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.hooks.base_hook import BaseHook
+
+SETTINGS = 'settings'
+SETTINGS_VERSION = 'settingsVersion'
+
+CLOUD_SQL_CREATE_VALIDATION = [
+    dict(name="name", allow_empty=False),
+    dict(name="settings", type="dict", fields=[
+        dict(name="tier", allow_empty=False),
+        dict(name="backupConfiguration", type="dict", fields=[
+            dict(name="binaryLogEnabled", optional=True),
+            dict(name="enabled", optional=True),
+            dict(name="replicationLogArchivingEnabled", optional=True),
+            dict(name="startTime", allow_empty=False, optional=True)
+        ], optional=True),
+        dict(name="activationPolicy", allow_empty=False, optional=True),
+        dict(name="authorizedGaeApplications", type="list", optional=True),
+        dict(name="crashSafeReplicationEnabled", optional=True),
+        dict(name="dataDiskSizeGb", optional=True),
+        dict(name="dataDiskType", allow_empty=False, optional=True),
+        dict(name="databaseFlags", type="list", optional=True),
+        dict(name="ipConfiguration", type="dict", fields=[
+            dict(name="authorizedNetworks", type="list", fields=[
+                dict(name="expirationTime", optional=True),
+                dict(name="name", allow_empty=False, optional=True),
+                dict(name="value", allow_empty=False, optional=True)
+            ], optional=True),
+            dict(name="ipv4Enabled", optional=True),
+            dict(name="privateNetwork", allow_empty=False, optional=True),
+            dict(name="requireSsl", optional=True),
+        ], optional=True),
+        dict(name="locationPreference", type="dict", fields=[
+            dict(name="followGaeApplication", allow_empty=False, optional=True),
+            dict(name="zone", allow_empty=False, optional=True),
+        ], optional=True),
+        dict(name="maintenanceWindow", type="dict", fields=[
+            dict(name="hour", optional=True),
+            dict(name="day", optional=True),
+            dict(name="updateTrack", allow_empty=False, optional=True),
+        ], optional=True),
+        dict(name="pricingPlan", allow_empty=False, optional=True),
+        dict(name="replicationType", allow_empty=False, optional=True),
+        dict(name="storageAutoResize", optional=True),
+        dict(name="storageAutoResizeLimit", optional=True),
+        dict(name="userLabels", type="dict", optional=True),
+    ]),
+    dict(name="databaseVersion", allow_empty=False, optional=True),
+    dict(name="failoverReplica", type="dict", fields=[
+        dict(name="name", allow_empty=False)
+    ], optional=True),
+    dict(name="masterInstanceName", allow_empty=False, optional=True),
+    dict(name="onPremisesConfiguration", type="dict", optional=True),
+    dict(name="region", allow_empty=False, optional=True),
+    dict(name="replicaConfiguration", type="dict", fields=[
+        dict(name="failoverTarget", optional=True),
+        dict(name="mysqlReplicaConfiguration", type="dict", fields=[
+            dict(name="caCertificate", allow_empty=False, optional=True),
+            dict(name="clientCertificate", allow_empty=False, optional=True),
+            dict(name="clientKey", allow_empty=False, optional=True),
+            dict(name="connectRetryInterval", optional=True),
+            dict(name="dumpFilePath", allow_empty=False, optional=True),
+            dict(name="masterHeartbeatPeriod", optional=True),
+            dict(name="password", allow_empty=False, optional=True),
+            dict(name="sslCipher", allow_empty=False, optional=True),
+            dict(name="username", allow_empty=False, optional=True),
+            dict(name="verifyServerCertificate", optional=True)
+        ], optional=True),
+    ], optional=True)
+]
+CLOUD_SQL_EXPORT_VALIDATION = [
+    dict(name="exportContext", type="dict", fields=[
+        dict(name="fileType", allow_empty=False),
+        dict(name="uri", allow_empty=False),
+        dict(name="databases", optional=True, type="list"),
+        dict(name="sqlExportOptions", type="dict", optional=True, fields=[
+            dict(name="tables", optional=True, type="list"),
+            dict(name="schemaOnly", optional=True)
+        ]),
+        dict(name="csvExportOptions", type="dict", optional=True, fields=[
+            dict(name="selectQuery")
+        ])
+    ])
+]
+CLOUD_SQL_IMPORT_VALIDATION = [
+    dict(name="importContext", type="dict", fields=[
+        dict(name="fileType", allow_empty=False),
+        dict(name="uri", allow_empty=False),
+        dict(name="database", optional=True, allow_empty=False),
+        dict(name="importUser", optional=True),
+        dict(name="csvImportOptions", type="dict", optional=True, fields=[
+            dict(name="table"),
+            dict(name="columns", type="list", optional=True)
+        ])
+    ])
+]
+CLOUD_SQL_DATABASE_CREATE_VALIDATION = [
+    dict(name="instance", allow_empty=False),
+    dict(name="name", allow_empty=False),
+    dict(name="project", allow_empty=False),
+]
+CLOUD_SQL_DATABASE_PATCH_VALIDATION = [
+    dict(name="instance", optional=True),
+    dict(name="name", optional=True),
+    dict(name="project", optional=True),
+    dict(name="etag", optional=True),
+    dict(name="charset", optional=True),
+    dict(name="collation", optional=True),
+]
+
+
+class CloudSqlBaseOperator(BaseOperator):
+    """
+    Abstract base operator for Google Cloud SQL operators to inherit from.
+
+    :param instance: Cloud SQL instance ID. This does not include the project ID.
+    :type instance: str
+    :param project_id: Optional, Google Cloud Platform Project ID.  f set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    """
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 *args, **kwargs):
+        self.project_id = project_id
+        self.instance = instance
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+        self._hook = CloudSqlHook(gcp_conn_id=self.gcp_conn_id,
+                                  api_version=self.api_version)
+        super(CloudSqlBaseOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if self.project_id == '':
+            raise AirflowException("The required parameter 'project_id' is empty")
+        if not self.instance:
+            raise AirflowException("The required parameter 'instance' is empty or None")
+
+    def _check_if_instance_exists(self, instance):
+        try:
+            return self._hook.get_instance(project_id=self.project_id,
+                                           instance=instance)
+        except HttpError as e:
+            status = e.resp.status
+            if status == 404:
+                return False
+            raise e
+
+    def _check_if_db_exists(self, db_name):
+        try:
+            return self._hook.get_database(
+                project_id=self.project_id,
+                instance=self.instance,
+                database=db_name)
+        except HttpError as e:
+            status = e.resp.status
+            if status == 404:
+                return False
+            raise e
+
+    def execute(self, context):
+        pass
+
+    @staticmethod
+    def _get_settings_version(instance):
+        return instance.get(SETTINGS).get(SETTINGS_VERSION)
+
+
+class CloudSqlInstanceCreateOperator(CloudSqlBaseOperator):
+    """
+    Creates a new Cloud SQL instance.
+    If an instance with the same name exists, no action will be taken and
+    the operator will succeed.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceCreateOperator`
+
+    :param body: Body required by the Cloud SQL insert API, as described in
+        https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert
+        #request-body
+    :type body: dict
+    :param instance: Cloud SQL instance ID. This does not include the project ID.
+    :type instance: str
+    :param project_id: Optional, Google Cloud Platform Project ID. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    :param validate_body: True if body should be validated, False otherwise.
+    :type validate_body: bool
+    """
+    # [START gcp_sql_create_template_fields]
+    template_fields = ('project_id', 'instance', 'gcp_conn_id', 'api_version')
+    # [END gcp_sql_create_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 body,
+                 instance,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.body = body
+        self.validate_body = validate_body
+        super(CloudSqlInstanceCreateOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstanceCreateOperator, self)._validate_inputs()
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty")
+
+    def _validate_body_fields(self):
+        if self.validate_body:
+            GcpBodyFieldValidator(CLOUD_SQL_CREATE_VALIDATION,
+                                  api_version=self.api_version).validate(self.body)
+
+    def execute(self, context):
+        self._validate_body_fields()
+        if not self._check_if_instance_exists(self.instance):
+            self._hook.create_instance(
+                project_id=self.project_id,
+                body=self.body)
+        else:
+            self.log.info("Cloud SQL instance with ID {} already exists. "
+                          "Aborting create.".format(self.instance))
+
+        instance_resource = self._hook.get_instance(project_id=self.project_id,
+                                                    instance=self.instance)
+        service_account_email = instance_resource["serviceAccountEmailAddress"]
+        task_instance = context['task_instance']
+        task_instance.xcom_push(key="service_account_email", value=service_account_email)
+
+
+class CloudSqlInstancePatchOperator(CloudSqlBaseOperator):
+    """
+    Updates settings of a Cloud SQL instance.
+
+    Caution: This is a partial update, so only included values for the settings will be
+    updated.
+
+    In the request body, supply the relevant portions of an instance resource, according
+    to the rules of patch semantics.
+    https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstancePatchOperator`
+
+    :param body: Body required by the Cloud SQL patch API, as described in
+        https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch#request-body
+    :type body: dict
+    :param instance: Cloud SQL instance ID. This does not include the project ID.
+    :type instance: str
+    :param project_id: Optional, Google Cloud Platform Project ID.  If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    """
+    # [START gcp_sql_patch_template_fields]
+    template_fields = ('project_id', 'instance', 'gcp_conn_id', 'api_version')
+    # [END gcp_sql_patch_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 body,
+                 instance,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 *args, **kwargs):
+        self.body = body
+        super(CloudSqlInstancePatchOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstancePatchOperator, self)._validate_inputs()
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty")
+
+    def execute(self, context):
+        if not self._check_if_instance_exists(self.instance):
+            raise AirflowException('Cloud SQL instance with ID {} does not exist. '
+                                   'Please specify another instance to patch.'
+                                   .format(self.instance))
+        else:
+            return self._hook.patch_instance(
+                project_id=self.project_id,
+                body=self.body,
+                instance=self.instance)
+
+
+class CloudSqlInstanceDeleteOperator(CloudSqlBaseOperator):
+    """
+    Deletes a Cloud SQL instance.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceDeleteOperator`
+
+    :param instance: Cloud SQL instance ID. This does not include the project ID.
+    :type instance: str
+    :param project_id: Optional, Google Cloud Platform Project ID. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    """
+    # [START gcp_sql_delete_template_fields]
+    template_fields = ('project_id', 'instance', 'gcp_conn_id', 'api_version')
+    # [END gcp_sql_delete_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 *args, **kwargs):
+        super(CloudSqlInstanceDeleteOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def execute(self, context):
+        if not self._check_if_instance_exists(self.instance):
+            print("Cloud SQL instance with ID {} does not exist. Aborting delete."
+                  .format(self.instance))
+            return True
+        else:
+            return self._hook.delete_instance(
+                project_id=self.project_id,
+                instance=self.instance)
+
+
+class CloudSqlInstanceDatabaseCreateOperator(CloudSqlBaseOperator):
+    """
+    Creates a new database inside a Cloud SQL instance.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceDatabaseCreateOperator`
+
+    :param instance: Database instance ID. This does not include the project ID.
+    :type instance: str
+    :param body: The request body, as described in
+        https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert#request-body
+    :type body: dict
+    :param project_id: Optional, Google Cloud Platform Project ID. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    :param validate_body: Whether the body should be validated. Defaults to True.
+    :type validate_body: bool
+    """
+    # [START gcp_sql_db_create_template_fields]
+    template_fields = ('project_id', 'instance', 'gcp_conn_id', 'api_version')
+    # [END gcp_sql_db_create_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 body,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.body = body
+        self.validate_body = validate_body
+        super(CloudSqlInstanceDatabaseCreateOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstanceDatabaseCreateOperator, self)._validate_inputs()
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty")
+
+    def _validate_body_fields(self):
+        if self.validate_body:
+            GcpBodyFieldValidator(CLOUD_SQL_DATABASE_CREATE_VALIDATION,
+                                  api_version=self.api_version).validate(self.body)
+
+    def execute(self, context):
+        self._validate_body_fields()
+        database = self.body.get("name")
+        if not database:
+            self.log.error("Body doesn't contain 'name'. Cannot check if the"
+                           " database already exists in the instance {}."
+                           .format(self.instance))
+            return False
+        if self._check_if_db_exists(database):
+            self.log.info("Cloud SQL instance with ID {} already contains database"
+                          " '{}'. Aborting database insert."
+                          .format(self.instance, database))
+            return True
+        else:
+            return self._hook.create_database(project_id=self.project_id,
+                                              instance=self.instance,
+                                              body=self.body)
+
+
+class CloudSqlInstanceDatabasePatchOperator(CloudSqlBaseOperator):
+    """
+    Updates a resource containing information about a database inside a Cloud SQL
+    instance using patch semantics.
+    See: https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceDatabasePatchOperator`
+
+    :param instance: Database instance ID. This does not include the project ID.
+    :type instance: str
+    :param database: Name of the database to be updated in the instance.
+    :type database: str
+    :param body: The request body, as described in
+        https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/patch#request-body
+    :type body: dict
+    :param project_id: Optional, Google Cloud Platform Project ID.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    :param validate_body: Whether the body should be validated. Defaults to True.
+    :type validate_body: bool
+    """
+    # [START gcp_sql_db_patch_template_fields]
+    template_fields = ('project_id', 'instance', 'database', 'gcp_conn_id',
+                       'api_version')
+    # [END gcp_sql_db_patch_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 database,
+                 body,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.database = database
+        self.body = body
+        self.validate_body = validate_body
+        super(CloudSqlInstanceDatabasePatchOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstanceDatabasePatchOperator, self)._validate_inputs()
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty")
+        if not self.database:
+            raise AirflowException("The required parameter 'database' is empty")
+
+    def _validate_body_fields(self):
+        if self.validate_body:
+            GcpBodyFieldValidator(CLOUD_SQL_DATABASE_PATCH_VALIDATION,
+                                  api_version=self.api_version).validate(self.body)
+
+    def execute(self, context):
+        self._validate_body_fields()
+        if not self._check_if_db_exists(self.database):
+            raise AirflowException("Cloud SQL instance with ID {} does not contain "
+                                   "database '{}'. "
+                                   "Please specify another database to patch."
+                                   .format(self.instance, self.database))
+        else:
+            return self._hook.patch_database(
+                project_id=self.project_id,
+                instance=self.instance,
+                database=self.database,
+                body=self.body)
+
+
+class CloudSqlInstanceDatabaseDeleteOperator(CloudSqlBaseOperator):
+    """
+    Deletes a database from a Cloud SQL instance.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceDatabaseDeleteOperator`
+
+    :param instance: Database instance ID. This does not include the project ID.
+    :type instance: str
+    :param database: Name of the database to be deleted in the instance.
+    :type database: str
+    :param project_id: Optional, Google Cloud Platform Project ID. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    """
+    # [START gcp_sql_db_delete_template_fields]
+    template_fields = ('project_id', 'instance', 'database', 'gcp_conn_id',
+                       'api_version')
+    # [END gcp_sql_db_delete_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 database,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 *args, **kwargs):
+        self.database = database
+        super(CloudSqlInstanceDatabaseDeleteOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstanceDatabaseDeleteOperator, self)._validate_inputs()
+        if not self.database:
+            raise AirflowException("The required parameter 'database' is empty")
+
+    def execute(self, context):
+        if not self._check_if_db_exists(self.database):
+            print("Cloud SQL instance with ID {} does not contain database '{}'. "
+                  "Aborting database delete."
+                  .format(self.instance, self.database))
+            return True
+        else:
+            return self._hook.delete_database(
+                project_id=self.project_id,
+                instance=self.instance,
+                database=self.database)
+
+
+class CloudSqlInstanceExportOperator(CloudSqlBaseOperator):
+    """
+    Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump
+    or CSV file.
+
+    Note: This operator is idempotent. If executed multiple times with the same
+    export file URI, the export file in GCS will simply be overridden.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceImportOperator`
+
+    :param instance: Cloud SQL instance ID. This does not include the project ID.
+    :type instance: str
+    :param body: The request body, as described in
+        https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/export#request-body
+    :type body: dict
+    :param project_id: Optional, Google Cloud Platform Project ID. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    :param validate_body: Whether the body should be validated. Defaults to True.
+    :type validate_body: bool
+    """
+    # [START gcp_sql_export_template_fields]
+    template_fields = ('project_id', 'instance', 'gcp_conn_id', 'api_version')
+    # [END gcp_sql_export_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 body,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.body = body
+        self.validate_body = validate_body
+        super(CloudSqlInstanceExportOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstanceExportOperator, self)._validate_inputs()
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty")
+
+    def _validate_body_fields(self):
+        if self.validate_body:
+            GcpBodyFieldValidator(CLOUD_SQL_EXPORT_VALIDATION,
+                                  api_version=self.api_version).validate(self.body)
+
+    def execute(self, context):
+        self._validate_body_fields()
+        return self._hook.export_instance(
+            project_id=self.project_id,
+            instance=self.instance,
+            body=self.body)
+
+
+class CloudSqlInstanceImportOperator(CloudSqlBaseOperator):
+    """
+    Imports data into a Cloud SQL instance from a SQL dump or CSV file in Cloud Storage.
+
+    CSV IMPORT:
+
+    This operator is NOT idempotent for a CSV import. If the same file is imported
+    multiple times, the imported data will be duplicated in the database.
+    Moreover, if there are any unique constraints the duplicate import may result in an
+    error.
+
+    SQL IMPORT:
+
+    This operator is idempotent for a SQL import if it was also exported by Cloud SQL.
+    The exported SQL contains 'DROP TABLE IF EXISTS' statements for all tables
+    to be imported.
+
+    If the import file was generated in a different way, idempotence is not guaranteed.
+    It has to be ensured on the SQL file level.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlInstanceImportOperator`
+
+    :param instance: Cloud SQL instance ID. This does not include the project ID.
+    :type instance: str
+    :param body: The request body, as described in
+        https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/export#request-body
+    :type body: dict
+    :param project_id: Optional, Google Cloud Platform Project ID. If set to None or missing,
+            the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1beta4).
+    :type api_version: str
+    :param validate_body: Whether the body should be validated. Defaults to True.
+    :type validate_body: bool
+    """
+    # [START gcp_sql_import_template_fields]
+    template_fields = ('project_id', 'instance', 'gcp_conn_id', 'api_version')
+    # [END gcp_sql_import_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 instance,
+                 body,
+                 project_id=None,
+                 gcp_conn_id='google_cloud_default',
+                 api_version='v1beta4',
+                 validate_body=True,
+                 *args, **kwargs):
+        self.body = body
+        self.validate_body = validate_body
+        super(CloudSqlInstanceImportOperator, self).__init__(
+            project_id=project_id, instance=instance, gcp_conn_id=gcp_conn_id,
+            api_version=api_version, *args, **kwargs)
+
+    def _validate_inputs(self):
+        super(CloudSqlInstanceImportOperator, self)._validate_inputs()
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty")
+
+    def _validate_body_fields(self):
+        if self.validate_body:
+            GcpBodyFieldValidator(CLOUD_SQL_IMPORT_VALIDATION,
+                                  api_version=self.api_version).validate(self.body)
+
+    def execute(self, context):
+        self._validate_body_fields()
+        return self._hook.import_instance(
+            project_id=self.project_id,
+            instance=self.instance,
+            body=self.body)
+
+
+class CloudSqlQueryOperator(BaseOperator):
+    """
+    Performs DML or DDL query on an existing Cloud Sql instance. It optionally uses
+    cloud-sql-proxy to establish secure connection with the database.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudSqlQueryOperator`
+
+    :param sql: SQL query or list of queries to run (should be DML or DDL query -
+        this operator does not return any data from the database,
+        so it is useless to pass it DQL queries. Note that it is responsibility of the
+        author of the queries to make sure that the queries are idempotent. For example
+        you can use CREATE TABLE IF NOT EXISTS to create a table.
+    :type sql: str or list[str]
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
+    :param autocommit: if True, each command is automatically committed.
+        (default value: False)
+    :type autocommit: bool
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform for
+        cloud-sql-proxy authentication.
+    :type gcp_conn_id: str
+    :param gcp_cloudsql_conn_id: The connection ID used to connect to Google Cloud SQL
+       its schema should be gcpcloudsql://.
+       See :class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook` for
+       details on how to define gcpcloudsql:// connection.
+    :type gcp_cloudsql_conn_id: str
+    """
+    # [START gcp_sql_query_template_fields]
+    template_fields = ('sql', 'gcp_cloudsql_conn_id', 'gcp_conn_id')
+    template_ext = ('.sql',)
+    # [END gcp_sql_query_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 sql,
+                 autocommit=False,
+                 parameters=None,
+                 gcp_conn_id='google_cloud_default',
+                 gcp_cloudsql_conn_id='google_cloud_sql_default',
+                 *args, **kwargs):
+        super(CloudSqlQueryOperator, self).__init__(*args, **kwargs)
+        self.sql = sql
+        self.gcp_conn_id = gcp_conn_id
+        self.gcp_cloudsql_conn_id = gcp_cloudsql_conn_id
+        self.autocommit = autocommit
+        self.parameters = parameters
+        self.gcp_connection = BaseHook.get_connection(self.gcp_conn_id)
+        self.cloudsql_db_hook = CloudSqlDatabaseHook(
+            gcp_cloudsql_conn_id=gcp_cloudsql_conn_id,
+            default_gcp_project_id=self.gcp_connection.extra_dejson.get(
+                'extra__google_cloud_platform__project'))
+        self.cloud_sql_proxy_runner = None
+        self.database_hook = None
+
+    def execute(self, context):
+        self.cloudsql_db_hook.validate_ssl_certs()
+        self.cloudsql_db_hook.create_connection()
+        try:
+            self.cloudsql_db_hook.validate_socket_path_length()
+            self.database_hook = self.cloudsql_db_hook.get_database_hook()
+            try:
+                try:
+                    if self.cloudsql_db_hook.use_proxy:
+                        self.cloud_sql_proxy_runner = self.cloudsql_db_hook.\
+                            get_sqlproxy_runner()
+                        self.cloudsql_db_hook.free_reserved_port()
+                        # There is very, very slim chance that the socket will
+                        # be taken over here by another bind(0).
+                        # It's quite unlikely to happen though!
+                        self.cloud_sql_proxy_runner.start_proxy()
+                    self.log.info('Executing: "%s"', self.sql)
+                    self.database_hook.run(self.sql, self.autocommit,
+                                           parameters=self.parameters)
+                finally:
+                    if self.cloud_sql_proxy_runner:
+                        self.cloud_sql_proxy_runner.stop_proxy()
+                        self.cloud_sql_proxy_runner = None
+            finally:
+                self.cloudsql_db_hook.cleanup_database_hook()
+        finally:
+            self.cloudsql_db_hook.delete_connection()
+            self.cloudsql_db_hook = None
diff --git a/airflow/contrib/operators/gcp_transfer_operator.py b/airflow/contrib/operators/gcp_transfer_operator.py
new file mode 100644
index 0000000000000..c961bdd97ece2
--- /dev/null
+++ b/airflow/contrib/operators/gcp_transfer_operator.py
@@ -0,0 +1,798 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from copy import deepcopy
+from datetime import date, time
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_transfer_hook import (
+    GCPTransferServiceHook,
+    GcpTransferJobsStatus,
+    TRANSFER_OPTIONS,
+    OBJECT_CONDITIONS,
+    PROJECT_ID,
+    BUCKET_NAME,
+    GCS_DATA_SINK,
+    STATUS,
+    DESCRIPTION,
+    GCS_DATA_SOURCE,
+    HTTP_DATA_SOURCE,
+    SECONDS,
+    MINUTES,
+    HOURS,
+    YEAR,
+    MONTH,
+    DAY,
+    START_TIME_OF_DAY,
+    SCHEDULE_END_DATE,
+    SCHEDULE_START_DATE,
+    SCHEDULE,
+    SECRET_ACCESS_KEY,
+    ACCESS_KEY_ID,
+    AWS_ACCESS_KEY,
+    AWS_S3_DATA_SOURCE,
+    TRANSFER_SPEC,
+)
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+try:
+    from airflow.contrib.hooks.aws_hook import AwsHook
+except ImportError:  # pragma: no cover
+    AwsHook = None  # type: ignore
+
+
+class TransferJobPreprocessor:
+    def __init__(self, body, aws_conn_id='aws_default'):
+        self.body = body
+        self.aws_conn_id = aws_conn_id
+
+    def _inject_aws_credentials(self):
+        if TRANSFER_SPEC not in self.body or AWS_S3_DATA_SOURCE not in self.body[TRANSFER_SPEC]:
+            return
+
+        aws_hook = AwsHook(self.aws_conn_id)
+        aws_credentials = aws_hook.get_credentials()
+        aws_access_key_id = aws_credentials.access_key
+        aws_secret_access_key = aws_credentials.secret_key
+        self.body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE][AWS_ACCESS_KEY] = {
+            ACCESS_KEY_ID: aws_access_key_id,
+            SECRET_ACCESS_KEY: aws_secret_access_key,
+        }
+
+    def _reformat_date(self, field_key):
+        schedule = self.body[SCHEDULE]
+        if field_key not in schedule:
+            return
+        if isinstance(schedule[field_key], date):
+            schedule[field_key] = self._convert_date_to_dict(schedule[field_key])
+
+    def _reformat_time(self, field_key):
+        schedule = self.body[SCHEDULE]
+        if field_key not in schedule:
+            return
+        if isinstance(schedule[field_key], time):
+            schedule[field_key] = self._convert_time_to_dict(schedule[field_key])
+
+    def _reformat_schedule(self):
+        if SCHEDULE not in self.body:
+            return
+        self._reformat_date(SCHEDULE_START_DATE)
+        self._reformat_date(SCHEDULE_END_DATE)
+        self._reformat_time(START_TIME_OF_DAY)
+
+    def process_body(self):
+        self._inject_aws_credentials()
+        self._reformat_schedule()
+        return self.body
+
+    @staticmethod
+    def _convert_date_to_dict(field_date):
+        """
+        Convert native python ``datetime.date`` object  to a format supported by the API
+        """
+        return {DAY: field_date.day, MONTH: field_date.month, YEAR: field_date.year}
+
+    @staticmethod
+    def _convert_time_to_dict(time):
+        """
+        Convert native python ``datetime.time`` object  to a format supported by the API
+        """
+        return {HOURS: time.hour, MINUTES: time.minute, SECONDS: time.second}
+
+
+class TransferJobValidator:
+    def __init__(self, body):
+        self.body = body
+
+    def _verify_data_source(self):
+        is_gcs = GCS_DATA_SOURCE in self.body[TRANSFER_SPEC]
+        is_aws_s3 = AWS_S3_DATA_SOURCE in self.body[TRANSFER_SPEC]
+        is_http = HTTP_DATA_SOURCE in self.body[TRANSFER_SPEC]
+
+        sources_count = sum([is_gcs, is_aws_s3, is_http])
+        if sources_count != 0 and sources_count != 1:
+            raise AirflowException(
+                "More than one data source detected. Please choose exactly one data source from: "
+                "gcsDataSource, awsS3DataSource and httpDataSource."
+            )
+
+    def _restrict_aws_credentials(self):
+        if AWS_S3_DATA_SOURCE not in self.body[TRANSFER_SPEC]:
+            return
+
+        if AWS_ACCESS_KEY in self.body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE]:
+            raise AirflowException(
+                "AWS credentials detected inside the body parameter (awsAccessKey). This is not allowed, "
+                "please use Airflow connections to store credentials."
+            )
+
+    def _restrict_empty_body(self):
+        if not self.body:
+            raise AirflowException("The required parameter 'body' is empty or None")
+
+    def validate_body(self):
+        self._restrict_empty_body()
+
+        if TRANSFER_SPEC not in self.body:
+            return
+
+        self._restrict_aws_credentials()
+        self._verify_data_source()
+
+
+class GcpTransferServiceJobCreateOperator(BaseOperator):
+    """
+    Creates a transfer job that runs periodically.
+
+    .. warning::
+
+        This operator is NOT idempotent. If you run it many times, many transfer
+        jobs will be created in the Google Cloud Platform.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceJobCreateOperator`
+
+    :param body: (Required) The request body, as described in
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/create#request-body
+        With three additional improvements:
+
+        * dates can be given in the form :class:`datetime.date`
+        * times can be given in the form :class:`datetime.time`
+        * credentials to Amazon Web Service should be stored in the connection and indicated by the
+          aws_conn_id parameter
+
+    :type body: dict
+    :param aws_conn_id: The connection ID used to retrieve credentials to
+        Amazon Web Service.
+    :type aws_conn_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    """
+
+    # [START gcp_transfer_job_create_template_fields]
+    template_fields = ('body', 'gcp_conn_id', 'aws_conn_id')
+    # [END gcp_transfer_job_create_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        body,
+        aws_conn_id='aws_default',
+        gcp_conn_id='google_cloud_default',
+        api_version='v1',
+        *args,
+        **kwargs
+    ):
+        super(GcpTransferServiceJobCreateOperator, self).__init__(*args, **kwargs)
+        self.body = deepcopy(body)
+        self.aws_conn_id = aws_conn_id
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        TransferJobValidator(body=self.body).validate_body()
+
+    def execute(self, context):
+        TransferJobPreprocessor(body=self.body, aws_conn_id=self.aws_conn_id).process_body()
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        return hook.create_transfer_job(body=self.body)
+
+
+class GcpTransferServiceJobUpdateOperator(BaseOperator):
+    """
+    Updates a transfer job that runs periodically.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceJobUpdateOperator`
+
+    :param job_name: (Required) Name of the job to be updated
+    :type job_name: str
+    :param body: (Required) The request body, as described in
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/patch#request-body
+        With three additional improvements:
+
+        * dates can be given in the form :class:`datetime.date`
+        * times can be given in the form :class:`datetime.time`
+        * credentials to Amazon Web Service should be stored in the connection and indicated by the
+          aws_conn_id parameter
+
+    :type body: dict
+    :param aws_conn_id: The connection ID used to retrieve credentials to
+        Amazon Web Service.
+    :type aws_conn_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    """
+
+    # [START gcp_transfer_job_update_template_fields]
+    template_fields = ('job_name', 'body', 'gcp_conn_id', 'aws_conn_id')
+    # [END gcp_transfer_job_update_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        job_name,
+        body,
+        aws_conn_id='aws_default',
+        gcp_conn_id='google_cloud_default',
+        api_version='v1',
+        *args,
+        **kwargs
+    ):
+        super(GcpTransferServiceJobUpdateOperator, self).__init__(*args, **kwargs)
+        self.job_name = job_name
+        self.body = body
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self.aws_conn_id = aws_conn_id
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        TransferJobValidator(body=self.body).validate_body()
+        if not self.job_name:
+            raise AirflowException("The required parameter 'job_name' is empty or None")
+
+    def execute(self, context):
+        TransferJobPreprocessor(body=self.body, aws_conn_id=self.aws_conn_id).process_body()
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        return hook.update_transfer_job(job_name=self.job_name, body=self.body)
+
+
+class GcpTransferServiceJobDeleteOperator(BaseOperator):
+    """
+    Delete a transfer job. This is a soft delete. After a transfer job is
+    deleted, the job and all the transfer executions are subject to garbage
+    collection. Transfer jobs become eligible for garbage collection
+    30 days after soft delete.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceJobDeleteOperator`
+
+    :param job_name: (Required) Name of the TRANSFER operation
+    :type job_name: str
+    :param project_id: (Optional) the ID of the project that owns the Transfer
+        Job. If set to None or missing, the default project_id from the GCP
+        connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    """
+
+    # [START gcp_transfer_job_delete_template_fields]
+    template_fields = ('job_name', 'project_id', 'gcp_conn_id', 'api_version')
+    # [END gcp_transfer_job_delete_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self, job_name, gcp_conn_id='google_cloud_default', api_version='v1', project_id=None, *args, **kwargs
+    ):
+        super(GcpTransferServiceJobDeleteOperator, self).__init__(*args, **kwargs)
+        self.job_name = job_name
+        self.project_id = project_id
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        if not self.job_name:
+            raise AirflowException("The required parameter 'job_name' is empty or None")
+
+    def execute(self, context):
+        self._validate_inputs()
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        hook.delete_transfer_job(job_name=self.job_name, project_id=self.project_id)
+
+
+class GcpTransferServiceOperationGetOperator(BaseOperator):
+    """
+    Gets the latest state of a long-running operation in Google Storage Transfer
+    Service.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceOperationGetOperator`
+
+    :param operation_name: (Required) Name of the transfer operation.
+    :type operation_name: str
+    :param gcp_conn_id: The connection ID used to connect to Google
+        Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    """
+
+    # [START gcp_transfer_operation_get_template_fields]
+    template_fields = ('operation_name', 'gcp_conn_id')
+    # [END gcp_transfer_operation_get_template_fields]
+
+    @apply_defaults
+    def __init__(self, operation_name, gcp_conn_id='google_cloud_default', api_version='v1', *args, **kwargs):
+        super(GcpTransferServiceOperationGetOperator, self).__init__(*args, **kwargs)
+        self.operation_name = operation_name
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        if not self.operation_name:
+            raise AirflowException("The required parameter 'operation_name' is empty or None")
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        operation = hook.get_transfer_operation(operation_name=self.operation_name)
+        return operation
+
+
+class GcpTransferServiceOperationsListOperator(BaseOperator):
+    """
+    Lists long-running operations in Google Storage Transfer
+    Service that match the specified filter.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceOperationsListOperator`
+
+    :param filter: (Required) A request filter, as described in
+            https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/list#body.QUERY_PARAMETERS.filter
+    :type filter: dict
+    :param gcp_conn_id: The connection ID used to connect to Google
+        Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    """
+
+    # [START gcp_transfer_operations_list_template_fields]
+    template_fields = ('filter', 'gcp_conn_id')
+    # [END gcp_transfer_operations_list_template_fields]
+
+    def __init__(self, filter, gcp_conn_id='google_cloud_default', api_version='v1', *args, **kwargs):
+        super(GcpTransferServiceOperationsListOperator, self).__init__(*args, **kwargs)
+        self.filter = filter
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        if not self.filter:
+            raise AirflowException("The required parameter 'filter' is empty or None")
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        operations_list = hook.list_transfer_operations(filter=self.filter)
+        self.log.info(operations_list)
+        return operations_list
+
+
+class GcpTransferServiceOperationPauseOperator(BaseOperator):
+    """
+    Pauses a transfer operation in Google Storage Transfer Service.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceOperationPauseOperator`
+
+    :param operation_name: (Required) Name of the transfer operation.
+    :type operation_name: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    :param api_version:  API version used (e.g. v1).
+    :type api_version: str
+    """
+
+    # [START gcp_transfer_operation_pause_template_fields]
+    template_fields = ('operation_name', 'gcp_conn_id', 'api_version')
+    # [END gcp_transfer_operation_pause_template_fields]
+
+    @apply_defaults
+    def __init__(self, operation_name, gcp_conn_id='google_cloud_default', api_version='v1', *args, **kwargs):
+        super(GcpTransferServiceOperationPauseOperator, self).__init__(*args, **kwargs)
+        self.operation_name = operation_name
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        if not self.operation_name:
+            raise AirflowException("The required parameter 'operation_name' is empty or None")
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        hook.pause_transfer_operation(operation_name=self.operation_name)
+
+
+class GcpTransferServiceOperationResumeOperator(BaseOperator):
+    """
+    Resumes a transfer operation in Google Storage Transfer Service.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceOperationResumeOperator`
+
+    :param operation_name: (Required) Name of the transfer operation.
+    :type operation_name: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    :type gcp_conn_id: str
+    """
+
+    # [START gcp_transfer_operation_resume_template_fields]
+    template_fields = ('operation_name', 'gcp_conn_id', 'api_version')
+    # [END gcp_transfer_operation_resume_template_fields]
+
+    @apply_defaults
+    def __init__(self, operation_name, gcp_conn_id='google_cloud_default', api_version='v1', *args, **kwargs):
+        self.operation_name = operation_name
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self._validate_inputs()
+        super(GcpTransferServiceOperationResumeOperator, self).__init__(*args, **kwargs)
+
+    def _validate_inputs(self):
+        if not self.operation_name:
+            raise AirflowException("The required parameter 'operation_name' is empty or None")
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        hook.resume_transfer_operation(operation_name=self.operation_name)
+
+
+class GcpTransferServiceOperationCancelOperator(BaseOperator):
+    """
+    Cancels a transfer operation in Google Storage Transfer Service.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GcpTransferServiceOperationCancelOperator`
+
+    :param operation_name: (Required) Name of the transfer operation.
+    :type operation_name: str
+    :param api_version: API version used (e.g. v1).
+    :type api_version: str
+    :param gcp_conn_id: The connection ID used to connect to Google
+        Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START gcp_transfer_operation_cancel_template_fields]
+    template_fields = ('operation_name', 'gcp_conn_id', 'api_version')
+    # [END gcp_transfer_operation_cancel_template_fields]
+
+    @apply_defaults
+    def __init__(self, operation_name, api_version='v1', gcp_conn_id='google_cloud_default', *args, **kwargs):
+        super(GcpTransferServiceOperationCancelOperator, self).__init__(*args, **kwargs)
+        self.operation_name = operation_name
+        self.api_version = api_version
+        self.gcp_conn_id = gcp_conn_id
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        if not self.operation_name:
+            raise AirflowException("The required parameter 'operation_name' is empty or None")
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(api_version=self.api_version, gcp_conn_id=self.gcp_conn_id)
+        hook.cancel_transfer_operation(operation_name=self.operation_name)
+
+
+class S3ToGoogleCloudStorageTransferOperator(BaseOperator):
+    """
+    Synchronizes an S3 bucket with a Google Cloud Storage bucket using the
+    GCP Storage Transfer Service.
+
+    .. warning::
+
+        This operator is NOT idempotent. If you run it many times, many transfer
+        jobs will be created in the Google Cloud Platform.
+
+    **Example**:
+
+    .. code-block:: python
+
+       s3_to_gcs_transfer_op = S3ToGoogleCloudStorageTransferOperator(
+            task_id='s3_to_gcs_transfer_example',
+            s3_bucket='my-s3-bucket',
+            project_id='my-gcp-project',
+            gcs_bucket='my-gcs-bucket',
+            dag=my_dag)
+
+    :param s3_bucket: The S3 bucket where to find the objects. (templated)
+    :type s3_bucket: str
+    :param gcs_bucket: The destination Google Cloud Storage bucket
+        where you want to store the files. (templated)
+    :type gcs_bucket: str
+    :param project_id: Optional ID of the Google Cloud Platform Console project that
+        owns the job
+    :type project_id: str
+    :param aws_conn_id: The source S3 connection
+    :type aws_conn_id: str
+    :param gcp_conn_id: The destination connection ID to use
+        when connecting to Google Cloud Storage.
+    :type gcp_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide delegation enabled.
+    :type delegate_to: str
+    :param description: Optional transfer service job description
+    :type description: str
+    :param schedule: Optional transfer service schedule;
+        If not set, run transfer job once as soon as the operator runs
+        The format is described
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs.
+        With two additional improvements:
+
+        * dates they can be passed as :class:`datetime.date`
+        * times they can be passed as :class:`datetime.time`
+
+    :type schedule: dict
+    :param object_conditions: Optional transfer service object conditions; see
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/TransferSpec
+    :type object_conditions: dict
+    :param transfer_options: Optional transfer service transfer options; see
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/TransferSpec
+    :type transfer_options: dict
+    :param wait: Wait for transfer to finish
+    :type wait: bool
+    :param timeout: Time to wait for the operation to end in seconds
+    :type timeout: int
+    """
+
+    template_fields = ('gcp_conn_id', 's3_bucket', 'gcs_bucket', 'description', 'object_conditions')
+    ui_color = '#e09411'
+
+    @apply_defaults
+    def __init__(
+        self,
+        s3_bucket,
+        gcs_bucket,
+        project_id=None,
+        aws_conn_id='aws_default',
+        gcp_conn_id='google_cloud_default',
+        delegate_to=None,
+        description=None,
+        schedule=None,
+        object_conditions=None,
+        transfer_options=None,
+        wait=True,
+        timeout=None,
+        *args,
+        **kwargs
+    ):
+
+        super(S3ToGoogleCloudStorageTransferOperator, self).__init__(*args, **kwargs)
+        self.s3_bucket = s3_bucket
+        self.gcs_bucket = gcs_bucket
+        self.project_id = project_id
+        self.aws_conn_id = aws_conn_id
+        self.gcp_conn_id = gcp_conn_id
+        self.delegate_to = delegate_to
+        self.description = description
+        self.schedule = schedule
+        self.object_conditions = object_conditions
+        self.transfer_options = transfer_options
+        self.wait = wait
+        self.timeout = timeout
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to)
+        body = self._create_body()
+
+        TransferJobPreprocessor(body=body, aws_conn_id=self.aws_conn_id).process_body()
+
+        job = hook.create_transfer_job(body=body)
+
+        if self.wait:
+            hook.wait_for_transfer_job(job, timeout=self.timeout)
+
+    def _create_body(self):
+        body = {
+            DESCRIPTION: self.description,
+            STATUS: GcpTransferJobsStatus.ENABLED,
+            TRANSFER_SPEC: {
+                AWS_S3_DATA_SOURCE: {BUCKET_NAME: self.s3_bucket},
+                GCS_DATA_SINK: {BUCKET_NAME: self.gcs_bucket},
+            },
+        }
+
+        if self.project_id is not None:
+            body[PROJECT_ID] = self.project_id
+
+        if self.schedule is not None:
+            body[SCHEDULE] = self.schedule
+
+        if self.object_conditions is not None:
+            body[TRANSFER_SPEC][OBJECT_CONDITIONS] = self.object_conditions
+
+        if self.transfer_options is not None:
+            body[TRANSFER_SPEC][TRANSFER_OPTIONS] = self.transfer_options
+
+        return body
+
+
+class GoogleCloudStorageToGoogleCloudStorageTransferOperator(BaseOperator):
+    """
+    Copies objects from a bucket to another using the GCP Storage Transfer
+    Service.
+
+    .. warning::
+
+        This operator is NOT idempotent. If you run it many times, many transfer
+        jobs will be created in the Google Cloud Platform.
+
+    **Example**:
+
+    .. code-block:: python
+
+       gcs_to_gcs_transfer_op = GoogleCloudStorageToGoogleCloudStorageTransferOperator(
+            task_id='gcs_to_gcs_transfer_example',
+            source_bucket='my-source-bucket',
+            destination_bucket='my-destination-bucket',
+            project_id='my-gcp-project',
+            dag=my_dag)
+
+    :param source_bucket: The source Google cloud storage bucket where the
+         object is. (templated)
+    :type source_bucket: str
+    :param destination_bucket: The destination Google cloud storage bucket
+        where the object should be. (templated)
+    :type destination_bucket: str
+    :param project_id: The ID of the Google Cloud Platform Console project that
+        owns the job
+    :type project_id: str
+    :param gcp_conn_id: Optional connection ID to use when connecting to Google Cloud
+        Storage.
+    :type gcp_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must have
+        domain-wide delegation enabled.
+    :type delegate_to: str
+    :param description: Optional transfer service job description
+    :type description: str
+    :param schedule: Optional transfer service schedule;
+        If not set, run transfer job once as soon as the operator runs
+        See:
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs.
+        With two additional improvements:
+
+        * dates they can be passed as :class:`datetime.date`
+        * times they can be passed as :class:`datetime.time`
+
+    :type schedule: dict
+    :param object_conditions: Optional transfer service object conditions; see
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/TransferSpec#ObjectConditions
+    :type object_conditions: dict
+    :param transfer_options: Optional transfer service transfer options; see
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/TransferSpec#TransferOptions
+    :type transfer_options: dict
+    :param wait: Wait for transfer to finish; defaults to `True`
+    :type wait: bool
+    :param timeout: Time to wait for the operation to end in seconds
+    :type timeout: int
+    """
+
+    template_fields = (
+        'gcp_conn_id',
+        'source_bucket',
+        'destination_bucket',
+        'description',
+        'object_conditions',
+    )
+    ui_color = '#e09411'
+
+    @apply_defaults
+    def __init__(
+        self,
+        source_bucket,
+        destination_bucket,
+        project_id=None,
+        gcp_conn_id='google_cloud_default',
+        delegate_to=None,
+        description=None,
+        schedule=None,
+        object_conditions=None,
+        transfer_options=None,
+        wait=True,
+        timeout=None,
+        *args,
+        **kwargs
+    ):
+
+        super(GoogleCloudStorageToGoogleCloudStorageTransferOperator, self).__init__(*args, **kwargs)
+        self.source_bucket = source_bucket
+        self.destination_bucket = destination_bucket
+        self.project_id = project_id
+        self.gcp_conn_id = gcp_conn_id
+        self.delegate_to = delegate_to
+        self.description = description
+        self.schedule = schedule
+        self.object_conditions = object_conditions
+        self.transfer_options = transfer_options
+        self.wait = wait
+        self.timeout = timeout
+
+    def execute(self, context):
+        hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to)
+
+        body = self._create_body()
+
+        TransferJobPreprocessor(body=body).process_body()
+
+        job = hook.create_transfer_job(body=body)
+
+        if self.wait:
+            hook.wait_for_transfer_job(job, timeout=self.timeout)
+
+    def _create_body(self):
+        body = {
+            DESCRIPTION: self.description,
+            STATUS: GcpTransferJobsStatus.ENABLED,
+            TRANSFER_SPEC: {
+                GCS_DATA_SOURCE: {BUCKET_NAME: self.source_bucket},
+                GCS_DATA_SINK: {BUCKET_NAME: self.destination_bucket},
+            },
+        }
+
+        if self.project_id is not None:
+            body[PROJECT_ID] = self.project_id
+
+        if self.schedule is not None:
+            body[SCHEDULE] = self.schedule
+
+        if self.object_conditions is not None:
+            body[TRANSFER_SPEC][OBJECT_CONDITIONS] = self.object_conditions
+
+        if self.transfer_options is not None:
+            body[TRANSFER_SPEC][TRANSFER_OPTIONS] = self.transfer_options
+
+        return body
diff --git a/airflow/contrib/operators/gcp_translate_operator.py b/airflow/contrib/operators/gcp_translate_operator.py
new file mode 100644
index 0000000000000..87d4aa4cfdb5c
--- /dev/null
+++ b/airflow/contrib/operators/gcp_translate_operator.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_translate_hook import CloudTranslateHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class CloudTranslateTextOperator(BaseOperator):
+    """
+    Translate a string or list of strings.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudTranslateTextOperator`
+
+    See https://cloud.google.com/translate/docs/translating-text
+
+    Execute method returns str or list.
+
+    This is a list of dictionaries for each queried value. Each
+    dictionary typically contains three keys (though not
+    all will be present in all cases).
+
+    * ``detectedSourceLanguage``: The detected language (as an
+      ISO 639-1 language code) of the text.
+    * ``translatedText``: The translation of the text into the
+      target language.
+    * ``input``: The corresponding input value.
+    * ``model``: The model used to translate the text.
+
+    If only a single value is passed, then only a single
+    dictionary is set as XCom return value.
+
+    :type values: str or list
+    :param values: String or list of strings to translate.
+
+    :type target_language: str
+    :param target_language: The language to translate results into. This
+      is required by the API and defaults to
+      the target language of the current instance.
+
+    :type format_: str or None
+    :param format_: (Optional) One of ``text`` or ``html``, to specify
+      if the input text is plain text or HTML.
+
+    :type source_language: str or None
+    :param source_language: (Optional) The language of the text to
+      be translated.
+
+    :type model: str or None
+    :param model: (Optional) The model used to translate the text, such
+      as ``'base'`` or ``'nmt'``.
+
+    """
+
+    # [START translate_template_fields]
+    template_fields = ('values', 'target_language', 'format_', 'source_language', 'model', 'gcp_conn_id')
+    # [END translate_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        values,
+        target_language,
+        format_,
+        source_language,
+        model,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudTranslateTextOperator, self).__init__(*args, **kwargs)
+        self.values = values
+        self.target_language = target_language
+        self.format_ = format_
+        self.source_language = source_language
+        self.model = model
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        _hook = CloudTranslateHook(gcp_conn_id=self.gcp_conn_id)
+        try:
+            translation = _hook.translate(
+                values=self.values,
+                target_language=self.target_language,
+                format_=self.format_,
+                source_language=self.source_language,
+                model=self.model,
+            )
+            self.log.debug("Translation %s", translation)
+            return translation
+        except ValueError as e:
+            self.log.error('An error has been thrown from translate method:')
+            self.log.error(e)
+            raise AirflowException(e)
diff --git a/airflow/contrib/operators/gcp_vision_operator.py b/airflow/contrib/operators/gcp_vision_operator.py
new file mode 100644
index 0000000000000..ba1c1b5058aab
--- /dev/null
+++ b/airflow/contrib/operators/gcp_vision_operator.py
@@ -0,0 +1,963 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from google.api_core.exceptions import AlreadyExists
+
+from airflow.contrib.hooks.gcp_vision_hook import CloudVisionHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class CloudVisionProductSetCreateOperator(BaseOperator):
+    """
+    Creates a new ProductSet resource.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductSetCreateOperator`
+
+    :param product_set: (Required) The ProductSet to create. If a dict is provided, it must be of the same
+        form as the protobuf message `ProductSet`.
+    :type product_set: dict or google.cloud.vision_v1.types.ProductSet
+    :param location: (Required) The region where the ProductSet should be created. Valid regions
+        (as of 2019-02-05) are: us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param project_id: (Optional) The project in which the ProductSet should be created. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param product_set_id: (Optional) A user-supplied resource id for this ProductSet.
+        If set, the server will attempt to use this value as the resource id. If it is
+        already in use, an error is returned with code ALREADY_EXISTS. Must be at most
+        128 characters long. It cannot contain the character /.
+    :type product_set_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_productset_create_template_fields]
+    template_fields = ("location", "project_id", "product_set_id", "gcp_conn_id")
+    # [END vision_productset_create_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        product_set,
+        location,
+        project_id=None,
+        product_set_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductSetCreateOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.project_id = project_id
+        self.product_set = product_set
+        self.product_set_id = product_set_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        try:
+            return self._hook.create_product_set(
+                location=self.location,
+                project_id=self.project_id,
+                product_set=self.product_set,
+                product_set_id=self.product_set_id,
+                retry=self.retry,
+                timeout=self.timeout,
+                metadata=self.metadata,
+            )
+        except AlreadyExists:
+            self.log.info(
+                "Product set with id %s already exists. Exiting from the create operation.",
+                self.product_set_id,
+            )
+            return self.product_set_id
+
+
+class CloudVisionProductSetGetOperator(BaseOperator):
+    """
+    Gets information associated with a ProductSet.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductSetGetOperator`
+
+    :param location: (Required) The region where the ProductSet is located. Valid regions (as of 2019-02-05)
+        are: us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product_set_id: (Required) The resource id of this ProductSet.
+    :type product_set_id: str
+    :param project_id: (Optional) The project in which the ProductSet is located. If set
+        to None or missing, the default `project_id` from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_productset_get_template_fields]
+    template_fields = ('location', 'project_id', 'product_set_id', 'gcp_conn_id')
+    # [END vision_productset_get_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        location,
+        product_set_id,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductSetGetOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.project_id = project_id
+        self.product_set_id = product_set_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        return self._hook.get_product_set(
+            location=self.location,
+            product_set_id=self.product_set_id,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionProductSetUpdateOperator(BaseOperator):
+    """
+    Makes changes to a `ProductSet` resource. Only display_name can be updated currently.
+
+    .. note:: To locate the `ProductSet` resource, its `name` in the form
+        `projects/PROJECT_ID/locations/LOC_ID/productSets/PRODUCT_SET_ID` is necessary.
+
+    You can provide the `name` directly as an attribute of the `product_set` object.
+    However, you can leave it blank and provide `location` and `product_set_id` instead
+    (and optionally `project_id` - if not present, the connection default will be used)
+    and the `name` will be created by the operator itself.
+
+    This mechanism exists for your convenience, to allow leaving the `project_id` empty
+    and having Airflow use the connection default `project_id`.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductSetUpdateOperator`
+
+    :param product_set: (Required) The ProductSet resource which replaces the one on the
+        server. If a dict is provided, it must be of the same form as the protobuf
+        message `ProductSet`.
+    :type product_set: dict or google.cloud.vision_v1.types.ProductSet
+    :param location: (Optional) The region where the ProductSet is located. Valid regions (as of 2019-02-05)
+        are: us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product_set_id: (Optional) The resource id of this ProductSet.
+    :type product_set_id: str
+    :param project_id: (Optional) The project in which the ProductSet should be created. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param update_mask: (Optional) The `FieldMask` that specifies which fields to update. If update_mask
+        isn’t specified, all mutable fields are to be updated. Valid mask path is display_name. If a dict is
+        provided, it must be of the same form as the protobuf message `FieldMask`.
+    :type update_mask: dict or google.cloud.vision_v1.types.FieldMask
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+
+    """
+
+    # [START vision_productset_update_template_fields]
+    template_fields = ('location', 'project_id', 'product_set_id', 'gcp_conn_id')
+    # [END vision_productset_update_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        product_set,
+        location=None,
+        product_set_id=None,
+        project_id=None,
+        update_mask=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductSetUpdateOperator, self).__init__(*args, **kwargs)
+        self.product_set = product_set
+        self.update_mask = update_mask
+        self.location = location
+        self.project_id = project_id
+        self.product_set_id = product_set_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        return self._hook.update_product_set(
+            location=self.location,
+            product_set_id=self.product_set_id,
+            project_id=self.project_id,
+            product_set=self.product_set,
+            update_mask=self.update_mask,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionProductSetDeleteOperator(BaseOperator):
+    """
+    Permanently deletes a `ProductSet`. `Products` and `ReferenceImages` in the
+    `ProductSet` are not deleted. The actual image files are not deleted from Google
+    Cloud Storage.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductSetDeleteOperator`
+
+    :param location: (Required) The region where the ProductSet is located. Valid regions (as of 2019-02-05)
+        are: us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product_set_id: (Required) The resource id of this ProductSet.
+    :type product_set_id: str
+    :param project_id: (Optional) The project in which the ProductSet should be created.
+        If set to None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+
+    """
+
+    # [START vision_productset_delete_template_fields]
+    template_fields = ('location', 'project_id', 'product_set_id', 'gcp_conn_id')
+    # [END vision_productset_delete_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        location,
+        product_set_id,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductSetDeleteOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.project_id = project_id
+        self.product_set_id = product_set_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        self._hook.delete_product_set(
+            location=self.location,
+            product_set_id=self.product_set_id,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionProductCreateOperator(BaseOperator):
+    """
+    Creates and returns a new product resource.
+
+    Possible errors regarding the `Product` object provided:
+
+    - Returns `INVALID_ARGUMENT` if `display_name` is missing or longer than 4096 characters.
+    - Returns `INVALID_ARGUMENT` if `description` is longer than 4096 characters.
+    - Returns `INVALID_ARGUMENT` if `product_category` is missing or invalid.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductCreateOperator`
+
+    :param location: (Required) The region where the Product should be created. Valid regions
+        (as of 2019-02-05) are: us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product: (Required) The product to create. If a dict is provided, it must be of the same form as
+        the protobuf message `Product`.
+    :type product: dict or google.cloud.vision_v1.types.Product
+    :param project_id: (Optional) The project in which the Product should be created. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param product_id: (Optional) A user-supplied resource id for this Product.
+        If set, the server will attempt to use this value as the resource id. If it is
+        already in use, an error is returned with code ALREADY_EXISTS. Must be at most
+        128 characters long. It cannot contain the character /.
+    :type product_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+
+    """
+
+    # [START vision_product_create_template_fields]
+    template_fields = ('location', 'project_id', 'product_id', 'gcp_conn_id')
+    # [END vision_product_create_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        location,
+        product,
+        project_id=None,
+        product_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductCreateOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.product = product
+        self.project_id = project_id
+        self.product_id = product_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        try:
+            return self._hook.create_product(
+                location=self.location,
+                product=self.product,
+                project_id=self.project_id,
+                product_id=self.product_id,
+                retry=self.retry,
+                timeout=self.timeout,
+                metadata=self.metadata,
+            )
+        except AlreadyExists:
+            self.log.info(
+                'Product with id %s already exists. Exiting from the create operation.', self.product_id
+            )
+            return self.product_id
+
+
+class CloudVisionProductGetOperator(BaseOperator):
+    """
+    Gets information associated with a `Product`.
+
+    Possible errors:
+
+    - Returns `NOT_FOUND` if the `Product` does not exist.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductGetOperator`
+
+    :param location: (Required) The region where the Product is located. Valid regions (as of 2019-02-05) are:
+        us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product_id: (Required) The resource id of this Product.
+    :type product_id: str
+    :param project_id: (Optional) The project in which the Product is located. If set to
+        None or missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+
+    """
+
+    # [START vision_product_get_template_fields]
+    template_fields = ('location', 'project_id', 'product_id', 'gcp_conn_id')
+    # [END vision_product_get_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        location,
+        product_id,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductGetOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.product_id = product_id
+        self.project_id = project_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        return self._hook.get_product(
+            location=self.location,
+            product_id=self.product_id,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionProductUpdateOperator(BaseOperator):
+    """
+    Makes changes to a Product resource. Only the display_name, description, and labels fields can be
+    updated right now.
+
+    If labels are updated, the change will not be reflected in queries until the next index time.
+
+    .. note:: To locate the `Product` resource, its `name` in the form
+        `projects/PROJECT_ID/locations/LOC_ID/products/PRODUCT_ID` is necessary.
+
+    You can provide the `name` directly as an attribute of the `product` object. However, you can leave it
+    blank and provide `location` and `product_id` instead (and optionally `project_id` - if not present,
+    the connection default will be used) and the `name` will be created by the operator itself.
+
+    This mechanism exists for your convenience, to allow leaving the `project_id` empty and having Airflow
+    use the connection default `project_id`.
+
+    Possible errors related to the provided `Product`:
+
+    - Returns `NOT_FOUND` if the Product does not exist.
+    - Returns `INVALID_ARGUMENT` if `display_name` is present in update_mask but is missing from the request
+        or longer than 4096 characters.
+    - Returns `INVALID_ARGUMENT` if `description` is present in update_mask but is longer than 4096
+        characters.
+    - Returns `INVALID_ARGUMENT` if `product_category` is present in update_mask.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductUpdateOperator`
+
+    :param product: (Required) The Product resource which replaces the one on the server. product.name is
+        immutable. If a dict is provided, it must be of the same form as the protobuf message `Product`.
+    :type product: dict or google.cloud.vision_v1.types.ProductSet
+    :param location: (Optional) The region where the Product is located. Valid regions (as of 2019-02-05) are:
+        us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product_id: (Optional) The resource id of this Product.
+    :type product_id: str
+    :param project_id: (Optional) The project in which the Product is located. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param update_mask: (Optional) The `FieldMask` that specifies which fields to update. If update_mask
+        isn’t specified, all mutable fields are to be updated. Valid mask paths include product_labels,
+        display_name, and description. If a dict is provided, it must be of the same form as the protobuf
+        message `FieldMask`.
+    :type update_mask: dict or google.cloud.vision_v1.types.FieldMask
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_product_update_template_fields]
+    template_fields = ('location', 'project_id', 'product_id', 'gcp_conn_id')
+    # [END vision_product_update_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        product,
+        location=None,
+        product_id=None,
+        project_id=None,
+        update_mask=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductUpdateOperator, self).__init__(*args, **kwargs)
+        self.product = product
+        self.location = location
+        self.product_id = product_id
+        self.project_id = project_id
+        self.update_mask = update_mask
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        return self._hook.update_product(
+            product=self.product,
+            location=self.location,
+            product_id=self.product_id,
+            project_id=self.project_id,
+            update_mask=self.update_mask,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionProductDeleteOperator(BaseOperator):
+    """
+    Permanently deletes a product and its reference images.
+
+    Metadata of the product and all its images will be deleted right away, but search queries against
+    ProductSets containing the product may still work until all related caches are refreshed.
+
+    Possible errors:
+
+    - Returns `NOT_FOUND` if the product does not exist.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionProductDeleteOperator`
+
+    :param location: (Required) The region where the Product is located. Valid regions (as of 2019-02-05) are:
+        us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param product_id: (Required) The resource id of this Product.
+    :type product_id: str
+    :param project_id: (Optional) The project in which the Product is located. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_product_delete_template_fields]
+    template_fields = ('location', 'project_id', 'product_id', 'gcp_conn_id')
+    # [END vision_product_delete_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        location,
+        product_id,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionProductDeleteOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.product_id = product_id
+        self.project_id = project_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self._hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+
+    def execute(self, context):
+        self._hook.delete_product(
+            location=self.location,
+            product_id=self.product_id,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionAnnotateImageOperator(BaseOperator):
+    """
+    Run image detection and annotation for an image.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionAnnotateImageOperator`
+
+    :param request: (Required) Individual file annotation requests.
+        If a dict is provided, it must be of the same form as the protobuf
+        message class:`google.cloud.vision_v1.types.AnnotateImageRequest`
+    :type request: dict or google.cloud.vision_v1.types.AnnotateImageRequest
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_annotate_image_template_fields]
+    template_fields = ('request', 'gcp_conn_id')
+    # [END vision_annotate_image_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self, request, retry=None, timeout=None, gcp_conn_id='google_cloud_default', *args, **kwargs
+    ):
+        super(CloudVisionAnnotateImageOperator, self).__init__(*args, **kwargs)
+        self.request = request
+        self.retry = retry
+        self.timeout = timeout
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+        return hook.annotate_image(request=self.request, retry=self.retry, timeout=self.timeout)
+
+
+class CloudVisionReferenceImageCreateOperator(BaseOperator):
+    """
+    Creates and returns a new ReferenceImage ID resource.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionReferenceImageCreateOperator`
+
+    :param location: (Required) The region where the Product is located. Valid regions (as of 2019-02-05) are:
+        us-east1, us-west1, europe-west1, asia-east1
+    :type location: str
+    :param reference_image: (Required) The reference image to create. If an image ID is specified, it is
+        ignored.
+        If a dict is provided, it must be of the same form as the protobuf message
+        :class:`google.cloud.vision_v1.types.ReferenceImage`
+    :type reference_image: dict or google.cloud.vision_v1.types.ReferenceImage
+    :param reference_image_id: (Optional) A user-supplied resource id for the ReferenceImage to be added.
+        If set, the server will attempt to use this value as the resource id. If it is already in use, an
+        error is returned with code ALREADY_EXISTS. Must be at most 128 characters long. It cannot contain
+        the character `/`.
+    :type reference_image_id: str
+    :param product_id: (Optional) The resource id of this Product.
+    :type product_id: str
+    :param project_id: (Optional) The project in which the Product is located. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_reference_image_create_template_fields]
+    template_fields = (
+        "location",
+        "reference_image",
+        "product_id",
+        "reference_image_id",
+        "project_id",
+        "gcp_conn_id",
+    )
+    # [END vision_reference_image_create_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        location,
+        reference_image,
+        product_id,
+        reference_image_id=None,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionReferenceImageCreateOperator, self).__init__(*args, **kwargs)
+        self.location = location
+        self.product_id = product_id
+        self.reference_image = reference_image
+        self.reference_image_id = reference_image_id
+        self.project_id = project_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        try:
+            hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+            return hook.create_reference_image(
+                location=self.location,
+                product_id=self.product_id,
+                reference_image=self.reference_image,
+                reference_image_id=self.reference_image_id,
+                project_id=self.project_id,
+                retry=self.retry,
+                timeout=self.timeout,
+                metadata=self.metadata,
+            )
+        except AlreadyExists:
+            self.log.info(
+                "ReferenceImage with id %s already exists. Exiting from the create operation.",
+                self.product_id,
+            )
+            return self.reference_image_id
+
+
+class CloudVisionAddProductToProductSetOperator(BaseOperator):
+    """
+    Adds a Product to the specified ProductSet. If the Product is already present, no change is made.
+
+    One Product can be added to at most 100 ProductSets.
+
+    Possible errors:
+
+    - Returns `NOT_FOUND` if the Product or the ProductSet doesn’t exist.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionAddProductToProductSetOperator`
+
+    :param product_set_id: (Required) The resource id for the ProductSet to modify.
+    :type product_set_id: str
+    :param product_id: (Required) The resource id of this Product.
+    :type product_id: str
+    :param location: (Required) The region where the ProductSet is located. Valid regions (as of 2019-02-05)
+        are: us-east1, us-west1, europe-west1, asia-east1
+    :type: str
+    :param project_id: (Optional) The project in which the Product is located. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_add_product_to_product_set_template_fields]
+    template_fields = ("location", "product_set_id", "product_id", "project_id", "gcp_conn_id")
+    # [END vision_add_product_to_product_set_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        product_set_id,
+        product_id,
+        location,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionAddProductToProductSetOperator, self).__init__(*args, **kwargs)
+        self.product_set_id = product_set_id
+        self.product_id = product_id
+        self.location = location
+        self.project_id = project_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+        return hook.add_product_to_product_set(
+            product_set_id=self.product_set_id,
+            product_id=self.product_id,
+            location=self.location,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+
+class CloudVisionRemoveProductFromProductSetOperator(BaseOperator):
+    """
+    Removes a Product from the specified ProductSet.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:CloudVisionRemoveProductFromProductSetOperator`
+
+    :param product_set_id: (Required) The resource id for the ProductSet to modify.
+    :type product_set_id: str
+    :param product_id: (Required) The resource id of this Product.
+    :type product_id: str
+    :param location: (Required) The region where the ProductSet is located. Valid regions (as of 2019-02-05)
+        are: us-east1, us-west1, europe-west1, asia-east1
+    :type: str
+    :param project_id: (Optional) The project in which the Product is located. If set to None or
+        missing, the default project_id from the GCP connection is used.
+    :type project_id: str
+    :param retry: (Optional) A retry object used to retry requests. If `None` is
+        specified, requests will not be retried.
+    :type retry: google.api_core.retry.Retry
+    :param timeout: (Optional) The amount of time, in seconds, to wait for the request to
+        complete. Note that if retry is specified, the timeout applies to each individual
+        attempt.
+    :type timeout: float
+    :param metadata: (Optional) Additional metadata that is provided to the method.
+    :type metadata: sequence[tuple[str, str]]
+    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START vision_remove_product_from_product_set_template_fields]
+    template_fields = ("location", "product_set_id", "product_id", "project_id", "gcp_conn_id")
+    # [END vision_remove_product_from_product_set_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        product_set_id,
+        product_id,
+        location,
+        project_id=None,
+        retry=None,
+        timeout=None,
+        metadata=None,
+        gcp_conn_id="google_cloud_default",
+        *args,
+        **kwargs
+    ):
+        super(CloudVisionRemoveProductFromProductSetOperator, self).__init__(*args, **kwargs)
+        self.product_set_id = product_set_id
+        self.product_id = product_id
+        self.location = location
+        self.project_id = project_id
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+
+    def execute(self, context):
+        hook = CloudVisionHook(gcp_conn_id=self.gcp_conn_id)
+        return hook.remove_product_from_product_set(
+            product_set_id=self.product_set_id,
+            product_id=self.product_id,
+            location=self.location,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
diff --git a/airflow/contrib/operators/gcs_acl_operator.py b/airflow/contrib/operators/gcs_acl_operator.py
new file mode 100644
index 0000000000000..03c72ce328c93
--- /dev/null
+++ b/airflow/contrib/operators/gcs_acl_operator.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class GoogleCloudStorageBucketCreateAclEntryOperator(BaseOperator):
+    """
+    Creates a new ACL entry on the specified bucket.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GoogleCloudStorageBucketCreateAclEntryOperator`
+
+    :param bucket: Name of a bucket.
+    :type bucket: str
+    :param entity: The entity holding the permission, in one of the following forms:
+        user-userId, user-email, group-groupId, group-email, domain-domain,
+        project-team-projectId, allUsers, allAuthenticatedUsers
+    :type entity: str
+    :param role: The access permission for the entity.
+        Acceptable values are: "OWNER", "READER", "WRITER".
+    :type role: str
+    :param user_project: (Optional) The project to be billed for this request.
+        Required for Requester Pays buckets.
+    :type user_project: str
+    :param google_cloud_storage_conn_id: The connection ID to use when
+        connecting to Google Cloud Storage.
+    :type google_cloud_storage_conn_id: str
+    """
+    # [START gcs_bucket_create_acl_template_fields]
+    template_fields = ('bucket', 'entity', 'role', 'user_project')
+    # [END gcs_bucket_create_acl_template_fields]
+
+    @apply_defaults
+    def __init__(self, bucket, entity, role, user_project=None,
+                 google_cloud_storage_conn_id='google_cloud_default', *args, **kwargs):
+        super(GoogleCloudStorageBucketCreateAclEntryOperator, self).__init__(*args,
+                                                                             **kwargs)
+        self.bucket = bucket
+        self.entity = entity
+        self.role = role
+        self.user_project = user_project
+        self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
+
+    def execute(self, context):
+        hook = GoogleCloudStorageHook(
+            google_cloud_storage_conn_id=self.google_cloud_storage_conn_id
+        )
+        hook.insert_bucket_acl(bucket=self.bucket, entity=self.entity, role=self.role,
+                               user_project=self.user_project)
+
+
+class GoogleCloudStorageObjectCreateAclEntryOperator(BaseOperator):
+    """
+    Creates a new ACL entry on the specified object.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GoogleCloudStorageObjectCreateAclEntryOperator`
+
+    :param bucket: Name of a bucket.
+    :type bucket: str
+    :param object_name: Name of the object. For information about how to URL encode object
+        names to be path safe, see:
+        https://cloud.google.com/storage/docs/json_api/#encoding
+    :type object_name: str
+    :param entity: The entity holding the permission, in one of the following forms:
+        user-userId, user-email, group-groupId, group-email, domain-domain,
+        project-team-projectId, allUsers, allAuthenticatedUsers
+    :type entity: str
+    :param role: The access permission for the entity.
+        Acceptable values are: "OWNER", "READER".
+    :type role: str
+    :param generation: (Optional) If present, selects a specific revision of this object
+        (as opposed to the latest version, the default).
+    :type generation: str
+    :param user_project: (Optional) The project to be billed for this request.
+        Required for Requester Pays buckets.
+    :type user_project: str
+    :param google_cloud_storage_conn_id: The connection ID to use when
+        connecting to Google Cloud Storage.
+    :type google_cloud_storage_conn_id: str
+    """
+    # [START gcs_object_create_acl_template_fields]
+    template_fields = ('bucket', 'object_name', 'entity', 'role', 'generation',
+                       'user_project')
+    # [END gcs_object_create_acl_template_fields]
+
+    @apply_defaults
+    def __init__(self,
+                 bucket,
+                 object_name,
+                 entity,
+                 role,
+                 generation=None,
+                 user_project=None,
+                 google_cloud_storage_conn_id='google_cloud_default',
+                 *args, **kwargs):
+        super(GoogleCloudStorageObjectCreateAclEntryOperator, self).__init__(*args,
+                                                                             **kwargs)
+        self.bucket = bucket
+        self.object_name = object_name
+        self.entity = entity
+        self.role = role
+        self.generation = generation
+        self.user_project = user_project
+        self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
+
+    def execute(self, context):
+        hook = GoogleCloudStorageHook(
+            google_cloud_storage_conn_id=self.google_cloud_storage_conn_id
+        )
+        hook.insert_object_acl(bucket=self.bucket, object_name=self.object_name,
+                               entity=self.entity, role=self.role,
+                               generation=self.generation, user_project=self.user_project)
diff --git a/airflow/contrib/operators/gcs_download_operator.py b/airflow/contrib/operators/gcs_download_operator.py
index ce272aedd501c..1d168d4660722 100644
--- a/airflow/contrib/operators/gcs_download_operator.py
+++ b/airflow/contrib/operators/gcs_download_operator.py
@@ -29,26 +29,26 @@ class GoogleCloudStorageDownloadOperator(BaseOperator):
     Downloads a file from Google Cloud Storage.
 
     :param bucket: The Google cloud storage bucket where the object is. (templated)
-    :type bucket: string
+    :type bucket: str
     :param object: The name of the object to download in the Google cloud
         storage bucket. (templated)
-    :type object: string
+    :type object: str
     :param filename: The file path on the local file system (where the
         operator is being executed) that the file should be downloaded to. (templated)
         If no filename passed, the downloaded data will not be stored on the local file
         system.
-    :type filename: string
+    :type filename: str
     :param store_to_xcom_key: If this param is set, the operator will push
         the contents of the downloaded file to XCom with the key set in this
         parameter. If not set, the downloaded data will not be pushed to XCom. (templated)
-    :type store_to_xcom_key: string
+    :type store_to_xcom_key: str
     :param google_cloud_storage_conn_id: The connection ID to use when
         connecting to Google cloud storage.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
     template_fields = ('bucket', 'object', 'filename', 'store_to_xcom_key',)
     ui_color = '#f0eee4'
diff --git a/airflow/contrib/operators/gcs_list_operator.py b/airflow/contrib/operators/gcs_list_operator.py
index 6474453afabec..056b349394016 100644
--- a/airflow/contrib/operators/gcs_list_operator.py
+++ b/airflow/contrib/operators/gcs_list_operator.py
@@ -17,6 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from typing import Iterable
+
 from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
@@ -30,21 +32,21 @@ class GoogleCloudStorageListOperator(BaseOperator):
      `xcom` in the downstream task.
 
     :param bucket: The Google cloud storage bucket to find the objects. (templated)
-    :type bucket: string
+    :type bucket: str
     :param prefix: Prefix string which filters objects whose name begin with
            this prefix. (templated)
-    :type prefix: string
+    :type prefix: str
     :param delimiter: The delimiter by which you want to filter the objects. (templated)
         For e.g to lists the CSV files from in a directory in GCS you would use
         delimiter='.csv'.
-    :type delimiter: string
+    :type delimiter: str
     :param google_cloud_storage_conn_id: The connection ID to use when
         connecting to Google cloud storage.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
 
     **Example**:
         The following Operator would list all the Avro files from ``sales/sales-2017``
@@ -58,7 +60,7 @@ class GoogleCloudStorageListOperator(BaseOperator):
                 google_cloud_storage_conn_id=google_cloud_conn_id
             )
     """
-    template_fields = ('bucket', 'prefix', 'delimiter')
+    template_fields = ('bucket', 'prefix', 'delimiter')  # type: Iterable[str]
     ui_color = '#f0eee4'
 
     @apply_defaults
diff --git a/airflow/contrib/operators/gcs_operator.py b/airflow/contrib/operators/gcs_operator.py
index ef5e8de5043f3..31c8f8fa0c91c 100644
--- a/airflow/contrib/operators/gcs_operator.py
+++ b/airflow/contrib/operators/gcs_operator.py
@@ -33,7 +33,11 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator):
             https://cloud.google.com/storage/docs/bucketnaming.html#requirements
 
     :param bucket_name: The name of the bucket. (templated)
-    :type bucket_name: string
+    :type bucket_name: str
+    :param resource: An optional dict with parameters for creating the bucket.
+            For information on available parameters, see Cloud Storage API doc:
+            https://cloud.google.com/storage/docs/json_api/v1/buckets/insert
+    :type resource: dict
     :param storage_class: This defines how objects in the bucket are stored
             and determines the SLA and the cost of storage (templated). Values include
 
@@ -42,41 +46,41 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator):
             - ``STANDARD``
             - ``NEARLINE``
             - ``COLDLINE``.
+
             If this value is not specified when the bucket is
             created, it will default to STANDARD.
-    :type storage_class: string
+    :type storage_class: str
     :param location: The location of the bucket. (templated)
         Object data for objects in the bucket resides in physical storage
         within this region. Defaults to US.
 
-        .. seealso::
-            https://developers.google.com/storage/docs/bucket-locations
+        .. seealso:: https://developers.google.com/storage/docs/bucket-locations
 
-    :type location: string
+    :type location: str
     :param project_id: The ID of the GCP Project. (templated)
-    :type project_id: string
+    :type project_id: str
     :param labels: User-provided labels, in key/value pairs.
     :type labels: dict
     :param google_cloud_storage_conn_id: The connection ID to use when
         connecting to Google cloud storage.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must
         have domain-wide delegation enabled.
-    :type delegate_to: string
-
-    **Example**:
-        The following Operator would create a new bucket ``test-bucket``
-        with ``MULTI_REGIONAL`` storage class in ``EU`` region ::
-
-            CreateBucket = GoogleCloudStorageCreateBucketOperator(
-                task_id='CreateNewBucket',
-                bucket_name='test-bucket',
-                storage_class='MULTI_REGIONAL',
-                location='EU',
-                labels={'env': 'dev', 'team': 'airflow'},
-                google_cloud_storage_conn_id='airflow-service-account'
-            )
+    :type delegate_to: str
+
+    :Example::
+    The following Operator would create a new bucket ``test-bucket``
+    with ``MULTI_REGIONAL`` storage class in ``EU`` region ::
+
+        CreateBucket = GoogleCloudStorageCreateBucketOperator(
+            task_id='CreateNewBucket',
+            bucket_name='test-bucket',
+            storage_class='MULTI_REGIONAL',
+            location='EU',
+            labels={'env': 'dev', 'team': 'airflow'},
+            google_cloud_storage_conn_id='airflow-service-account'
+        )
     """
 
     template_fields = ('bucket_name', 'storage_class',
@@ -86,6 +90,7 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator):
     @apply_defaults
     def __init__(self,
                  bucket_name,
+                 resource=None,
                  storage_class='MULTI_REGIONAL',
                  location='US',
                  project_id=None,
@@ -96,6 +101,7 @@ def __init__(self,
                  **kwargs):
         super(GoogleCloudStorageCreateBucketOperator, self).__init__(*args, **kwargs)
         self.bucket_name = bucket_name
+        self.resource = resource
         self.storage_class = storage_class
         self.location = location
         self.project_id = project_id
@@ -116,6 +122,7 @@ def execute(self, context):
         )
 
         hook.create_bucket(bucket_name=self.bucket_name,
+                           resource=self.resource,
                            storage_class=self.storage_class,
                            location=self.location,
                            project_id=self.project_id,
diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py
index 3a7798030cf49..4fd1f6a4ef88c 100644
--- a/airflow/contrib/operators/gcs_to_bq.py
+++ b/airflow/contrib/operators/gcs_to_bq.py
@@ -34,43 +34,48 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator):
     point the operator to a Google cloud storage object name. The object in
     Google cloud storage must be a JSON file with the schema fields in it.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GoogleCloudStorageToBigQueryOperator`
+
     :param bucket: The bucket to load from. (templated)
-    :type bucket: string
+    :type bucket: str
     :param source_objects: List of Google cloud storage URIs to load from. (templated)
         If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI.
-    :type object: list
-    :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table>
-        BigQuery table to load data into. If <project> is not included,
-        project will be the project defined in the connection json. (templated)
-    :type destination_project_dataset_table: string
+    :type source_objects: list[str]
+    :param destination_project_dataset_table: The dotted
+        ``(<project>.|<project>:)<dataset>.<table>`` BigQuery table to load data into.
+        If ``<project>`` is not included, project will be the project defined in
+        the connection json. (templated)
+    :type destination_project_dataset_table: str
     :param schema_fields: If set, the schema field list as defined here:
         https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load
         Should not be set when source_format is 'DATASTORE_BACKUP'.
     :type schema_fields: list
     :param schema_object: If set, a GCS object path pointing to a .json file that
         contains the schema for the table. (templated)
-    :param schema_object: string
+    :type schema_object: str
     :param source_format: File format to export.
-    :type source_format: string
+    :type source_format: str
     :param compression: [Optional] The compression type of the data source.
         Possible values include GZIP and NONE.
         The default value is NONE.
         This setting is ignored for Google Cloud Bigtable,
         Google Cloud Datastore backups and Avro formats.
-    :type compression: string
+    :type compression: str
     :param create_disposition: The create disposition if the table doesn't exist.
-    :type create_disposition: string
+    :type create_disposition: str
     :param skip_leading_rows: Number of rows to skip when loading from a CSV.
     :type skip_leading_rows: int
     :param write_disposition: The write disposition if the table already exists.
-    :type write_disposition: string
+    :type write_disposition: str
     :param field_delimiter: The delimiter to use when loading from a CSV.
-    :type field_delimiter: string
+    :type field_delimiter: str
     :param max_bad_records: The maximum number of bad records that BigQuery can
         ignore when running the job.
     :type max_bad_records: int
     :param quote_character: The value that is used to quote data sections in a CSV file.
-    :type quote_character: string
+    :type quote_character: str
     :param ignore_unknown_values: [Optional] Indicates if BigQuery should allow
         extra values that are not represented in the table schema.
         If true, the extra values are ignored. If false, records with extra columns
@@ -78,7 +83,7 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator):
         invalid error is returned in the job result.
     :type ignore_unknown_values: bool
     :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).
-    :type allow_quoted_newlines: boolean
+    :type allow_quoted_newlines: bool
     :param allow_jagged_rows: Accept rows that are missing trailing optional columns.
         The missing values are treated as nulls. If false, records with missing trailing
         columns are treated as bad records, and if there are too many bad records, an
@@ -86,21 +91,21 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator):
         for other formats.
     :type allow_jagged_rows: bool
     :param max_id_key: If set, the name of a column in the BigQuery table
-        that's to be loaded. Thsi will be used to select the MAX value from
+        that's to be loaded. This will be used to select the MAX value from
         BigQuery after the load occurs. The results will be returned by the
         execute() command, which in turn gets stored in XCom for future
         operators to use. This can be helpful with incremental loads--during
         future executions, you can pick up from the max ID.
-    :type max_id_key: string
+    :type max_id_key: str
     :param bigquery_conn_id: Reference to a specific BigQuery hook.
-    :type bigquery_conn_id: string
+    :type bigquery_conn_id: str
     :param google_cloud_storage_conn_id: Reference to a specific Google
         cloud storage hook.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any. For this to
         work, the service account making the request must have domain-wide
         delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param schema_update_options: Allows the schema of the destination
         table to be updated as a side effect of the load job.
     :type schema_update_options: list
@@ -114,6 +119,14 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator):
         Note that 'field' is not available in concurrency with
         dataset.table$partition.
     :type time_partitioning: dict
+    :param cluster_fields: Request that the result of this load be stored sorted
+        by one or more columns. This is only available in conjunction with
+        time_partitioning. The order of columns given determines the sort order.
+        Not applicable for external tables.
+    :type cluster_fields: list[str]
+    :param autodetect: [Optional] Indicates if we should automatically infer the
+        options and schema for CSV and JSON sources. (Default: ``False``)
+    :type autodetect: bool
     """
     template_fields = ('bucket', 'source_objects',
                        'schema_object', 'destination_project_dataset_table')
@@ -143,14 +156,20 @@ def __init__(self,
                  google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
                  schema_update_options=(),
-                 src_fmt_configs={},
+                 src_fmt_configs=None,
                  external_table=False,
-                 time_partitioning={},
+                 time_partitioning=None,
+                 cluster_fields=None,
+                 autodetect=False,
                  *args, **kwargs):
 
         super(GoogleCloudStorageToBigQueryOperator, self).__init__(*args, **kwargs)
 
         # GCS config
+        if src_fmt_configs is None:
+            src_fmt_configs = {}
+        if time_partitioning is None:
+            time_partitioning = {}
         self.bucket = bucket
         self.source_objects = source_objects
         self.schema_object = schema_object
@@ -179,20 +198,27 @@ def __init__(self,
         self.schema_update_options = schema_update_options
         self.src_fmt_configs = src_fmt_configs
         self.time_partitioning = time_partitioning
+        self.cluster_fields = cluster_fields
+        self.autodetect = autodetect
 
     def execute(self, context):
         bq_hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
                                delegate_to=self.delegate_to)
 
-        if not self.schema_fields and \
-                self.schema_object and \
-                self.source_format != 'DATASTORE_BACKUP':
-            gcs_hook = GoogleCloudStorageHook(
-                google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
-                delegate_to=self.delegate_to)
-            schema_fields = json.loads(gcs_hook.download(
-                self.bucket,
-                self.schema_object).decode("utf-8"))
+        if not self.schema_fields:
+            if self.schema_object and self.source_format != 'DATASTORE_BACKUP':
+                gcs_hook = GoogleCloudStorageHook(
+                    google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
+                    delegate_to=self.delegate_to)
+                schema_fields = json.loads(gcs_hook.download(
+                    self.bucket,
+                    self.schema_object).decode("utf-8"))
+            elif self.schema_object is None and self.autodetect is False:
+                raise ValueError('At least one of `schema_fields`, `schema_object`, '
+                                 'or `autodetect` must be passed.')
+            else:
+                schema_fields = None
+
         else:
             schema_fields = self.schema_fields
 
@@ -223,6 +249,7 @@ def execute(self, context):
                 schema_fields=schema_fields,
                 source_uris=source_uris,
                 source_format=self.source_format,
+                autodetect=self.autodetect,
                 create_disposition=self.create_disposition,
                 skip_leading_rows=self.skip_leading_rows,
                 write_disposition=self.write_disposition,
@@ -234,7 +261,8 @@ def execute(self, context):
                 allow_jagged_rows=self.allow_jagged_rows,
                 schema_update_options=self.schema_update_options,
                 src_fmt_configs=self.src_fmt_configs,
-                time_partitioning=self.time_partitioning)
+                time_partitioning=self.time_partitioning,
+                cluster_fields=self.cluster_fields)
 
         if self.max_id_key:
             cursor.execute('SELECT MAX({}) FROM {}'.format(
diff --git a/airflow/contrib/operators/gcs_to_gcs.py b/airflow/contrib/operators/gcs_to_gcs.py
index 256685f90b2e1..b694b802fc878 100644
--- a/airflow/contrib/operators/gcs_to_gcs.py
+++ b/airflow/contrib/operators/gcs_to_gcs.py
@@ -28,79 +28,86 @@ class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator):
 
     :param source_bucket: The source Google cloud storage bucket where the
          object is. (templated)
-    :type source_bucket: string
+    :type source_bucket: str
     :param source_object: The source name of the object to copy in the Google cloud
         storage bucket. (templated)
-        If wildcards are used in this argument:
-            You can use only one wildcard for objects (filenames) within your
-            bucket. The wildcard can appear inside the object name or at the
-            end of the object name. Appending a wildcard to the bucket name is
-            unsupported.
-    :type source_object: string
+        You can use only one wildcard for objects (filenames) within your
+        bucket. The wildcard can appear inside the object name or at the
+        end of the object name. Appending a wildcard to the bucket name is
+        unsupported.
+    :type source_object: str
     :param destination_bucket: The destination Google cloud storage bucket
-    where the object should be. (templated)
-    :type destination_bucket: string
+        where the object should be. (templated)
+    :type destination_bucket: str
     :param destination_object: The destination name of the object in the
         destination Google cloud storage bucket. (templated)
         If a wildcard is supplied in the source_object argument, this is the
         prefix that will be prepended to the final destination objects' paths.
         Note that the source path's part before the wildcard will be removed;
         if it needs to be retained it should be appended to destination_object.
-        For example, with prefix ``foo/*`` and destination_object `'blah/``, the
+        For example, with prefix ``foo/*`` and destination_object ``blah/``, the
         file ``foo/baz`` will be copied to ``blah/baz``; to retain the prefix write
         the destination_object as e.g. ``blah/foo``, in which case the copied file
         will be named ``blah/foo/baz``.
-    :type destination_object: string
+    :type destination_object: str
     :param move_object: When move object is True, the object is moved instead
-    of copied to the new location.
-                        This is the equivalent of a mv command as opposed to a
-                        cp command.
+        of copied to the new location. This is the equivalent of a mv command
+        as opposed to a cp command.
     :type move_object: bool
     :param google_cloud_storage_conn_id: The connection ID to use when
         connecting to Google cloud storage.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
-
-    **Examples**:
-        The following Operator would copy a single file named
-        ``sales/sales-2017/january.avro`` in the ``data`` bucket to the file named
-        ``copied_sales/2017/january-backup.avro` in the ``data_backup`` bucket ::
-            copy_single_file = GoogleCloudStorageToGoogleCloudStorageOperator(
-                task_id='copy_single_file',
-                source_bucket='data',
-                source_object='sales/sales-2017/january.avro',
-                destination_bucket='data_backup',
-                destination_object='copied_sales/2017/january-backup.avro',
-                google_cloud_storage_conn_id=google_cloud_conn_id
-            )
+    :type delegate_to: str
+    :param last_modified_time: When specified, if the object(s) were
+        modified after last_modified_time, they will be copied/moved.
+        If tzinfo has not been set, UTC will be assumed.
+    :type last_modified_time: datetime.datetime
 
-        The following Operator would copy all the Avro files from ``sales/sales-2017``
-        folder (i.e. with names starting with that prefix) in ``data`` bucket to the
-        ``copied_sales/2017`` folder in the ``data_backup`` bucket. ::
-            copy_files = GoogleCloudStorageToGoogleCloudStorageOperator(
-                task_id='copy_files',
-                source_bucket='data',
-                source_object='sales/sales-2017/*.avro',
-                destination_bucket='data_backup',
-                destination_object='copied_sales/2017/',
-                google_cloud_storage_conn_id=google_cloud_conn_id
-            )
+    :Example:
+
+    The following Operator would copy a single file named
+    ``sales/sales-2017/january.avro`` in the ``data`` bucket to the file named
+    ``copied_sales/2017/january-backup.avro`` in the ``data_backup`` bucket ::
+
+        copy_single_file = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id='copy_single_file',
+            source_bucket='data',
+            source_object='sales/sales-2017/january.avro',
+            destination_bucket='data_backup',
+            destination_object='copied_sales/2017/january-backup.avro',
+            google_cloud_storage_conn_id=google_cloud_conn_id
+        )
+
+    The following Operator would copy all the Avro files from ``sales/sales-2017``
+    folder (i.e. with names starting with that prefix) in ``data`` bucket to the
+    ``copied_sales/2017`` folder in the ``data_backup`` bucket. ::
+
+        copy_files = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id='copy_files',
+            source_bucket='data',
+            source_object='sales/sales-2017/*.avro',
+            destination_bucket='data_backup',
+            destination_object='copied_sales/2017/',
+            google_cloud_storage_conn_id=google_cloud_conn_id
+        )
+
+    The following Operator would move all the Avro files from ``sales/sales-2017``
+    folder (i.e. with names starting with that prefix) in ``data`` bucket to the
+    same folder in the ``data_backup`` bucket, deleting the original files in the
+    process. ::
+
+        move_files = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id='move_files',
+            source_bucket='data',
+            source_object='sales/sales-2017/*.avro',
+            destination_bucket='data_backup',
+            move_object=True,
+            google_cloud_storage_conn_id=google_cloud_conn_id
+        )
 
-        The following Operator would move all the Avro files from ``sales/sales-2017``
-        folder (i.e. with names starting with that prefix) in ``data`` bucket to the
-        same folder in the ``data_backup`` bucket, deleting the original files in the
-        process. ::
-            move_files = GoogleCloudStorageToGoogleCloudStorageOperator(
-                task_id='move_files',
-                source_bucket='data',
-                source_object='sales/sales-2017/*.avro',
-                destination_bucket='data_backup',
-                move_object=True,
-                google_cloud_storage_conn_id=google_cloud_conn_id
-            )
     """
     template_fields = ('source_bucket', 'source_object', 'destination_bucket',
                        'destination_object',)
@@ -115,6 +122,7 @@ def __init__(self,
                  move_object=False,
                  google_cloud_storage_conn_id='google_cloud_default',
                  delegate_to=None,
+                 last_modified_time=None,
                  *args,
                  **kwargs):
         super(GoogleCloudStorageToGoogleCloudStorageOperator,
@@ -126,6 +134,7 @@ def __init__(self,
         self.move_object = move_object
         self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
         self.delegate_to = delegate_to
+        self.last_modified_time = last_modified_time
         self.wildcard = '*'
 
     def execute(self, context):
@@ -141,6 +150,13 @@ def execute(self, context):
             objects = hook.list(self.source_bucket, prefix=prefix, delimiter=delimiter)
 
             for source_object in objects:
+                if self.last_modified_time is not None:
+                    # Check to see if object was modified after last_modified_time
+                    if hook.is_updated_after(self.source_bucket, source_object,
+                                             self.last_modified_time):
+                        pass
+                    else:
+                        continue
                 if self.destination_object is None:
                     destination_object = source_object
                 else:
@@ -157,6 +173,14 @@ def execute(self, context):
                     hook.delete(self.source_bucket, source_object)
 
         else:
+            if self.last_modified_time is not None:
+                if hook.is_updated_after(self.source_bucket,
+                                         self.source_object,
+                                         self.last_modified_time):
+                    pass
+                else:
+                    return
+
             self.log.info(
                 log_message.format(self.source_bucket, self.source_object,
                                    self.destination_bucket or self.source_bucket,
diff --git a/airflow/contrib/operators/gcs_to_gcs_transfer_operator.py b/airflow/contrib/operators/gcs_to_gcs_transfer_operator.py
new file mode 100644
index 0000000000000..41be57a1bbfed
--- /dev/null
+++ b/airflow/contrib/operators/gcs_to_gcs_transfer_operator.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import warnings
+
+from airflow.contrib.operators.gcp_transfer_operator import (  # noqa
+    GoogleCloudStorageToGoogleCloudStorageTransferOperator,
+)
+
+warnings.warn(
+    "This module is deprecated. Please use `airflow.contrib.operators.gcp_transfer_operator`",
+    DeprecationWarning,
+)
diff --git a/airflow/contrib/operators/gcs_to_s3.py b/airflow/contrib/operators/gcs_to_s3.py
index a87aa3af5c531..6029661f370c6 100644
--- a/airflow/contrib/operators/gcs_to_s3.py
+++ b/airflow/contrib/operators/gcs_to_s3.py
@@ -28,25 +28,36 @@ class GoogleCloudStorageToS3Operator(GoogleCloudStorageListOperator):
     Synchronizes a Google Cloud Storage bucket with an S3 bucket.
 
     :param bucket: The Google Cloud Storage bucket to find the objects. (templated)
-    :type bucket: string
+    :type bucket: str
     :param prefix: Prefix string which filters objects whose name begin with
         this prefix. (templated)
-    :type prefix: string
+    :type prefix: str
     :param delimiter: The delimiter by which you want to filter the objects. (templated)
         For e.g to lists the CSV files from in a directory in GCS you would use
         delimiter='.csv'.
-    :type delimiter: string
+    :type delimiter: str
     :param google_cloud_storage_conn_id: The connection ID to use when
         connecting to Google Cloud Storage.
-    :type google_cloud_storage_conn_id: string
+    :type google_cloud_storage_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param dest_aws_conn_id: The destination S3 connection
     :type dest_aws_conn_id: str
     :param dest_s3_key: The base S3 key to be used to store the files. (templated)
     :type dest_s3_key: str
+    :param dest_verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type dest_verify: bool or str
     """
     template_fields = ('bucket', 'prefix', 'delimiter', 'dest_s3_key')
     ui_color = '#f0eee4'
@@ -60,6 +71,7 @@ def __init__(self,
                  delegate_to=None,
                  dest_aws_conn_id=None,
                  dest_s3_key=None,
+                 dest_verify=None,
                  replace=False,
                  *args,
                  **kwargs):
@@ -75,12 +87,13 @@ def __init__(self,
         )
         self.dest_aws_conn_id = dest_aws_conn_id
         self.dest_s3_key = dest_s3_key
+        self.dest_verify = dest_verify
         self.replace = replace
 
     def execute(self, context):
         # use the super to list all files in an Google Cloud Storage bucket
         files = super(GoogleCloudStorageToS3Operator, self).execute(context)
-        s3_hook = S3Hook(aws_conn_id=self.dest_aws_conn_id)
+        s3_hook = S3Hook(aws_conn_id=self.dest_aws_conn_id, verify=self.dest_verify)
 
         if not self.replace:
             # if we are not replacing -> list all files in the S3 bucket
@@ -88,7 +101,7 @@ def execute(self, context):
             # Google Cloud Storage and not in S3
             bucket_name, _ = S3Hook.parse_s3_url(self.dest_s3_key)
             existing_files = s3_hook.list_keys(bucket_name)
-            files = set(files) - set(existing_files)
+            files = list(set(files) - set(existing_files))
 
         if files:
             hook = GoogleCloudStorageHook(
diff --git a/airflow/contrib/operators/imap_attachment_to_s3_operator.py b/airflow/contrib/operators/imap_attachment_to_s3_operator.py
new file mode 100644
index 0000000000000..3b9f143591efb
--- /dev/null
+++ b/airflow/contrib/operators/imap_attachment_to_s3_operator.py
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.imap_hook import ImapHook
+from airflow.hooks.S3_hook import S3Hook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class ImapAttachmentToS3Operator(BaseOperator):
+    """
+    Transfers a mail attachment from a mail server into s3 bucket.
+
+    :param imap_attachment_name: The file name of the mail attachment that you want to transfer.
+    :type imap_attachment_name: str
+    :param s3_key: The destination file name in the s3 bucket for the attachment.
+    :type s3_key: str
+    :param imap_mail_folder: The folder on the mail server to look for the attachment.
+    :type imap_mail_folder: str
+    :param imap_check_regex: If set checks the `imap_attachment_name` for a regular expression.
+    :type imap_check_regex: bool
+    :param s3_overwrite: If set overwrites the s3 key if already exists.
+    :type s3_overwrite: bool
+    :param imap_conn_id: The reference to the connection details of the mail server.
+    :type imap_conn_id: str
+    :param s3_conn_id: The reference to the s3 connection details.
+    :type s3_conn_id: str
+    """
+    template_fields = ('imap_attachment_name', 's3_key')
+
+    @apply_defaults
+    def __init__(self,
+                 imap_attachment_name,
+                 s3_key,
+                 imap_mail_folder='INBOX',
+                 imap_check_regex=False,
+                 s3_overwrite=False,
+                 imap_conn_id='imap_default',
+                 s3_conn_id='aws_default',
+                 *args,
+                 **kwargs):
+        super(ImapAttachmentToS3Operator, self).__init__(*args, **kwargs)
+        self.imap_attachment_name = imap_attachment_name
+        self.s3_key = s3_key
+        self.imap_mail_folder = imap_mail_folder
+        self.imap_check_regex = imap_check_regex
+        self.s3_overwrite = s3_overwrite
+        self.imap_conn_id = imap_conn_id
+        self.s3_conn_id = s3_conn_id
+
+    def execute(self, context):
+        """
+        This function executes the transfer from the email server (via imap) into s3.
+
+        :param context: The context while executing.
+        :type context: dict
+        """
+        self.log.info(
+            'Transferring mail attachment %s from mail server via imap to s3 key %s...',
+            self.imap_attachment_name, self.s3_key
+        )
+
+        with ImapHook(imap_conn_id=self.imap_conn_id) as imap_hook:
+            imap_mail_attachments = imap_hook.retrieve_mail_attachments(
+                name=self.imap_attachment_name,
+                mail_folder=self.imap_mail_folder,
+                check_regex=self.imap_check_regex,
+                latest_only=True
+            )
+
+        s3_hook = S3Hook(aws_conn_id=self.s3_conn_id)
+        s3_hook.load_bytes(bytes_data=imap_mail_attachments[0][1], key=self.s3_key)
diff --git a/airflow/contrib/operators/jenkins_job_trigger_operator.py b/airflow/contrib/operators/jenkins_job_trigger_operator.py
index 37185f3e13f42..3af87f3f3bed7 100644
--- a/airflow/contrib/operators/jenkins_job_trigger_operator.py
+++ b/airflow/contrib/operators/jenkins_job_trigger_operator.py
@@ -26,36 +26,27 @@
 from airflow.contrib.hooks.jenkins_hook import JenkinsHook
 import jenkins
 from jenkins import JenkinsException
-from six.moves.urllib.request import Request, urlopen
+from requests import Request
+import six
 from six.moves.urllib.error import HTTPError, URLError
 
-try:
-    basestring
-except NameError:
-    basestring = str  # For python3 compatibility
 
-
-# TODO Use jenkins_urlopen instead when it will be available
-# in the stable python-jenkins version (> 0.4.15)
-def jenkins_request_with_headers(jenkins_server, req, add_crumb=True):
+def jenkins_request_with_headers(jenkins_server, req):
     """
     We need to get the headers in addition to the body answer
     to get the location from them
-    This function is just a copy of the one present in python-jenkins library
+    This function uses jenkins_request method from python-jenkins library
     with just the return call changed
+
     :param jenkins_server: The server to query
     :param req: The request to execute
-    :param add_crumb: Boolean to indicate if it should add crumb to the request
-    :return:
+    :return: Dict containing the response body (key body)
+        and the headers coming along (headers)
     """
     try:
-        if jenkins_server.auth:
-            req.add_header('Authorization', jenkins_server.auth)
-        if add_crumb:
-            jenkins_server.maybe_add_crumb(req)
-        response = urlopen(req, timeout=jenkins_server.timeout)
-        response_body = response.read()
-        response_headers = response.info()
+        response = jenkins_server.jenkins_request(req)
+        response_body = response.content
+        response_headers = response.headers
         if response_body is None:
             raise jenkins.EmptyResponseException(
                 "Error communicating with server[%s]: "
@@ -94,14 +85,15 @@ class JenkinsJobTriggerOperator(BaseOperator):
     This operator depend on python-jenkins library,
     version >= 0.4.15 to communicate with jenkins server.
     You'll also need to configure a Jenkins connection in the connections screen.
+
     :param jenkins_connection_id: The jenkins connection to use for this job
-    :type jenkins_connection_id: string
+    :type jenkins_connection_id: str
     :param job_name: The name of the job to trigger
-    :type job_name: string
+    :type job_name: str
     :param parameters: The parameters block to provide to jenkins. (templated)
-    :type parameters: string
+    :type parameters: str
     :param sleep_time: How long will the operator sleep between each status
-    request for the job (min 1, default 10)
+        request for the job (min 1, default 10)
     :type sleep_time: int
     :param max_try_before_job_appears: The maximum number of requests to make
         while waiting for the job to appears on jenkins server (default 10)
@@ -135,13 +127,14 @@ def build_job(self, jenkins_server):
         It returned a dict with 2 keys : body and headers.
         headers contains also a dict-like object which can be queried to get
         the location to poll in the queue.
+
         :param jenkins_server: The jenkins server where the job should be triggered
         :return: Dict containing the response body (key body)
-        and the headers coming along (headers)
+            and the headers coming along (headers)
         """
         # Warning if the parameter is too long, the URL can be longer than
         # the maximum allowed size
-        if self.parameters and isinstance(self.parameters, basestring):
+        if self.parameters and isinstance(self.parameters, six.string_types):
             import ast
             self.parameters = ast.literal_eval(self.parameters)
 
@@ -150,7 +143,7 @@ def build_job(self, jenkins_server):
             self.parameters = None
 
         request = Request(jenkins_server.build_job_url(self.job_name,
-                                                       self.parameters, None), b'')
+                                                       self.parameters, None))
         return jenkins_request_with_headers(jenkins_server, request)
 
     def poll_job_in_queue(self, location, jenkins_server):
@@ -163,6 +156,7 @@ def poll_job_in_queue(self, location, jenkins_server):
         returned by the build_job call and poll this file.
         When a 'executable' block appears in the json, it means the job execution started
         and the field 'number' then contains the build number.
+
         :param location: Location to poll, returned in the header of the build_job call
         :param jenkins_server: The jenkins server to poll
         :return: The build_number corresponding to the triggered job
diff --git a/airflow/contrib/operators/kubernetes_pod_operator.py b/airflow/contrib/operators/kubernetes_pod_operator.py
index bf656f12fbb88..fcbdcfae55328 100644
--- a/airflow/contrib/operators/kubernetes_pod_operator.py
+++ b/airflow/contrib/operators/kubernetes_pod_operator.py
@@ -21,13 +21,6 @@
 from airflow.contrib.kubernetes import kube_client, pod_generator, pod_launcher
 from airflow.contrib.kubernetes.pod import Resources
 from airflow.utils.state import State
-from airflow.contrib.kubernetes.volume_mount import VolumeMount  # noqa
-from airflow.contrib.kubernetes.volume import Volume  # noqa
-from airflow.contrib.kubernetes.secret import Secret  # noqa
-
-template_fields = ('templates_dict',)
-template_ext = tuple()
-ui_color = '#ffefeb'
 
 
 class KubernetesPodOperator(BaseOperator):
@@ -37,18 +30,24 @@ class KubernetesPodOperator(BaseOperator):
     :param image: Docker image you wish to launch. Defaults to dockerhub.io,
         but fully qualified URLS will point to custom repositories
     :type image: str
-    :param: namespace: the namespace to run within kubernetes
-    :type: namespace: str
+    :param namespace: the namespace to run within kubernetes
+    :type namespace: str
     :param cmds: entrypoint of the container. (templated)
         The docker images's entrypoint is used if this is not provide.
-    :type cmds: list of str
-    :param arguments: arguments of to the entrypoint. (templated)
+    :type cmds: list[str]
+    :param arguments: arguments of the entrypoint. (templated)
         The docker image's CMD is used if this is not provided.
-    :type arguments: list of str
+    :type arguments: list[str]
+    :param image_pull_policy: Specify a policy to cache or always pull an image
+    :type image_pull_policy: str
+    :param image_pull_secrets: Any image pull secrets to be given to the pod.
+                               If more than one secret is required, provide a
+                               comma separated list: secret_a,secret_b
+    :type image_pull_secrets: str
     :param volume_mounts: volumeMounts for launched pod
-    :type volume_mounts: list of VolumeMount
+    :type volume_mounts: list[airflow.contrib.kubernetes.volume_mount.VolumeMount]
     :param volumes: volumes for launched pod. Includes ConfigMaps and PersistentVolumes
-    :type volumes: list of Volume
+    :type volumes: list[airflow.contrib.kubernetes.volume.Volume]
     :param labels: labels to apply to the Pod
     :type labels: dict
     :param startup_timeout_seconds: timeout in seconds to startup the pod
@@ -60,22 +59,35 @@ class KubernetesPodOperator(BaseOperator):
     :type env_vars: dict
     :param secrets: Kubernetes secrets to inject in the container,
         They can be exposed as environment vars or files in a volume.
-    :type secrets: list of Secret
+    :type secrets: list[airflow.contrib.kubernetes.secret.Secret]
     :param in_cluster: run kubernetes client with in_cluster configuration
     :type in_cluster: bool
     :param cluster_context: context that points to kubernetes cluster.
         Ignored when in_cluster is True. If None, current-context is used.
-    :type cluster_context: string
+    :type cluster_context: str
     :param get_logs: get the stdout of the container as logs of the tasks
     :type get_logs: bool
     :param affinity: A dict containing a group of affinity scheduling rules
     :type affinity: dict
+    :param node_selectors: A dict containing a group of scheduling rules
+    :type node_selectors: dict
     :param config_file: The path to the Kubernetes config file
     :type config_file: str
     :param xcom_push: If xcom_push is True, the content of the file
         /airflow/xcom/return.json in the container will also be pushed to an
         XCom when the container completes.
     :type xcom_push: bool
+    :param is_delete_operator_pod: What to do when the pod reaches its final
+        state, or the execution is interrupted.
+        If False (default): do nothing, If True: delete the pod
+    :type is_delete_operator_pod: bool
+    :param hostnetwork: If True enable host networking on the pod
+    :type hostnetwork: bool
+    :param tolerations: A list of kubernetes tolerations
+    :type tolerations: list tolerations
+    :param configmaps: A list of configmap names objects that we
+        want mount as env variables
+    :type configmaps: list[str]
     """
     template_fields = ('cmds', 'arguments', 'env_vars', 'config_file')
 
@@ -100,19 +112,31 @@ def execute(self, context):
                 labels=self.labels,
             )
 
+            pod.service_account_name = self.service_account_name
             pod.secrets = self.secrets
             pod.envs = self.env_vars
             pod.image_pull_policy = self.image_pull_policy
+            pod.image_pull_secrets = self.image_pull_secrets
             pod.annotations = self.annotations
             pod.resources = self.resources
             pod.affinity = self.affinity
+            pod.node_selectors = self.node_selectors
+            pod.hostnetwork = self.hostnetwork
+            pod.tolerations = self.tolerations
+            pod.configmaps = self.configmaps
+            pod.security_context = self.security_context
 
             launcher = pod_launcher.PodLauncher(kube_client=client,
                                                 extract_xcom=self.xcom_push)
-            (final_state, result) = launcher.run_pod(
-                pod,
-                startup_timeout=self.startup_timeout_seconds,
-                get_logs=self.get_logs)
+            try:
+                (final_state, result) = launcher.run_pod(
+                    pod,
+                    startup_timeout=self.startup_timeout_seconds,
+                    get_logs=self.get_logs)
+            finally:
+                if self.is_delete_operator_pod:
+                    launcher.delete_pod(pod)
+
             if final_state != State.SUCCESS:
                 raise AirflowException(
                     'Pod returned a failure: {state}'.format(state=final_state)
@@ -144,6 +168,14 @@ def __init__(self,
                  affinity=None,
                  config_file=None,
                  xcom_push=False,
+                 node_selectors=None,
+                 image_pull_secrets=None,
+                 service_account_name="default",
+                 is_delete_operator_pod=False,
+                 hostnetwork=False,
+                 tolerations=None,
+                 configmaps=None,
+                 security_context=None,
                  *args,
                  **kwargs):
         super(KubernetesPodOperator, self).__init__(*args, **kwargs)
@@ -162,8 +194,16 @@ def __init__(self,
         self.cluster_context = cluster_context
         self.get_logs = get_logs
         self.image_pull_policy = image_pull_policy
+        self.node_selectors = node_selectors or {}
         self.annotations = annotations or {}
         self.affinity = affinity or {}
         self.xcom_push = xcom_push
         self.resources = resources or Resources()
         self.config_file = config_file
+        self.image_pull_secrets = image_pull_secrets
+        self.service_account_name = service_account_name
+        self.is_delete_operator_pod = is_delete_operator_pod
+        self.hostnetwork = hostnetwork
+        self.tolerations = tolerations or []
+        self.configmaps = configmaps or []
+        self.security_context = security_context or {}
diff --git a/airflow/contrib/operators/mlengine_operator.py b/airflow/contrib/operators/mlengine_operator.py
index 9fe966d387ab1..a5ac41911d93d 100644
--- a/airflow/contrib/operators/mlengine_operator.py
+++ b/airflow/contrib/operators/mlengine_operator.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 import re
 
-from apiclient import errors
+from googleapiclient.errors import HttpError
 
 from airflow.contrib.hooks.gcp_mlengine_hook import MLEngineHook
 from airflow.exceptions import AirflowException
@@ -42,7 +42,7 @@ def _normalize_mlengine_job_id(job_id):
 
     # Add a prefix when a job_id starts with a digit or a template
     match = re.search(r'\d|\{{2}', job_id)
-    if match and match.start() is 0:
+    if match and match.start() == 0:
         job = 'z_{}'.format(job_id)
     else:
         job = job_id
@@ -68,17 +68,16 @@ class MLEngineBatchPredictionOperator(BaseOperator):
 
     NOTE: For model origin, users should consider exactly one from the
     three options below:
-    1. Populate 'uri' field only, which should be a GCS location that
-    points to a tensorflow savedModel directory.
-    2. Populate 'model_name' field only, which refers to an existing
-    model, and the default version of the model will be used.
-    3. Populate both 'model_name' and 'version_name' fields, which
-    refers to a specific version of a specific model.
 
-    In options 2 and 3, both model and version name should contain the
-    minimal identifier. For instance, call
+    1. Populate ``uri`` field only, which should be a GCS location that
+       points to a tensorflow savedModel directory.
+    2. Populate ``model_name`` field only, which refers to an existing
+       model, and the default version of the model will be used.
+    3. Populate both ``model_name`` and ``version_name`` fields, which
+       refers to a specific version of a specific model.
 
-    ::
+    In options 2 and 3, both model and version name should contain the
+    minimal identifier. For instance, call::
 
         MLEngineBatchPredictionOperator(
             ...,
@@ -87,52 +86,52 @@ class MLEngineBatchPredictionOperator(BaseOperator):
             ...)
 
     if the desired model version is
-    "projects/my_project/models/my_model/versions/my_version".
+    ``projects/my_project/models/my_model/versions/my_version``.
 
     See https://cloud.google.com/ml-engine/reference/rest/v1/projects.jobs
     for further documentation on the parameters.
 
     :param project_id: The Google Cloud project name where the
         prediction job is submitted. (templated)
-    :type project_id: string
+    :type project_id: str
 
     :param job_id: A unique id for the prediction job on Google Cloud
         ML Engine. (templated)
-    :type job_id: string
+    :type job_id: str
 
     :param data_format: The format of the input data.
         It will default to 'DATA_FORMAT_UNSPECIFIED' if is not provided
         or is not one of ["TEXT", "TF_RECORD", "TF_RECORD_GZIP"].
-    :type data_format: string
+    :type data_format: str
 
     :param input_paths: A list of GCS paths of input data for batch
-        prediction. Accepting wildcard operator *, but only at the end. (templated)
-    :type input_paths: list of string
+        prediction. Accepting wildcard operator ``*``, but only at the end. (templated)
+    :type input_paths: list[str]
 
     :param output_path: The GCS path where the prediction results are
         written to. (templated)
-    :type output_path: string
+    :type output_path: str
 
     :param region: The Google Compute Engine region to run the
         prediction job in. (templated)
-    :type region: string
+    :type region: str
 
     :param model_name: The Google Cloud ML Engine model to use for prediction.
         If version_name is not provided, the default version of this
         model will be used.
         Should not be None if version_name is provided.
         Should be None if uri is provided. (templated)
-    :type model_name: string
+    :type model_name: str
 
     :param version_name: The Google Cloud ML Engine model version to use for
         prediction.
         Should be None if uri is provided. (templated)
-    :type version_name: string
+    :type version_name: str
 
     :param uri: The GCS path of the saved model to use for prediction.
         Should be None if model_name is provided.
         It should be a GCS path pointing to a tensorflow SavedModel. (templated)
-    :type uri: string
+    :type uri: str
 
     :param max_worker_count: The maximum number of workers to be used
         for parallel processing. Defaults to 10 if not specified.
@@ -140,19 +139,19 @@ class MLEngineBatchPredictionOperator(BaseOperator):
 
     :param runtime_version: The Google Cloud ML Engine runtime version to use
         for batch prediction.
-    :type runtime_version: string
+    :type runtime_version: str
 
     :param gcp_conn_id: The connection ID used for connection to Google
         Cloud Platform.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
 
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must
-        have doamin-wide delegation enabled.
-    :type delegate_to: string
+        have domain-wide delegation enabled.
+    :type delegate_to: str
 
-    Raises:
-        ``ValueError``: if a unique model/version origin cannot be determined.
+    :raises: ``ValueError``: if a unique model/version origin cannot be
+        determined.
     """
 
     template_fields = [
@@ -264,12 +263,13 @@ def check_existing_job(existing_job):
         try:
             finished_prediction_job = hook.create_job(
                 self._project_id, prediction_request, check_existing_job)
-        except errors.HttpError:
+        except HttpError:
             raise
 
         if finished_prediction_job['state'] != 'SUCCEEDED':
-            self.log.error('MLEngine batch prediction job failed: {}'.format(
-                str(finished_prediction_job)))
+            self.log.error(
+                'MLEngine batch prediction job failed: %s', str(finished_prediction_job)
+            )
             raise RuntimeError(finished_prediction_job['errorMessage'])
 
         return finished_prediction_job['predictionOutput']
@@ -281,8 +281,7 @@ class MLEngineModelOperator(BaseOperator):
 
     :param project_id: The Google Cloud project name to which MLEngine
         model belongs. (templated)
-    :type project_id: string
-
+    :type project_id: str
     :param model: A dictionary containing the information about the model.
         If the `operation` is `create`, then the `model` parameter should
         contain all the information about this model such as `name`.
@@ -290,19 +289,17 @@ class MLEngineModelOperator(BaseOperator):
         If the `operation` is `get`, the `model` parameter
         should contain the `name` of the model.
     :type model: dict
-
     :param operation: The operation to perform. Available operations are:
 
         * ``create``: Creates a new model as provided by the `model` parameter.
         * ``get``: Gets a particular model where the name is specified in `model`.
-
+    :type operation: str
     :param gcp_conn_id: The connection ID to use when fetching connection info.
-    :type gcp_conn_id: string
-
+    :type gcp_conn_id: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
 
     template_fields = [
@@ -342,17 +339,17 @@ class MLEngineVersionOperator(BaseOperator):
 
     :param project_id: The Google Cloud project name to which MLEngine
         model belongs.
-    :type project_id: string
+    :type project_id: str
 
     :param model_name: The name of the Google Cloud ML Engine model that the version
         belongs to. (templated)
-    :type model_name: string
+    :type model_name: str
 
     :param version_name: A name to use for the version being operated upon.
         If not None and the `version` argument is None or does not have a value for
         the `name` key, then this will be populated in the payload for the
         `name` key. (templated)
-    :type version_name: string
+    :type version_name: str
 
     :param version: A dictionary containing the information about the version.
         If the `operation` is `create`, `version` should contain all the
@@ -381,15 +378,15 @@ class MLEngineVersionOperator(BaseOperator):
             model specified by `model_name`).
             The name of the version should be specified in the `version`
             parameter.
-    :type operation: string
+    :type operation: str
 
     :param gcp_conn_id: The connection ID to use when fetching connection info.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
 
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     """
 
     template_fields = [
@@ -427,7 +424,9 @@ def execute(self, context):
             gcp_conn_id=self._gcp_conn_id, delegate_to=self._delegate_to)
 
         if self._operation == 'create':
-            assert self._version is not None
+            if not self._version:
+                raise ValueError("version attribute of {} could not "
+                                 "be empty".format(self.__class__.__name__))
             return hook.create_version(self._project_id, self._model_name,
                                        self._version)
         elif self._operation == 'set_default':
@@ -448,56 +447,56 @@ class MLEngineTrainingOperator(BaseOperator):
 
     :param project_id: The Google Cloud project name within which MLEngine
         training job should run (templated).
-    :type project_id: string
+    :type project_id: str
 
     :param job_id: A unique templated id for the submitted Google MLEngine
         training job. (templated)
-    :type job_id: string
+    :type job_id: str
 
     :param package_uris: A list of package locations for MLEngine training job,
         which should include the main training program + any additional
         dependencies. (templated)
-    :type package_uris: string
+    :type package_uris: str
 
     :param training_python_module: The Python module name to run within MLEngine
         training job after installing 'package_uris' packages. (templated)
-    :type training_python_module: string
+    :type training_python_module: str
 
     :param training_args: A list of templated command line arguments to pass to
         the MLEngine training program. (templated)
-    :type training_args: string
+    :type training_args: str
 
     :param region: The Google Compute Engine region to run the MLEngine training
         job in (templated).
-    :type region: string
+    :type region: str
 
     :param scale_tier: Resource tier for MLEngine training job. (templated)
-    :type scale_tier: string
+    :type scale_tier: str
 
     :param runtime_version: The Google Cloud ML runtime version to use for
         training. (templated)
-    :type runtime_version: string
+    :type runtime_version: str
 
     :param python_version: The version of Python used in training. (templated)
-    :type python_version: string
+    :type python_version: str
 
     :param job_dir: A Google Cloud Storage path in which to store training
         outputs and other data needed for training. (templated)
-    :type job_dir: string
+    :type job_dir: str
 
     :param gcp_conn_id: The connection ID to use when fetching connection info.
-    :type gcp_conn_id: string
+    :type gcp_conn_id: str
 
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
 
     :param mode: Can be one of 'DRY_RUN'/'CLOUD'. In 'DRY_RUN' mode, no real
         training job will be launched, but the MLEngine training job request
         will be printed out. In 'CLOUD' mode, a real MLEngine training job
         creation request will be issued.
-    :type mode: string
+    :type mode: str
     """
 
     template_fields = [
@@ -586,8 +585,7 @@ def execute(self, context):
 
         if self._mode == 'DRY_RUN':
             self.log.info('In dry_run mode.')
-            self.log.info('MLEngine Training job request is: {}'.format(
-                training_request))
+            self.log.info('MLEngine Training job request is: %s', training_request)
             return
 
         hook = MLEngineHook(
@@ -602,10 +600,9 @@ def check_existing_job(existing_job):
         try:
             finished_training_job = hook.create_job(
                 self._project_id, training_request, check_existing_job)
-        except errors.HttpError:
+        except HttpError:
             raise
 
         if finished_training_job['state'] != 'SUCCEEDED':
-            self.log.error('MLEngine training job failed: {}'.format(
-                str(finished_training_job)))
+            self.log.error('MLEngine training job failed: %s', str(finished_training_job))
             raise RuntimeError(finished_training_job['errorMessage'])
diff --git a/airflow/contrib/operators/mongo_to_s3.py b/airflow/contrib/operators/mongo_to_s3.py
index 43b5d8b6c357a..4a052d4b4af24 100644
--- a/airflow/contrib/operators/mongo_to_s3.py
+++ b/airflow/contrib/operators/mongo_to_s3.py
@@ -1,21 +1,27 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 import json
 
 from airflow.contrib.hooks.mongo_hook import MongoHook
 from airflow.hooks.S3_hook import S3Hook
 from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
 from bson import json_util
 
 
@@ -34,6 +40,7 @@ class MongoToS3Operator(BaseOperator):
     template_fields = ['s3_key', 'mongo_query']
     # pylint: disable=too-many-instance-attributes
 
+    @apply_defaults
     def __init__(self,
                  mongo_conn_id,
                  s3_conn_id,
@@ -96,7 +103,8 @@ def execute(self, context):
 
         return True
 
-    def _stringify(self, iterable, joinable='\n'):
+    @staticmethod
+    def _stringify(iterable, joinable='\n'):
         """
         Takes an iterable (pymongo Cursor or Array) containing dictionaries and
         returns a stringified version using python join
@@ -105,7 +113,8 @@ def _stringify(self, iterable, joinable='\n'):
             [json.dumps(doc, default=json_util.default) for doc in iterable]
         )
 
-    def transform(self, docs):
+    @staticmethod
+    def transform(docs):
         """
         Processes pyMongo cursor and returns an iterable with each element being
                 a JSON serializable dictionary
diff --git a/airflow/contrib/operators/mysql_to_gcs.py b/airflow/contrib/operators/mysql_to_gcs.py
index 4d1bb7b329e6c..c168b75e75e55 100644
--- a/airflow/contrib/operators/mysql_to_gcs.py
+++ b/airflow/contrib/operators/mysql_to_gcs.py
@@ -31,13 +31,56 @@
 from MySQLdb.constants import FIELD_TYPE
 from tempfile import NamedTemporaryFile
 from six import string_types
+import unicodecsv as csv
 
 PY3 = sys.version_info[0] == 3
 
 
 class MySqlToGoogleCloudStorageOperator(BaseOperator):
-    """
-    Copy data from MySQL to Google cloud storage in JSON format.
+    """Copy data from MySQL to Google cloud storage in JSON or CSV format.
+
+    The JSON data files generated are newline-delimited to enable them to be
+    loaded into BigQuery.
+    Reference: https://cloud.google.com/bigquery/docs/
+    loading-data-cloud-storage-json#limitations
+
+    :param sql: The SQL to execute on the MySQL table.
+    :type sql: str
+    :param bucket: The bucket to upload to.
+    :type bucket: str
+    :param filename: The filename to use as the object name when uploading
+        to Google cloud storage. A {} should be specified in the filename
+        to allow the operator to inject file numbers in cases where the
+        file is split due to size.
+    :type filename: str
+    :param schema_filename: If set, the filename to use as the object name
+        when uploading a .json file containing the BigQuery schema fields
+        for the table that was dumped from MySQL.
+    :type schema_filename: str
+    :param approx_max_file_size_bytes: This operator supports the ability
+        to split large table dumps into multiple files (see notes in the
+        filenamed param docs above). Google cloud storage allows for files
+        to be a maximum of 4GB. This param allows developers to specify the
+        file size of the splits.
+    :type approx_max_file_size_bytes: long
+    :param mysql_conn_id: Reference to a specific MySQL hook.
+    :type mysql_conn_id: str
+    :param google_cloud_storage_conn_id: Reference to a specific Google
+        cloud storage hook.
+    :type google_cloud_storage_conn_id: str
+    :param schema: The schema to use, if any. Should be a list of dict or
+        a str. Pass a string if using Jinja template, otherwise, pass a list of
+        dict. Examples could be seen: https://cloud.google.com/bigquery/docs
+        /schemas#specifying_a_json_schema_file
+    :type schema: str or list
+    :param delegate_to: The account to impersonate, if any. For this to
+        work, the service account making the request must have domain-wide
+        delegation enabled.
+    :type delegate_to: str
+    :param export_format: Desired format of files to be exported.
+    :type export_format: str
+    :param field_delimiter: The delimiter to be used for CSV files.
+    :type field_delimiter: str
     """
     template_fields = ('sql', 'bucket', 'filename', 'schema_filename', 'schema')
     template_ext = ('.sql',)
@@ -54,42 +97,10 @@ def __init__(self,
                  google_cloud_storage_conn_id='google_cloud_default',
                  schema=None,
                  delegate_to=None,
+                 export_format='json',
+                 field_delimiter=',',
                  *args,
                  **kwargs):
-        """
-        :param sql: The SQL to execute on the MySQL table.
-        :type sql: string
-        :param bucket: The bucket to upload to.
-        :type bucket: string
-        :param filename: The filename to use as the object name when uploading
-            to Google cloud storage. A {} should be specified in the filename
-            to allow the operator to inject file numbers in cases where the
-            file is split due to size.
-        :type filename: string
-        :param schema_filename: If set, the filename to use as the object name
-            when uploading a .json file containing the BigQuery schema fields
-            for the table that was dumped from MySQL.
-        :type schema_filename: string
-        :param approx_max_file_size_bytes: This operator supports the ability
-            to split large table dumps into multiple files (see notes in the
-            filenamed param docs above). Google cloud storage allows for files
-            to be a maximum of 4GB. This param allows developers to specify the
-            file size of the splits.
-        :type approx_max_file_size_bytes: long
-        :param mysql_conn_id: Reference to a specific MySQL hook.
-        :type mysql_conn_id: string
-        :param google_cloud_storage_conn_id: Reference to a specific Google
-            cloud storage hook.
-        :type google_cloud_storage_conn_id: string
-        :param schema: The schema to use, if any. Should be a list of dict or
-            a str. Pass a string if using Jinja template, otherwise, pass a list of
-            dict. Examples could be seen: https://cloud.google.com/bigquery/docs
-            /schemas#specifying_a_json_schema_file
-        :type schema: str or list
-        :param delegate_to: The account to impersonate, if any. For this to
-            work, the service account making the request must have domain-wide
-            delegation enabled.
-        """
         super(MySqlToGoogleCloudStorageOperator, self).__init__(*args, **kwargs)
         self.sql = sql
         self.bucket = bucket
@@ -100,6 +111,8 @@ def __init__(self,
         self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
         self.schema = schema
         self.delegate_to = delegate_to
+        self.export_format = export_format.lower()
+        self.field_delimiter = field_delimiter
 
     def execute(self, context):
         cursor = self._query_mysql()
@@ -107,17 +120,19 @@ def execute(self, context):
 
         # If a schema is set, create a BQ schema JSON file.
         if self.schema_filename:
-            files_to_upload.update(self._write_local_schema_file(cursor))
+            files_to_upload.append(self._write_local_schema_file(cursor))
 
         # Flush all files before uploading.
-        for file_handle in files_to_upload.values():
-            file_handle.flush()
+        for tmp_file in files_to_upload:
+            tmp_file_handle = tmp_file.get('file_handle')
+            tmp_file_handle.flush()
 
         self._upload_to_gcs(files_to_upload)
 
         # Close all temp file handles.
-        for file_handle in files_to_upload.values():
-            file_handle.close()
+        for tmp_file in files_to_upload:
+            tmp_file_handle = tmp_file.get('file_handle')
+            tmp_file_handle.close()
 
     def _query_mysql(self):
         """
@@ -141,41 +156,73 @@ def _write_local_data_files(self, cursor):
         col_type_dict = self._get_col_type_dict()
         file_no = 0
         tmp_file_handle = NamedTemporaryFile(delete=True)
-        tmp_file_handles = {self.filename.format(file_no): tmp_file_handle}
+        if self.export_format == 'csv':
+            file_mime_type = 'text/csv'
+        else:
+            file_mime_type = 'application/json'
+        files_to_upload = [{
+            'file_name': self.filename.format(file_no),
+            'file_handle': tmp_file_handle,
+            'file_mime_type': file_mime_type
+        }]
+
+        if self.export_format == 'csv':
+            csv_writer = self._configure_csv_file(tmp_file_handle, schema)
 
         for row in cursor:
             # Convert datetime objects to utc seconds, and decimals to floats.
             # Convert binary type object to string encoded with base64.
             row = self._convert_types(schema, col_type_dict, row)
-            row_dict = dict(zip(schema, row))
 
-            # TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.
-            s = json.dumps(row_dict)
-            if PY3:
-                s = s.encode('utf-8')
-            tmp_file_handle.write(s)
+            if self.export_format == 'csv':
+                csv_writer.writerow(row)
+            else:
+                row_dict = dict(zip(schema, row))
 
-            # Append newline to make dumps BigQuery compatible.
-            tmp_file_handle.write(b'\n')
+                # TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.
+                s = json.dumps(row_dict, sort_keys=True)
+                if PY3:
+                    s = s.encode('utf-8')
+                tmp_file_handle.write(s)
+
+                # Append newline to make dumps BigQuery compatible.
+                tmp_file_handle.write(b'\n')
 
             # Stop if the file exceeds the file size limit.
             if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
                 file_no += 1
                 tmp_file_handle = NamedTemporaryFile(delete=True)
-                tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle
+                files_to_upload.append({
+                    'file_name': self.filename.format(file_no),
+                    'file_handle': tmp_file_handle,
+                    'file_mime_type': file_mime_type
+                })
+
+                if self.export_format == 'csv':
+                    csv_writer = self._configure_csv_file(tmp_file_handle, schema)
 
-        return tmp_file_handles
+        return files_to_upload
+
+    def _configure_csv_file(self, file_handle, schema):
+        """Configure a csv writer with the file_handle and write schema
+        as headers for the new file.
+        """
+        csv_writer = csv.writer(file_handle, encoding='utf-8',
+                                delimiter=self.field_delimiter)
+        csv_writer.writerow(schema)
+        return csv_writer
 
     def _write_local_schema_file(self, cursor):
         """
-        Takes a cursor, and writes the BigQuery schema for the results to a
-        local file system.
+        Takes a cursor, and writes the BigQuery schema in .json format for the
+        results to a local file system.
 
         :return: A dictionary where key is a filename to be used as an object
             name in GCS, and values are file handles to local files that
             contains the BigQuery schema fields in .json format.
         """
         schema_str = None
+        schema_file_mime_type = 'application/json'
         tmp_schema_file_handle = NamedTemporaryFile(delete=True)
         if self.schema is not None and isinstance(self.schema, string_types):
             schema_str = self.schema
@@ -199,13 +246,18 @@ def _write_local_schema_file(self, cursor):
                     'type': field_type,
                     'mode': field_mode,
                 })
-            schema_str = json.dumps(schema)
+            schema_str = json.dumps(schema, sort_keys=True)
         if PY3:
             schema_str = schema_str.encode('utf-8')
         tmp_schema_file_handle.write(schema_str)
 
         self.log.info('Using schema for %s: %s', self.schema_filename, schema_str)
-        return {self.schema_filename: tmp_schema_file_handle}
+        schema_file_to_upload = {
+            'file_name': self.schema_filename,
+            'file_handle': tmp_schema_file_handle,
+            'file_mime_type': schema_file_mime_type
+        }
+        return schema_file_to_upload
 
     def _upload_to_gcs(self, files_to_upload):
         """
@@ -215,10 +267,13 @@ def _upload_to_gcs(self, files_to_upload):
         hook = GoogleCloudStorageHook(
             google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
             delegate_to=self.delegate_to)
-        for object, tmp_file_handle in files_to_upload.items():
-            hook.upload(self.bucket, object, tmp_file_handle.name, 'application/json')
+        for tmp_file in files_to_upload:
+            hook.upload(self.bucket, tmp_file.get('file_name'),
+                        tmp_file.get('file_handle').name,
+                        mime_type=tmp_file.get('file_mime_type'))
 
-    def _convert_types(self, schema, col_type_dict, row):
+    @staticmethod
+    def _convert_types(schema, col_type_dict, row):
         """
         Takes a value from MySQLdb, and converts it to a value that's safe for
         JSON/Google cloud storage/BigQuery. Dates are converted to UTC seconds.
diff --git a/airflow/contrib/operators/opsgenie_alert_operator.py b/airflow/contrib/operators/opsgenie_alert_operator.py
new file mode 100644
index 0000000000000..c46d234932d0d
--- /dev/null
+++ b/airflow/contrib/operators/opsgenie_alert_operator.py
@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from airflow.contrib.hooks.opsgenie_alert_hook import OpsgenieAlertHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class OpsgenieAlertOperator(BaseOperator):
+    """
+    This operator allows you to post alerts to Opsgenie.
+    Accepts a connection that has an Opsgenie API key as the connection's password.
+    This operator sets the domain to conn_id.host, and if not set will default
+    to ``https://api.opsgenie.com``.
+
+    Each Opsgenie API key can be pre-configured to a team integration.
+    You can override these defaults in this operator.
+
+    :param opsgenie_conn_id: The name of the Opsgenie connection to use
+    :type opsgenie_conn_id: str
+    :param message: The Message of the Opsgenie alert (templated)
+    :type message: str
+    :param alias: Client-defined identifier of the alert (templated)
+    :type alias: str
+    :param description: Description field of the alert (templated)
+    :type description: str
+    :param responders: Teams, users, escalations and schedules that
+        the alert will be routed to send notifications.
+    :type responders: list[dict]
+    :param visibleTo: Teams and users that the alert will become visible
+        to without sending any notification.
+    :type visibleTo: list[dict]
+    :param actions: Custom actions that will be available for the alert.
+    :type actions: list[str]
+    :param tags: Tags of the alert.
+    :type tags: list[str]
+    :param details: Map of key-value pairs to use as custom properties of the alert.
+    :type details: dict
+    :param entity: Entity field of the alert that is
+        generally used to specify which domain alert is related to. (templated)
+    :type entity: str
+    :param source: Source field of the alert. Default value is
+        IP address of the incoming request.
+    :type source: str
+    :param priority: Priority level of the alert. Default value is P3. (templated)
+    :type priority: str
+    :param user: Display name of the request owner.
+    :type user: str
+    :param note: Additional note that will be added while creating the alert. (templated)
+    :type note: str
+    """
+    template_fields = ('message', 'alias', 'description', 'entity', 'priority', 'note')
+
+    @apply_defaults
+    def __init__(self,
+                 message,
+                 opsgenie_conn_id='opsgenie_default',
+                 alias=None,
+                 description=None,
+                 responders=None,
+                 visibleTo=None,
+                 actions=None,
+                 tags=None,
+                 details=None,
+                 entity=None,
+                 source=None,
+                 priority=None,
+                 user=None,
+                 note=None,
+                 *args,
+                 **kwargs
+                 ):
+        super(OpsgenieAlertOperator, self).__init__(*args, **kwargs)
+
+        self.message = message
+        self.opsgenie_conn_id = opsgenie_conn_id
+        self.alias = alias
+        self.description = description
+        self.responders = responders
+        self.visibleTo = visibleTo
+        self.actions = actions
+        self.tags = tags
+        self.details = details
+        self.entity = entity
+        self.source = source
+        self.priority = priority
+        self.user = user
+        self.note = note
+        self.hook = None
+
+    def _build_opsgenie_payload(self):
+        """
+        Construct the Opsgenie JSON payload. All relevant parameters are combined here
+        to a valid Opsgenie JSON payload.
+
+        :return: Opsgenie payload (dict) to send
+        """
+        payload = {}
+
+        for key in [
+            "message", "alias", "description", "responders",
+            "visibleTo", "actions", "tags", "details", "entity",
+            "source", "priority", "user", "note"
+        ]:
+            val = getattr(self, key)
+            if val:
+                payload[key] = val
+        return payload
+
+    def execute(self, context):
+        """
+        Call the OpsgenieAlertHook to post message
+        """
+        self.hook = OpsgenieAlertHook(self.opsgenie_conn_id)
+        self.hook.execute(self._build_opsgenie_payload())
diff --git a/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py b/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py
new file mode 100644
index 0000000000000..967f84828d9fd
--- /dev/null
+++ b/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.hooks.oracle_hook import OracleHook
+from airflow.contrib.hooks.azure_data_lake_hook import AzureDataLakeHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.utils.file import TemporaryDirectory
+
+import unicodecsv as csv
+import os
+
+
+class OracleToAzureDataLakeTransfer(BaseOperator):
+    """
+    Moves data from Oracle to Azure Data Lake. The operator runs the query against
+    Oracle and stores the file locally before loading it into Azure Data Lake.
+
+
+    :param filename: file name to be used by the csv file.
+    :type filename: str
+    :param azure_data_lake_conn_id: destination azure data lake connection.
+    :type azure_data_lake_conn_id: str
+    :param azure_data_lake_path: destination path in azure data lake to put the file.
+    :type azure_data_lake_path: str
+    :param oracle_conn_id: source Oracle connection.
+    :type oracle_conn_id: str
+    :param sql: SQL query to execute against the Oracle database. (templated)
+    :type sql: str
+    :param sql_params: Parameters to use in sql query. (templated)
+    :type sql_params: str
+    :param delimiter: field delimiter in the file.
+    :type delimiter: str
+    :param encoding: encoding type for the file.
+    :type encoding: str
+    :param quotechar: Character to use in quoting.
+    :type quotechar: str
+    :param quoting: Quoting strategy. See unicodecsv quoting for more information.
+    :type quoting: str
+    """
+
+    template_fields = ('filename', 'sql', 'sql_params')
+    ui_color = '#e08c8c'
+
+    @apply_defaults
+    def __init__(
+            self,
+            filename,
+            azure_data_lake_conn_id,
+            azure_data_lake_path,
+            oracle_conn_id,
+            sql,
+            sql_params=None,
+            delimiter=",",
+            encoding="utf-8",
+            quotechar='"',
+            quoting=csv.QUOTE_MINIMAL,
+            *args, **kwargs):
+        super(OracleToAzureDataLakeTransfer, self).__init__(*args, **kwargs)
+        if sql_params is None:
+            sql_params = {}
+        self.filename = filename
+        self.oracle_conn_id = oracle_conn_id
+        self.sql = sql
+        self.sql_params = sql_params
+        self.azure_data_lake_conn_id = azure_data_lake_conn_id
+        self.azure_data_lake_path = azure_data_lake_path
+        self.delimiter = delimiter
+        self.encoding = encoding
+        self.quotechar = quotechar
+        self.quoting = quoting
+
+    def _write_temp_file(self, cursor, path_to_save):
+        with open(path_to_save, 'wb') as csvfile:
+            csv_writer = csv.writer(csvfile, delimiter=self.delimiter,
+                                    encoding=self.encoding, quotechar=self.quotechar,
+                                    quoting=self.quoting)
+            csv_writer.writerow(map(lambda field: field[0], cursor.description))
+            csv_writer.writerows(cursor)
+            csvfile.flush()
+
+    def execute(self, context):
+        oracle_hook = OracleHook(oracle_conn_id=self.oracle_conn_id)
+        azure_data_lake_hook = AzureDataLakeHook(
+            azure_data_lake_conn_id=self.azure_data_lake_conn_id)
+
+        self.log.info("Dumping Oracle query results to local file")
+        conn = oracle_hook.get_conn()
+        cursor = conn.cursor()
+        cursor.execute(self.sql, self.sql_params)
+
+        with TemporaryDirectory(prefix='airflow_oracle_to_azure_op_') as temp:
+            self._write_temp_file(cursor, os.path.join(temp, self.filename))
+            self.log.info("Uploading local file to Azure Data Lake")
+            azure_data_lake_hook.upload_file(os.path.join(temp, self.filename),
+                                             os.path.join(self.azure_data_lake_path,
+                                                          self.filename))
+        cursor.close()
+        conn.close()
diff --git a/airflow/contrib/operators/oracle_to_oracle_transfer.py b/airflow/contrib/operators/oracle_to_oracle_transfer.py
new file mode 100644
index 0000000000000..58613c4f33489
--- /dev/null
+++ b/airflow/contrib/operators/oracle_to_oracle_transfer.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.hooks.oracle_hook import OracleHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class OracleToOracleTransfer(BaseOperator):
+    """
+    Moves data from Oracle to Oracle.
+
+
+    :param oracle_destination_conn_id: destination Oracle connection.
+    :type oracle_destination_conn_id: str
+    :param destination_table: destination table to insert rows.
+    :type destination_table: str
+    :param oracle_source_conn_id: source Oracle connection.
+    :type oracle_source_conn_id: str
+    :param source_sql: SQL query to execute against the source Oracle
+        database. (templated)
+    :type source_sql: str
+    :param source_sql_params: Parameters to use in sql query. (templated)
+    :type source_sql_params: dict
+    :param rows_chunk: number of rows per chunk to commit.
+    :type rows_chunk: int
+    """
+
+    template_fields = ('source_sql', 'source_sql_params')
+    ui_color = '#e08c8c'
+
+    @apply_defaults
+    def __init__(
+            self,
+            oracle_destination_conn_id,
+            destination_table,
+            oracle_source_conn_id,
+            source_sql,
+            source_sql_params=None,
+            rows_chunk=5000,
+            *args, **kwargs):
+        super(OracleToOracleTransfer, self).__init__(*args, **kwargs)
+        if source_sql_params is None:
+            source_sql_params = {}
+        self.oracle_destination_conn_id = oracle_destination_conn_id
+        self.destination_table = destination_table
+        self.oracle_source_conn_id = oracle_source_conn_id
+        self.source_sql = source_sql
+        self.source_sql_params = source_sql_params
+        self.rows_chunk = rows_chunk
+
+    def _execute(self, src_hook, dest_hook, context):
+        with src_hook.get_conn() as src_conn:
+            cursor = src_conn.cursor()
+            self.log.info("Querying data from source: %s", self.oracle_source_conn_id)
+            cursor.execute(self.source_sql, self.source_sql_params)
+            target_fields = list(map(lambda field: field[0], cursor.description))
+
+            rows_total = 0
+            rows = cursor.fetchmany(self.rows_chunk)
+            while len(rows) > 0:
+                rows_total = rows_total + len(rows)
+                dest_hook.bulk_insert_rows(self.destination_table, rows,
+                                           target_fields=target_fields,
+                                           commit_every=self.rows_chunk)
+                rows = cursor.fetchmany(self.rows_chunk)
+                self.log.info("Total inserted: %s rows", rows_total)
+
+            self.log.info("Finished data transfer.")
+            cursor.close()
+
+    def execute(self, context):
+        src_hook = OracleHook(oracle_conn_id=self.oracle_source_conn_id)
+        dest_hook = OracleHook(oracle_conn_id=self.oracle_destination_conn_id)
+        self._execute(src_hook, dest_hook, context)
diff --git a/airflow/contrib/operators/postgres_to_gcs_operator.py b/airflow/contrib/operators/postgres_to_gcs_operator.py
index 88b4d00e39790..78da78ee2f20d 100644
--- a/airflow/contrib/operators/postgres_to_gcs_operator.py
+++ b/airflow/contrib/operators/postgres_to_gcs_operator.py
@@ -56,18 +56,18 @@ def __init__(self,
                  **kwargs):
         """
         :param sql: The SQL to execute on the Postgres table.
-        :type sql: string
+        :type sql: str
         :param bucket: The bucket to upload to.
-        :type bucket: string
+        :type bucket: str
         :param filename: The filename to use as the object name when uploading
             to Google Cloud Storage. A {} should be specified in the filename
             to allow the operator to inject file numbers in cases where the
             file is split due to size.
-        :type filename: string
+        :type filename: str
         :param schema_filename: If set, the filename to use as the object name
             when uploading a .json file containing the BigQuery schema fields
             for the table that was dumped from Postgres.
-        :type schema_filename: string
+        :type schema_filename: str
         :param approx_max_file_size_bytes: This operator supports the ability
             to split large table dumps into multiple files (see notes in the
             filenamed param docs above). Google Cloud Storage allows for files
@@ -75,10 +75,10 @@ def __init__(self,
             file size of the splits.
         :type approx_max_file_size_bytes: long
         :param postgres_conn_id: Reference to a specific Postgres hook.
-        :type postgres_conn_id: string
+        :type postgres_conn_id: str
         :param google_cloud_storage_conn_id: Reference to a specific Google
             cloud storage hook.
-        :type google_cloud_storage_conn_id: string
+        :type google_cloud_storage_conn_id: str
         :param delegate_to: The account to impersonate, if any. For this to
             work, the service account making the request must have domain-wide
             delegation enabled.
@@ -133,28 +133,38 @@ def _write_local_data_files(self, cursor):
             contain the data for the GCS objects.
         """
         schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description))
-        file_no = 0
-        tmp_file_handle = NamedTemporaryFile(delete=True)
-        tmp_file_handles = {self.filename.format(file_no): tmp_file_handle}
-
-        for row in cursor:
-            # Convert datetime objects to utc seconds, and decimals to floats
-            row = map(self.convert_types, row)
-            row_dict = dict(zip(schema, row))
-
-            s = json.dumps(row_dict, sort_keys=True)
-            if PY3:
-                s = s.encode('utf-8')
-            tmp_file_handle.write(s)
-
-            # Append newline to make dumps BigQuery compatible.
-            tmp_file_handle.write(b'\n')
-
-            # Stop if the file exceeds the file size limit.
-            if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
-                file_no += 1
-                tmp_file_handle = NamedTemporaryFile(delete=True)
-                tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle
+        tmp_file_handles = {}
+        row_no = 0
+
+        def _create_new_file():
+            handle = NamedTemporaryFile(delete=True)
+            filename = self.filename.format(len(tmp_file_handles))
+            tmp_file_handles[filename] = handle
+            return handle
+
+        # Don't create a file if there is nothing to write
+        if cursor.rowcount > 0:
+            tmp_file_handle = _create_new_file()
+
+            for row in cursor:
+                # Convert datetime objects to utc seconds, and decimals to floats
+                row = map(self.convert_types, row)
+                row_dict = dict(zip(schema, row))
+
+                s = json.dumps(row_dict, sort_keys=True)
+                if PY3:
+                    s = s.encode('utf-8')
+                tmp_file_handle.write(s)
+
+                # Append newline to make dumps BigQuery compatible.
+                tmp_file_handle.write(b'\n')
+
+                # Stop if the file exceeds the file size limit.
+                if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
+                    tmp_file_handle = _create_new_file()
+                row_no += 1
+
+        self.log.info('Received %s rows over %s files', row_no, len(tmp_file_handles))
 
         return tmp_file_handles
 
diff --git a/airflow/contrib/operators/pubsub_operator.py b/airflow/contrib/operators/pubsub_operator.py
index 2d55b19f8595d..7c4d2fde5a560 100644
--- a/airflow/contrib/operators/pubsub_operator.py
+++ b/airflow/contrib/operators/pubsub_operator.py
@@ -67,19 +67,19 @@ def __init__(
             **kwargs):
         """
         :param project: the GCP project ID where the topic will be created
-        :type project: string
+        :type project: str
         :param topic: the topic to create. Do not include the
             full topic path. In other words, instead of
             ``projects/{project}/topics/{topic}``, provide only
             ``{topic}``. (templated)
-        :type topic: string
+        :type topic: str
         :param gcp_conn_id: The connection ID to use connecting to
             Google Cloud Platform.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request
             must have domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(PubSubTopicCreateOperator, self).__init__(*args, **kwargs)
 
@@ -163,28 +163,28 @@ def __init__(
             **kwargs):
         """
         :param topic_project: the GCP project ID where the topic exists
-        :type topic_project: string
+        :type topic_project: str
         :param topic: the topic to create. Do not include the
             full topic path. In other words, instead of
             ``projects/{project}/topics/{topic}``, provide only
             ``{topic}``. (templated)
-        :type topic: string
+        :type topic: str
         :param subscription: the Pub/Sub subscription name. If empty, a random
             name will be generated using the uuid module
-        :type subscription: string
+        :type subscription: str
         :param subscription_project: the GCP project ID where the subscription
             will be created. If empty, ``topic_project`` will be used.
-        :type subscription_project: string
+        :type subscription_project: str
         :param ack_deadline_secs: Number of seconds that a subscriber has to
             acknowledge each message pulled from the subscription
         :type ack_deadline_secs: int
         :param gcp_conn_id: The connection ID to use connecting to
             Google Cloud Platform.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request
             must have domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(PubSubSubscriptionCreateOperator, self).__init__(*args, **kwargs)
 
@@ -248,22 +248,22 @@ def __init__(
             **kwargs):
         """
         :param project: the GCP project ID in which to work (templated)
-        :type project: string
+        :type project: str
         :param topic: the topic to delete. Do not include the
             full topic path. In other words, instead of
             ``projects/{project}/topics/{topic}``, provide only
             ``{topic}``. (templated)
-        :type topic: string
+        :type topic: str
         :param fail_if_not_exists: If True and the topic does not exist, fail
             the task
         :type fail_if_not_exists: bool
         :param gcp_conn_id: The connection ID to use connecting to
             Google Cloud Platform.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request
             must have domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(PubSubTopicDeleteOperator, self).__init__(*args, **kwargs)
 
@@ -324,22 +324,22 @@ def __init__(
             **kwargs):
         """
         :param project: the GCP project ID in which to work (templated)
-        :type project: string
+        :type project: str
         :param subscription: the subscription to delete. Do not include the
             full subscription path. In other words, instead of
             ``projects/{project}/subscription/{subscription}``, provide only
             ``{subscription}``. (templated)
-        :type subscription: string
+        :type subscription: str
         :param fail_if_not_exists: If True and the subscription does not exist,
             fail the task
         :type fail_if_not_exists: bool
         :param gcp_conn_id: The connection ID to use connecting to
             Google Cloud Platform.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request
             must have domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(PubSubSubscriptionDeleteOperator, self).__init__(*args, **kwargs)
 
@@ -378,7 +378,7 @@ class PubSubPublishOperator(BaseOperator):
             create_topic=True,
             dag=dag)
 
-     ``project`` , ``topic``, and ``messages`` are templated so you can use
+    ``project`` , ``topic``, and ``messages`` are templated so you can use
     variables in them.
     """
     template_fields = ['project', 'topic', 'messages']
@@ -396,12 +396,12 @@ def __init__(
             **kwargs):
         """
         :param project: the GCP project ID in which to work (templated)
-        :type project: string
+        :type project: str
         :param topic: the topic to which to publish. Do not include the
             full topic path. In other words, instead of
             ``projects/{project}/topics/{topic}``, provide only
             ``{topic}``. (templated)
-        :type topic: string
+        :type topic: str
         :param messages: a list of messages to be published to the
             topic. Each message is a dict with one or more of the
             following keys-value mappings:
@@ -413,11 +413,11 @@ def __init__(
         :type messages: list
         :param gcp_conn_id: The connection ID to use connecting to
             Google Cloud Platform.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request
             must have domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(PubSubPublishOperator, self).__init__(*args, **kwargs)
 
diff --git a/airflow/contrib/operators/qubole_check_operator.py b/airflow/contrib/operators/qubole_check_operator.py
index 0e8d75e16729e..73e3a7b15f661 100644
--- a/airflow/contrib/operators/qubole_check_operator.py
+++ b/airflow/contrib/operators/qubole_check_operator.py
@@ -28,7 +28,7 @@ class QuboleCheckOperator(CheckOperator, QuboleOperator):
     """
     Performs checks against Qubole Commands. ``QuboleCheckOperator`` expects
     a command that will be executed on QDS.
-    By default, each value on first row of the result of this Qubole Commmand
+    By default, each value on first row of the result of this Qubole Command
     is evaluated using python ``bool`` casting. If any of the
     values return ``False``, the check is failed and errors out.
 
@@ -69,7 +69,8 @@ class QuboleCheckOperator(CheckOperator, QuboleOperator):
             which the checks have to be performed.
 
     .. note:: All fields in common with template fields of
-            QuboleOperator and CheckOperator are template-supported.
+        QuboleOperator and CheckOperator are template-supported.
+
     """
 
     template_fields = QuboleOperator.template_fields + CheckOperator.template_fields
@@ -128,13 +129,13 @@ class QuboleValueCheckOperator(ValueCheckOperator, QuboleOperator):
     :type qubole_conn_id: str
 
     :param pass_value: Expected value of the query results.
-    :type pass_value: str/int/float
+    :type pass_value: str or int or float
 
     :param tolerance: Defines the permissible pass_value range, for example if
         tolerance is 2, the Qubole command output can be anything between
         -2*pass_value and 2*pass_value, without the operator erring out.
 
-    :type tolerance: int/float
+    :type tolerance: int or float
 
 
     kwargs:
@@ -215,11 +216,11 @@ def get_sql_from_qbol_cmd(params):
 def handle_airflow_exception(airflow_exception, hook):
     cmd = hook.cmd
     if cmd is not None:
-        if cmd.is_success:
+        if cmd.is_success(cmd.status):
             qubole_command_results = hook.get_query_results()
             qubole_command_id = cmd.id
             exception_message = '\nQubole Command Id: {qubole_command_id}' \
                                 '\nQubole Command Results:' \
                                 '\n{qubole_command_results}'.format(**locals())
             raise AirflowException(str(airflow_exception) + exception_message)
-    raise AirflowException(airflow_exception.message)
+    raise AirflowException(str(airflow_exception))
diff --git a/airflow/contrib/operators/qubole_operator.py b/airflow/contrib/operators/qubole_operator.py
index 82ee293b934bc..7da3be7d05792 100755
--- a/airflow/contrib/operators/qubole_operator.py
+++ b/airflow/contrib/operators/qubole_operator.py
@@ -17,6 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from typing import Iterable
+
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
 from airflow.contrib.hooks.qubole_hook import QuboleHook
@@ -57,7 +59,7 @@ class QuboleOperator(BaseOperator):
                 copied into the working directory where the qubole command is being
                 executed.
             :archives: list of archives in s3 bucket as archive1,archive2 format. These
-                will be unarchived intothe working directory where the qubole command is
+                will be unarchived into the working directory where the qubole command is
                 being executed
             :parameters: any extra args which need to be passed to script (only when
                 script_location is supplied)
@@ -103,15 +105,19 @@ class QuboleOperator(BaseOperator):
             :boundary_query: Query to be used get range of row IDs to be extracted
             :split_column: Column used as row ID to split data into ranges (mode 2)
 
-    .. note:: Following fields are template-supported : ``query``, ``script_location``,
+    .. note:
+
+        Following fields are template-supported : ``query``, ``script_location``,
         ``sub_command``, ``script``, ``files``, ``archives``, ``program``, ``cmdline``,
         ``sql``, ``where_clause``, ``extract_query``, ``boundary_query``, ``macros``,
         ``tags``, ``name``, ``parameters``, ``dbtap_id``, ``hive_table``, ``db_table``,
         ``split_column``, ``note_id``, ``db_update_keys``, ``export_dir``,
         ``partition_spec``, ``qubole_conn_id``, ``arguments``, ``user_program_arguments``.
-         You can also use ``.txt`` files for template driven use cases.
+        You can also use ``.txt`` files for template driven use cases.
+
+    .. note:
 
-    .. note:: In QuboleOperator there is a default handler for task failures and retries,
+        In QuboleOperator there is a default handler for task failures and retries,
         which generally kills the command running at QDS for the corresponding task
         instance. You can override this behavior by providing your own failure and retry
         handler in task definition.
@@ -122,9 +128,9 @@ class QuboleOperator(BaseOperator):
                        'extract_query', 'boundary_query', 'macros', 'name', 'parameters',
                        'dbtap_id', 'hive_table', 'db_table', 'split_column', 'note_id',
                        'db_update_keys', 'export_dir', 'partition_spec', 'qubole_conn_id',
-                       'arguments', 'user_program_arguments', 'cluster_label')
+                       'arguments', 'user_program_arguments', 'cluster_label')  # type: Iterable[str]
 
-    template_ext = ('.txt',)
+    template_ext = ('.txt',)  # type: Iterable[str]
     ui_color = '#3064A1'
     ui_fgcolor = '#fff'
 
diff --git a/airflow/contrib/operators/s3_copy_object_operator.py b/airflow/contrib/operators/s3_copy_object_operator.py
new file mode 100644
index 0000000000000..43651707f8925
--- /dev/null
+++ b/airflow/contrib/operators/s3_copy_object_operator.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.hooks.S3_hook import S3Hook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class S3CopyObjectOperator(BaseOperator):
+    """
+    Creates a copy of an object that is already stored in S3.
+
+    Note: the S3 connection used here needs to have access to both
+    source and destination bucket/key.
+
+    :param source_bucket_key: The key of the source object. (templated)
+
+        It can be either full s3:// style url or relative path from root level.
+
+        When it's specified as a full s3:// url, please omit source_bucket_name.
+    :type source_bucket_key: str
+    :param dest_bucket_key: The key of the object to copy to. (templated)
+
+        The convention to specify `dest_bucket_key` is the same as `source_bucket_key`.
+    :type dest_bucket_key: str
+    :param source_bucket_name: Name of the S3 bucket where the source object is in. (templated)
+
+        It should be omitted when `source_bucket_key` is provided as a full s3:// url.
+    :type source_bucket_name: str
+    :param dest_bucket_name: Name of the S3 bucket to where the object is copied. (templated)
+
+        It should be omitted when `dest_bucket_key` is provided as a full s3:// url.
+    :type dest_bucket_name: str
+    :param source_version_id: Version ID of the source object (OPTIONAL)
+    :type source_version_id: str
+    :param aws_conn_id: Connection id of the S3 connection to use
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+
+        You can provide the following values:
+
+        - False: do not validate SSL certificates. SSL will still be used,
+                 but SSL certificates will not be
+                 verified.
+        - path/to/cert/bundle.pem: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
+    """
+
+    template_fields = ('source_bucket_key', 'dest_bucket_key',
+                       'source_bucket_name', 'dest_bucket_name')
+
+    @apply_defaults
+    def __init__(
+            self,
+            source_bucket_key,
+            dest_bucket_key,
+            source_bucket_name=None,
+            dest_bucket_name=None,
+            source_version_id=None,
+            aws_conn_id='aws_default',
+            verify=None,
+            *args, **kwargs):
+        super(S3CopyObjectOperator, self).__init__(*args, **kwargs)
+
+        self.source_bucket_key = source_bucket_key
+        self.dest_bucket_key = dest_bucket_key
+        self.source_bucket_name = source_bucket_name
+        self.dest_bucket_name = dest_bucket_name
+        self.source_version_id = source_version_id
+        self.aws_conn_id = aws_conn_id
+        self.verify = verify
+
+    def execute(self, context):
+        s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
+        s3_hook.copy_object(self.source_bucket_key, self.dest_bucket_key,
+                            self.source_bucket_name, self.dest_bucket_name,
+                            self.source_version_id)
diff --git a/airflow/contrib/operators/s3_delete_objects_operator.py b/airflow/contrib/operators/s3_delete_objects_operator.py
new file mode 100644
index 0000000000000..926667b497195
--- /dev/null
+++ b/airflow/contrib/operators/s3_delete_objects_operator.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.exceptions import AirflowException
+from airflow.hooks.S3_hook import S3Hook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class S3DeleteObjectsOperator(BaseOperator):
+    """
+    To enable users to delete single object or multiple objects from
+    a bucket using a single HTTP request.
+
+    Users may specify up to 1000 keys to delete.
+
+    :param bucket: Name of the bucket in which you are going to delete object(s). (templated)
+    :type bucket: str
+    :param keys: The key(s) to delete from S3 bucket. (templated)
+
+        When ``keys`` is a string, it's supposed to be the key name of
+        the single object to delete.
+
+        When ``keys`` is a list, it's supposed to be the list of the
+        keys to delete.
+
+        You may specify up to 1000 keys.
+    :type keys: str or list
+    :param aws_conn_id: Connection id of the S3 connection to use
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used,
+                 but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
+    """
+
+    template_fields = ('keys', 'bucket')
+
+    @apply_defaults
+    def __init__(
+            self,
+            bucket,
+            keys,
+            aws_conn_id='aws_default',
+            verify=None,
+            *args, **kwargs):
+        super(S3DeleteObjectsOperator, self).__init__(*args, **kwargs)
+        self.bucket = bucket
+        self.keys = keys
+        self.aws_conn_id = aws_conn_id
+        self.verify = verify
+
+    def execute(self, context):
+        s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
+
+        response = s3_hook.delete_objects(bucket=self.bucket, keys=self.keys)
+
+        deleted_keys = [x['Key'] for x in response.get("Deleted", [])]
+        self.log.info("Deleted: %s", deleted_keys)
+
+        if "Errors" in response:
+            errors_keys = [x['Key'] for x in response.get("Errors", [])]
+            raise AirflowException("Errors when deleting: {}".format(errors_keys))
diff --git a/airflow/contrib/operators/s3_list_operator.py b/airflow/contrib/operators/s3_list_operator.py
index b85691b005fb9..9b13f0744c3de 100644
--- a/airflow/contrib/operators/s3_list_operator.py
+++ b/airflow/contrib/operators/s3_list_operator.py
@@ -17,6 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from typing import Iterable
+
 from airflow.hooks.S3_hook import S3Hook
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
@@ -30,14 +32,26 @@ class S3ListOperator(BaseOperator):
     used by `xcom` in the downstream task.
 
     :param bucket: The S3 bucket where to find the objects. (templated)
-    :type bucket: string
+    :type bucket: str
     :param prefix: Prefix string to filters the objects whose name begin with
         such prefix. (templated)
-    :type prefix: string
+    :type prefix: str
     :param delimiter: the delimiter marks key hierarchy. (templated)
-    :type delimiter: string
+    :type delimiter: str
     :param aws_conn_id: The connection ID to use when connecting to S3 storage.
-    :type aws_conn_id: string
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
+
 
     **Example**:
         The following operator would list all the files
@@ -52,7 +66,7 @@ class S3ListOperator(BaseOperator):
                 aws_conn_id='aws_customers_conn'
             )
     """
-    template_fields = ('bucket', 'prefix', 'delimiter')
+    template_fields = ('bucket', 'prefix', 'delimiter')  # type: Iterable[str]
     ui_color = '#ffd700'
 
     @apply_defaults
@@ -61,6 +75,7 @@ def __init__(self,
                  prefix='',
                  delimiter='',
                  aws_conn_id='aws_default',
+                 verify=None,
                  *args,
                  **kwargs):
         super(S3ListOperator, self).__init__(*args, **kwargs)
@@ -68,13 +83,15 @@ def __init__(self,
         self.prefix = prefix
         self.delimiter = delimiter
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
 
     def execute(self, context):
-        hook = S3Hook(aws_conn_id=self.aws_conn_id)
+        hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
 
         self.log.info(
-            'Getting the list of files from bucket: {0} in prefix: {1} (Delimiter {2})'.
-            format(self.bucket, self.prefix, self.delimiter))
+            'Getting the list of files from bucket: %s in prefix: %s (Delimiter {%s)',
+            self.bucket, self.prefix, self.delimiter
+        )
 
         return hook.list_keys(
             bucket_name=self.bucket,
diff --git a/airflow/contrib/operators/s3_to_gcs_operator.py b/airflow/contrib/operators/s3_to_gcs_operator.py
index 2898af1071773..ad456a5c9bae0 100644
--- a/airflow/contrib/operators/s3_to_gcs_operator.py
+++ b/airflow/contrib/operators/s3_to_gcs_operator.py
@@ -33,31 +33,44 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator):
     destination path.
 
     :param bucket: The S3 bucket where to find the objects. (templated)
-    :type bucket: string
+    :type bucket: str
     :param prefix: Prefix string which filters objects whose name begin with
         such prefix. (templated)
-    :type prefix: string
+    :type prefix: str
     :param delimiter: the delimiter marks key hierarchy. (templated)
-    :type delimiter: string
+    :type delimiter: str
     :param aws_conn_id: The source S3 connection
-    :type aws_conn_id: string
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
     :param dest_gcs_conn_id: The destination connection ID to use
         when connecting to Google Cloud Storage.
-    :type dest_gcs_conn_id: string
+    :type dest_gcs_conn_id: str
     :param dest_gcs: The destination Google Cloud Storage bucket and prefix
         where you want to store the files. (templated)
-    :type dest_gcs: string
+    :type dest_gcs: str
     :param delegate_to: The account to impersonate, if any.
         For this to work, the service account making the request must have
         domain-wide delegation enabled.
-    :type delegate_to: string
+    :type delegate_to: str
     :param replace: Whether you want to replace existing destination files
         or not.
     :type replace: bool
 
 
     **Example**:
+
     .. code-block:: python
+
        s3_to_gcs_op = S3ToGoogleCloudStorageOperator(
             task_id='s3_to_gcs_example',
             bucket='my-s3-bucket',
@@ -80,6 +93,7 @@ def __init__(self,
                  prefix='',
                  delimiter='',
                  aws_conn_id='aws_default',
+                 verify=None,
                  dest_gcs_conn_id=None,
                  dest_gcs=None,
                  delegate_to=None,
@@ -98,6 +112,7 @@ def __init__(self,
         self.dest_gcs = dest_gcs
         self.delegate_to = delegate_to
         self.replace = replace
+        self.verify = verify
 
         if dest_gcs and not self._gcs_object_is_directory(self.dest_gcs):
             self.log.info(
@@ -137,16 +152,17 @@ def execute(self, context):
                     else:
                         existing_files.append(f)
 
-            files = set(files) - set(existing_files)
+            files = list(set(files) - set(existing_files))
             if len(files) > 0:
-                self.log.info('{0} files are going to be synced: {1}.'.format(
-                    len(files), files))
+                self.log.info(
+                    '%s files are going to be synced: %s.', len(files), files
+                )
             else:
                 self.log.info(
                     'There are no new files to sync. Have a nice day!')
 
         if files:
-            hook = S3Hook(aws_conn_id=self.aws_conn_id)
+            hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
 
             for file in files:
                 # GCS hook builds its own in-memory file so we have to create
@@ -184,7 +200,8 @@ def execute(self, context):
 
     # Following functionality may be better suited in
     # airflow/contrib/hooks/gcs_hook.py
-    def _gcs_object_is_directory(self, object):
+    @staticmethod
+    def _gcs_object_is_directory(object):
         bucket, blob = _parse_gcs_url(object)
 
         return len(blob) == 0 or blob.endswith('/')
diff --git a/airflow/contrib/operators/s3_to_gcs_transfer_operator.py b/airflow/contrib/operators/s3_to_gcs_transfer_operator.py
new file mode 100644
index 0000000000000..aaae9bff2111e
--- /dev/null
+++ b/airflow/contrib/operators/s3_to_gcs_transfer_operator.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import warnings
+
+from airflow.contrib.operators.gcp_transfer_operator import S3ToGoogleCloudStorageTransferOperator  # noqa
+
+warnings.warn(
+    "This module is deprecated. Please use `airflow.contrib.operators.gcp_transfer_operator`",
+    DeprecationWarning,
+)
diff --git a/airflow/contrib/operators/s3_to_sftp_operator.py b/airflow/contrib/operators/s3_to_sftp_operator.py
new file mode 100644
index 0000000000000..43ef269032db8
--- /dev/null
+++ b/airflow/contrib/operators/s3_to_sftp_operator.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.models import BaseOperator
+from airflow.hooks.S3_hook import S3Hook
+from airflow.contrib.hooks.ssh_hook import SSHHook
+from tempfile import NamedTemporaryFile
+from urllib.parse import urlparse
+from airflow.utils.decorators import apply_defaults
+
+
+class S3ToSFTPOperator(BaseOperator):
+    """
+    This operator enables the transferring of files from S3 to a SFTP server.
+
+    :param sftp_conn_id: The sftp connection id. The name or identifier for
+        establishing a connection to the SFTP server.
+    :type sftp_conn_id: string
+    :param sftp_path: The sftp remote path. This is the specified file path for
+        uploading file to the SFTP server.
+    :type sftp_path: string
+    :param s3_conn_id: The s3 connection id. The name or identifier for
+        establishing a connection to S3
+    :type s3_conn_id: string
+    :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from
+        where the file is downloaded.
+    :type s3_bucket: string
+    :param s3_key: The targeted s3 key. This is the specified file path for
+        downloading the file from S3.
+    :type s3_key: string
+    """
+
+    template_fields = ('s3_key', 'sftp_path')
+
+    @apply_defaults
+    def __init__(self,
+                 s3_bucket,
+                 s3_key,
+                 sftp_path,
+                 sftp_conn_id='ssh_default',
+                 s3_conn_id='aws_default',
+                 *args,
+                 **kwargs):
+        super(S3ToSFTPOperator, self).__init__(*args, **kwargs)
+        self.sftp_conn_id = sftp_conn_id
+        self.sftp_path = sftp_path
+        self.s3_bucket = s3_bucket
+        self.s3_key = s3_key
+        self.s3_conn_id = s3_conn_id
+
+    @staticmethod
+    def get_s3_key(s3_key):
+        """This parses the correct format for S3 keys
+            regardless of how the S3 url is passed."""
+
+        parsed_s3_key = urlparse(s3_key)
+        return parsed_s3_key.path.lstrip('/')
+
+    def execute(self, context):
+        self.s3_key = self.get_s3_key(self.s3_key)
+        ssh_hook = SSHHook(ssh_conn_id=self.sftp_conn_id)
+        s3_hook = S3Hook(self.s3_conn_id)
+
+        s3_client = s3_hook.get_conn()
+        sftp_client = ssh_hook.get_conn().open_sftp()
+
+        with NamedTemporaryFile("w") as f:
+            s3_client.download_file(self.s3_bucket, self.s3_key, f.name)
+            sftp_client.put(f.name, self.sftp_path)
diff --git a/airflow/contrib/operators/sagemaker_base_operator.py b/airflow/contrib/operators/sagemaker_base_operator.py
new file mode 100644
index 0000000000000..d6717fd6b41fc
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_base_operator.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from typing import Iterable
+
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class SageMakerBaseOperator(BaseOperator):
+    """
+    This is the base operator for all SageMaker operators.
+
+    :param config: The configuration necessary to start a training job (templated)
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: str
+    """
+
+    template_fields = ['config']
+    template_ext = ()
+    ui_color = '#ededed'
+
+    integer_fields = []  # type: Iterable[Iterable[str]]
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 aws_conn_id='aws_default',
+                 *args, **kwargs):
+        super(SageMakerBaseOperator, self).__init__(*args, **kwargs)
+
+        self.aws_conn_id = aws_conn_id
+        self.config = config
+        self.hook = None
+
+    def parse_integer(self, config, field):
+        if len(field) == 1:
+            if isinstance(config, list):
+                for sub_config in config:
+                    self.parse_integer(sub_config, field)
+                return
+            head = field[0]
+            if head in config:
+                config[head] = int(config[head])
+            return
+
+        if isinstance(config, list):
+            for sub_config in config:
+                self.parse_integer(sub_config, field)
+            return
+
+        head, tail = field[0], field[1:]
+        if head in config:
+            self.parse_integer(config[head], tail)
+        return
+
+    def parse_config_integers(self):
+        # Parse the integer fields of training config to integers
+        # in case the config is rendered by Jinja and all fields are str
+        for field in self.integer_fields:
+            self.parse_integer(self.config, field)
+
+    def expand_role(self):
+        pass
+
+    def preprocess_config(self):
+        self.log.info(
+            'Preprocessing the config and doing required s3_operations'
+        )
+        self.hook = SageMakerHook(aws_conn_id=self.aws_conn_id)
+
+        self.hook.configure_s3_resources(self.config)
+        self.parse_config_integers()
+        self.expand_role()
+
+        self.log.info(
+            'After preprocessing the config is:\n {}'.format(
+                json.dumps(self.config, sort_keys=True, indent=4, separators=(',', ': ')))
+        )
+
+    def execute(self, context):
+        raise NotImplementedError('Please implement execute() in sub class!')
diff --git a/airflow/contrib/operators/sagemaker_endpoint_config_operator.py b/airflow/contrib/operators/sagemaker_endpoint_config_operator.py
new file mode 100644
index 0000000000000..e4f8ce7c9103f
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_endpoint_config_operator.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerEndpointConfigOperator(SageMakerBaseOperator):
+
+    """
+    Create a SageMaker endpoint config.
+
+    This operator returns The ARN of the endpoint config created in Amazon SageMaker
+
+    :param config: The configuration necessary to create an endpoint config.
+
+        For details of the configuration parameter see :py:meth:`SageMaker.Client.create_endpoint_config`
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: str
+    """
+
+    integer_fields = [
+        ['ProductionVariants', 'InitialInstanceCount']
+    ]
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 *args, **kwargs):
+        super(SageMakerEndpointConfigOperator, self).__init__(config=config,
+                                                              *args, **kwargs)
+
+        self.config = config
+
+    def execute(self, context):
+        self.preprocess_config()
+
+        self.log.info('Creating SageMaker Endpoint Config %s.', self.config['EndpointConfigName'])
+        response = self.hook.create_endpoint_config(self.config)
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            raise AirflowException(
+                'Sagemaker endpoint config creation failed: %s' % response)
+        else:
+            return {
+                'EndpointConfig': self.hook.describe_endpoint_config(
+                    self.config['EndpointConfigName']
+                )
+            }
diff --git a/airflow/contrib/operators/sagemaker_endpoint_operator.py b/airflow/contrib/operators/sagemaker_endpoint_operator.py
new file mode 100644
index 0000000000000..45af3381cb9cd
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_endpoint_operator.py
@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerEndpointOperator(SageMakerBaseOperator):
+
+    """
+    Create a SageMaker endpoint.
+
+    This operator returns The ARN of the endpoint created in Amazon SageMaker
+
+    :param config:
+        The configuration necessary to create an endpoint.
+
+        If you need to create a SageMaker endpoint based on an existed
+        SageMaker model and an existed SageMaker endpoint config::
+
+            config = endpoint_configuration;
+
+        If you need to create all of SageMaker model, SageMaker endpoint-config and SageMaker endpoint::
+
+            config = {
+                'Model': model_configuration,
+                'EndpointConfig': endpoint_config_configuration,
+                'Endpoint': endpoint_configuration
+            }
+
+        For details of the configuration parameter of model_configuration see
+        :py:meth:`SageMaker.Client.create_model`
+
+        For details of the configuration parameter of endpoint_config_configuration see
+        :py:meth:`SageMaker.Client.create_endpoint_config`
+
+        For details of the configuration parameter of endpoint_configuration see
+        :py:meth:`SageMaker.Client.create_endpoint`
+
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: str
+    :param wait_for_completion: Whether the operator should wait until the endpoint creation finishes.
+    :type wait_for_completion: bool
+    :param check_interval: If wait is set to True, this is the time interval, in seconds, that this operation
+        waits before polling the status of the endpoint creation.
+    :type check_interval: int
+    :param max_ingestion_time: If wait is set to True, this operation fails if the endpoint creation doesn't
+        finish within max_ingestion_time seconds. If you set this parameter to None it never times out.
+    :type max_ingestion_time: int
+    :param operation: Whether to create an endpoint or update an endpoint. Must be either 'create or 'update'.
+    :type operation: str
+    """
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 wait_for_completion=True,
+                 check_interval=30,
+                 max_ingestion_time=None,
+                 operation='create',
+                 *args, **kwargs):
+        super(SageMakerEndpointOperator, self).__init__(config=config,
+                                                        *args, **kwargs)
+
+        self.config = config
+        self.wait_for_completion = wait_for_completion
+        self.check_interval = check_interval
+        self.max_ingestion_time = max_ingestion_time
+        self.operation = operation.lower()
+        if self.operation not in ['create', 'update']:
+            raise ValueError('Invalid value! Argument operation has to be one of "create" and "update"')
+        self.create_integer_fields()
+
+    def create_integer_fields(self):
+        if 'EndpointConfig' in self.config:
+            self.integer_fields = [
+                ['EndpointConfig', 'ProductionVariants', 'InitialInstanceCount']
+            ]
+
+    def expand_role(self):
+        if 'Model' not in self.config:
+            return
+        hook = AwsHook(self.aws_conn_id)
+        config = self.config['Model']
+        if 'ExecutionRoleArn' in config:
+            config['ExecutionRoleArn'] = hook.expand_role(config['ExecutionRoleArn'])
+
+    def execute(self, context):
+        self.preprocess_config()
+
+        model_info = self.config.get('Model')
+        endpoint_config_info = self.config.get('EndpointConfig')
+        endpoint_info = self.config.get('Endpoint', self.config)
+
+        if model_info:
+            self.log.info('Creating SageMaker model %s.', model_info['ModelName'])
+            self.hook.create_model(model_info)
+
+        if endpoint_config_info:
+            self.log.info('Creating endpoint config %s.', endpoint_config_info['EndpointConfigName'])
+            self.hook.create_endpoint_config(endpoint_config_info)
+
+        if self.operation == 'create':
+            sagemaker_operation = self.hook.create_endpoint
+            log_str = 'Creating'
+        elif self.operation == 'update':
+            sagemaker_operation = self.hook.update_endpoint
+            log_str = 'Updating'
+        else:
+            raise ValueError('Invalid value! Argument operation has to be one of "create" and "update"')
+
+        self.log.info('%s SageMaker endpoint %s.', log_str, endpoint_info['EndpointName'])
+
+        response = sagemaker_operation(
+            endpoint_info,
+            wait_for_completion=self.wait_for_completion,
+            check_interval=self.check_interval,
+            max_ingestion_time=self.max_ingestion_time
+        )
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            raise AirflowException(
+                'Sagemaker endpoint creation failed: %s' % response)
+        else:
+            return {
+                'EndpointConfig': self.hook.describe_endpoint_config(
+                    endpoint_info['EndpointConfigName']
+                ),
+                'Endpoint': self.hook.describe_endpoint(
+                    endpoint_info['EndpointName']
+                )
+            }
diff --git a/airflow/contrib/operators/sagemaker_model_operator.py b/airflow/contrib/operators/sagemaker_model_operator.py
new file mode 100644
index 0000000000000..5bfc1feb3149b
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_model_operator.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerModelOperator(SageMakerBaseOperator):
+
+    """
+    Create a SageMaker model.
+
+    This operator returns The ARN of the model created in Amazon SageMaker
+
+    :param config: The configuration necessary to create a model.
+
+        For details of the configuration parameter see :py:meth:`SageMaker.Client.create_model`
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: str
+    """
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 *args, **kwargs):
+        super(SageMakerModelOperator, self).__init__(config=config,
+                                                     *args, **kwargs)
+
+        self.config = config
+
+    def expand_role(self):
+        if 'ExecutionRoleArn' in self.config:
+            hook = AwsHook(self.aws_conn_id)
+            self.config['ExecutionRoleArn'] = hook.expand_role(self.config['ExecutionRoleArn'])
+
+    def execute(self, context):
+        self.preprocess_config()
+
+        self.log.info('Creating SageMaker Model %s.', self.config['ModelName'])
+        response = self.hook.create_model(self.config)
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            raise AirflowException('Sagemaker model creation failed: %s' % response)
+        else:
+            return {
+                'Model': self.hook.describe_model(
+                    self.config['ModelName']
+                )
+            }
diff --git a/airflow/contrib/operators/sagemaker_training_operator.py b/airflow/contrib/operators/sagemaker_training_operator.py
new file mode 100644
index 0000000000000..f10e13e9736c5
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_training_operator.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerTrainingOperator(SageMakerBaseOperator):
+    """
+    Initiate a SageMaker training job.
+
+    This operator returns The ARN of the training job created in Amazon SageMaker.
+
+    :param config: The configuration necessary to start a training job (templated).
+
+        For details of the configuration parameter see :py:meth:`SageMaker.Client.create_training_job`
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: str
+    :param wait_for_completion: If wait is set to True, the time interval, in seconds,
+        that the operation waits to check the status of the training job.
+    :type wait_for_completion: bool
+    :param print_log: if the operator should print the cloudwatch log during training
+    :type print_log: bool
+    :param check_interval: if wait is set to be true, this is the time interval
+        in seconds which the operator will check the status of the training job
+    :type check_interval: int
+    :param max_ingestion_time: If wait is set to True, the operation fails if the training job
+        doesn't finish within max_ingestion_time seconds. If you set this parameter to None,
+        the operation does not timeout.
+    :type max_ingestion_time: int
+    """
+
+    integer_fields = [
+        ['ResourceConfig', 'InstanceCount'],
+        ['ResourceConfig', 'VolumeSizeInGB'],
+        ['StoppingCondition', 'MaxRuntimeInSeconds']
+    ]
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 wait_for_completion=True,
+                 print_log=True,
+                 check_interval=30,
+                 max_ingestion_time=None,
+                 *args, **kwargs):
+        super(SageMakerTrainingOperator, self).__init__(config=config,
+                                                        *args, **kwargs)
+
+        self.wait_for_completion = wait_for_completion
+        self.print_log = print_log
+        self.check_interval = check_interval
+        self.max_ingestion_time = max_ingestion_time
+
+    def expand_role(self):
+        if 'RoleArn' in self.config:
+            hook = AwsHook(self.aws_conn_id)
+            self.config['RoleArn'] = hook.expand_role(self.config['RoleArn'])
+
+    def execute(self, context):
+        self.preprocess_config()
+
+        self.log.info('Creating SageMaker Training Job %s.', self.config['TrainingJobName'])
+
+        response = self.hook.create_training_job(
+            self.config,
+            wait_for_completion=self.wait_for_completion,
+            print_log=self.print_log,
+            check_interval=self.check_interval,
+            max_ingestion_time=self.max_ingestion_time
+        )
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            raise AirflowException('Sagemaker Training Job creation failed: %s' % response)
+        else:
+            return {
+                'Training': self.hook.describe_training_job(
+                    self.config['TrainingJobName']
+                )
+            }
diff --git a/airflow/contrib/operators/sagemaker_transform_operator.py b/airflow/contrib/operators/sagemaker_transform_operator.py
new file mode 100644
index 0000000000000..9dcf9ab32764e
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_transform_operator.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerTransformOperator(SageMakerBaseOperator):
+    """
+    Initiate a SageMaker transform job.
+
+    This operator returns The ARN of the model created in Amazon SageMaker.
+
+    :param config: The configuration necessary to start a transform job (templated).
+
+        If you need to create a SageMaker transform job based on an existed SageMaker model::
+
+            config = transform_config
+
+        If you need to create both SageMaker model and SageMaker Transform job::
+
+            config = {
+                'Model': model_config,
+                'Transform': transform_config
+            }
+
+        For details of the configuration parameter of transform_config see
+        :py:meth:`SageMaker.Client.create_transform_job`
+
+        For details of the configuration parameter of model_config, See:
+        :py:meth:`SageMaker.Client.create_model`
+
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: string
+    :param wait_for_completion: Set to True to wait until the transform job finishes.
+    :type wait_for_completion: bool
+    :param check_interval: If wait is set to True, the time interval, in seconds,
+        that this operation waits to check the status of the transform job.
+    :type check_interval: int
+    :param max_ingestion_time: If wait is set to True, the operation fails
+        if the transform job doesn't finish within max_ingestion_time seconds. If you
+        set this parameter to None, the operation does not timeout.
+    :type max_ingestion_time: int
+    """
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 wait_for_completion=True,
+                 check_interval=30,
+                 max_ingestion_time=None,
+                 *args, **kwargs):
+        super(SageMakerTransformOperator, self).__init__(config=config,
+                                                         *args, **kwargs)
+        self.config = config
+        self.wait_for_completion = wait_for_completion
+        self.check_interval = check_interval
+        self.max_ingestion_time = max_ingestion_time
+        self.create_integer_fields()
+
+    def create_integer_fields(self):
+        self.integer_fields = [
+            ['Transform', 'TransformResources', 'InstanceCount'],
+            ['Transform', 'MaxConcurrentTransforms'],
+            ['Transform', 'MaxPayloadInMB']
+        ]
+        if 'Transform' not in self.config:
+            for field in self.integer_fields:
+                field.pop(0)
+
+    def expand_role(self):
+        if 'Model' not in self.config:
+            return
+        config = self.config['Model']
+        if 'ExecutionRoleArn' in config:
+            hook = AwsHook(self.aws_conn_id)
+            config['ExecutionRoleArn'] = hook.expand_role(config['ExecutionRoleArn'])
+
+    def execute(self, context):
+        self.preprocess_config()
+
+        model_config = self.config.get('Model')
+        transform_config = self.config.get('Transform', self.config)
+
+        if model_config:
+            self.log.info('Creating SageMaker Model %s for transform job', model_config['ModelName'])
+            self.hook.create_model(model_config)
+
+        self.log.info('Creating SageMaker transform Job %s.', transform_config['TransformJobName'])
+        response = self.hook.create_transform_job(
+            transform_config,
+            wait_for_completion=self.wait_for_completion,
+            check_interval=self.check_interval,
+            max_ingestion_time=self.max_ingestion_time)
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            raise AirflowException('Sagemaker transform Job creation failed: %s' % response)
+        else:
+            return {
+                'Model': self.hook.describe_model(
+                    transform_config['ModelName']
+                ),
+                'Transform': self.hook.describe_transform_job(
+                    transform_config['TransformJobName']
+                )
+            }
diff --git a/airflow/contrib/operators/sagemaker_tuning_operator.py b/airflow/contrib/operators/sagemaker_tuning_operator.py
new file mode 100644
index 0000000000000..2a92bf1b81cf7
--- /dev/null
+++ b/airflow/contrib/operators/sagemaker_tuning_operator.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.aws_hook import AwsHook
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerTuningOperator(SageMakerBaseOperator):
+    """
+    Initiate a SageMaker hyperparameter tuning job.
+
+    This operator returns The ARN of the tuning job created in Amazon SageMaker.
+
+    :param config: The configuration necessary to start a tuning job (templated).
+
+        For details of the configuration parameter see
+        :py:meth:`SageMaker.Client.create_hyper_parameter_tuning_job`
+    :type config: dict
+    :param aws_conn_id: The AWS connection ID to use.
+    :type aws_conn_id: str
+    :param wait_for_completion: Set to True to wait until the tuning job finishes.
+    :type wait_for_completion: bool
+    :param check_interval: If wait is set to True, the time interval, in seconds,
+        that this operation waits to check the status of the tuning job.
+    :type check_interval: int
+    :param max_ingestion_time: If wait is set to True, the operation fails
+        if the tuning job doesn't finish within max_ingestion_time seconds. If you
+        set this parameter to None, the operation does not timeout.
+    :type max_ingestion_time: int
+    """
+
+    integer_fields = [
+        ['HyperParameterTuningJobConfig', 'ResourceLimits', 'MaxNumberOfTrainingJobs'],
+        ['HyperParameterTuningJobConfig', 'ResourceLimits', 'MaxParallelTrainingJobs'],
+        ['TrainingJobDefinition', 'ResourceConfig', 'InstanceCount'],
+        ['TrainingJobDefinition', 'ResourceConfig', 'VolumeSizeInGB'],
+        ['TrainingJobDefinition', 'StoppingCondition', 'MaxRuntimeInSeconds']
+    ]
+
+    @apply_defaults
+    def __init__(self,
+                 config,
+                 wait_for_completion=True,
+                 check_interval=30,
+                 max_ingestion_time=None,
+                 *args, **kwargs):
+        super(SageMakerTuningOperator, self).__init__(config=config,
+                                                      *args, **kwargs)
+        self.config = config
+        self.wait_for_completion = wait_for_completion
+        self.check_interval = check_interval
+        self.max_ingestion_time = max_ingestion_time
+
+    def expand_role(self):
+        if 'TrainingJobDefinition' in self.config:
+            config = self.config['TrainingJobDefinition']
+            if 'RoleArn' in config:
+                hook = AwsHook(self.aws_conn_id)
+                config['RoleArn'] = hook.expand_role(config['RoleArn'])
+
+    def execute(self, context):
+        self.preprocess_config()
+
+        self.log.info(
+            'Creating SageMaker Hyper-Parameter Tuning Job %s', self.config['HyperParameterTuningJobName']
+        )
+
+        response = self.hook.create_tuning_job(
+            self.config,
+            wait_for_completion=self.wait_for_completion,
+            check_interval=self.check_interval,
+            max_ingestion_time=self.max_ingestion_time
+        )
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            raise AirflowException('Sagemaker Tuning Job creation failed: %s' % response)
+        else:
+            return {
+                'Tuning': self.hook.describe_tuning_job(
+                    self.config['HyperParameterTuningJobName']
+                )
+            }
diff --git a/airflow/contrib/operators/segment_track_event_operator.py b/airflow/contrib/operators/segment_track_event_operator.py
index 04f6ae6d41d4b..5d2f2db539083 100644
--- a/airflow/contrib/operators/segment_track_event_operator.py
+++ b/airflow/contrib/operators/segment_track_event_operator.py
@@ -27,16 +27,16 @@ class SegmentTrackEventOperator(BaseOperator):
     Send Track Event to Segment for a specified user_id and event
 
     :param user_id: The ID for this user in your database. (templated)
-    :type user_id: string
+    :type user_id: str
     :param event: The name of the event you're tracking. (templated)
-    :type event: string
+    :type event: str
     :param properties: A dictionary of properties for the event. (templated)
     :type properties: dict
     :param segment_conn_id: The connection ID to use when connecting to Segment.
-    :type segment_conn_id: string
+    :type segment_conn_id: str
     :param segment_debug_mode: Determines whether Segment should run in debug mode.
         Defaults to False
-    :type segment_debug_mode: boolean
+    :type segment_debug_mode: bool
     """
     template_fields = ('user_id', 'event', 'properties')
     ui_color = '#ffd700'
diff --git a/airflow/contrib/operators/sftp_operator.py b/airflow/contrib/operators/sftp_operator.py
index 153f440d41186..4af5fe9a59979 100644
--- a/airflow/contrib/operators/sftp_operator.py
+++ b/airflow/contrib/operators/sftp_operator.py
@@ -16,6 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import os
+
 from airflow.contrib.hooks.ssh_hook import SSHHook
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
@@ -30,23 +32,48 @@ class SFTPOperation(object):
 class SFTPOperator(BaseOperator):
     """
     SFTPOperator for transferring files from remote host to local or vice a versa.
-    This operator uses ssh_hook to open sftp trasport channel that serve as basis
+    This operator uses ssh_hook to open sftp transport channel that serve as basis
     for file transfer.
 
-    :param ssh_hook: predefined ssh_hook to use for remote execution
-    :type ssh_hook: :class:`SSHHook`
-    :param ssh_conn_id: connection id from airflow Connections
+    :param ssh_hook: predefined ssh_hook to use for remote execution.
+        Either `ssh_hook` or `ssh_conn_id` needs to be provided.
+    :type ssh_hook: airflow.contrib.hooks.ssh_hook.SSHHook
+    :param ssh_conn_id: connection id from airflow Connections.
+        `ssh_conn_id` will be ignored if `ssh_hook` is provided.
     :type ssh_conn_id: str
-    :param remote_host: remote host to connect
+    :param remote_host: remote host to connect (templated)
+        Nullable. If provided, it will replace the `remote_host` which was
+        defined in `ssh_hook` or predefined in the connection of `ssh_conn_id`.
     :type remote_host: str
     :param local_filepath: local file path to get or put. (templated)
     :type local_filepath: str
     :param remote_filepath: remote file path to get or put. (templated)
     :type remote_filepath: str
-    :param operation: specify operation 'get' or 'put', defaults to get
-    :type get: bool
+    :param operation: specify operation 'get' or 'put', defaults to put
+    :type operation: str
+    :param confirm: specify if the SFTP operation should be confirmed, defaults to True
+    :type confirm: bool
+    :param create_intermediate_dirs: create missing intermediate directories when
+        copying from remote to local and vice-versa. Default is False.
+
+        Example: The following task would copy ``file.txt`` to the remote host
+        at ``/tmp/tmp1/tmp2/`` while creating ``tmp``,``tmp1`` and ``tmp2`` if they
+        don't exist. If the parameter is not passed it would error as the directory
+        does not exist. ::
+
+            put_file = SFTPOperator(
+                task_id="test_sftp",
+                ssh_conn_id="ssh_default",
+                local_filepath="/tmp/file.txt",
+                remote_filepath="/tmp/tmp1/tmp2/file.txt",
+                operation="put",
+                create_intermediate_dirs=True,
+                dag=dag
+            )
+
+    :type create_intermediate_dirs: bool
     """
-    template_fields = ('local_filepath', 'remote_filepath')
+    template_fields = ('local_filepath', 'remote_filepath', 'remote_host')
 
     @apply_defaults
     def __init__(self,
@@ -56,6 +83,8 @@ def __init__(self,
                  local_filepath=None,
                  remote_filepath=None,
                  operation=SFTPOperation.PUT,
+                 confirm=True,
+                 create_intermediate_dirs=False,
                  *args,
                  **kwargs):
         super(SFTPOperator, self).__init__(*args, **kwargs)
@@ -65,6 +94,8 @@ def __init__(self,
         self.local_filepath = local_filepath
         self.remote_filepath = remote_filepath
         self.operation = operation
+        self.confirm = confirm
+        self.create_intermediate_dirs = create_intermediate_dirs
         if not (self.operation.lower() == SFTPOperation.GET or
                 self.operation.lower() == SFTPOperation.PUT):
             raise TypeError("unsupported operation value {0}, expected {1} or {2}"
@@ -73,30 +104,77 @@ def __init__(self,
     def execute(self, context):
         file_msg = None
         try:
-            if self.ssh_conn_id and not self.ssh_hook:
-                self.ssh_hook = SSHHook(ssh_conn_id=self.ssh_conn_id)
+            if self.ssh_conn_id:
+                if self.ssh_hook and isinstance(self.ssh_hook, SSHHook):
+                    self.log.info("ssh_conn_id is ignored when ssh_hook is provided.")
+                else:
+                    self.log.info("ssh_hook is not provided or invalid. " +
+                                  "Trying ssh_conn_id to create SSHHook.")
+                    self.ssh_hook = SSHHook(ssh_conn_id=self.ssh_conn_id)
 
             if not self.ssh_hook:
-                raise AirflowException("can not operate without ssh_hook or ssh_conn_id")
+                raise AirflowException("Cannot operate without ssh_hook or ssh_conn_id.")
 
             if self.remote_host is not None:
+                self.log.info("remote_host is provided explicitly. " +
+                              "It will replace the remote_host which was defined " +
+                              "in ssh_hook or predefined in connection of ssh_conn_id.")
                 self.ssh_hook.remote_host = self.remote_host
 
-            ssh_client = self.ssh_hook.get_conn()
-            sftp_client = ssh_client.open_sftp()
-            if self.operation.lower() == SFTPOperation.GET:
-                file_msg = "from {0} to {1}".format(self.remote_filepath,
-                                                    self.local_filepath)
-                self.log.debug("Starting to transfer %s", file_msg)
-                sftp_client.get(self.remote_filepath, self.local_filepath)
-            else:
-                file_msg = "from {0} to {1}".format(self.local_filepath,
-                                                    self.remote_filepath)
-                self.log.debug("Starting to transfer file %s", file_msg)
-                sftp_client.put(self.local_filepath, self.remote_filepath)
+            with self.ssh_hook.get_conn() as ssh_client:
+                sftp_client = ssh_client.open_sftp()
+                if self.operation.lower() == SFTPOperation.GET:
+                    local_folder = os.path.dirname(self.local_filepath)
+                    if self.create_intermediate_dirs:
+                        # Create Intermediate Directories if it doesn't exist
+                        try:
+                            os.makedirs(local_folder)
+                        except OSError:
+                            if not os.path.isdir(local_folder):
+                                raise
+                    file_msg = "from {0} to {1}".format(self.remote_filepath,
+                                                        self.local_filepath)
+                    self.log.debug("Starting to transfer %s", file_msg)
+                    sftp_client.get(self.remote_filepath, self.local_filepath)
+                else:
+                    remote_folder = os.path.dirname(self.remote_filepath)
+                    if self.create_intermediate_dirs:
+                        _make_intermediate_dirs(
+                            sftp_client=sftp_client,
+                            remote_directory=remote_folder,
+                        )
+                    file_msg = "from {0} to {1}".format(self.local_filepath,
+                                                        self.remote_filepath)
+                    self.log.debug("Starting to transfer file %s", file_msg)
+                    sftp_client.put(self.local_filepath,
+                                    self.remote_filepath,
+                                    confirm=self.confirm)
 
         except Exception as e:
             raise AirflowException("Error while transferring {0}, error: {1}"
                                    .format(file_msg, str(e)))
 
         return None
+
+
+def _make_intermediate_dirs(sftp_client, remote_directory):
+    """
+    Create all the intermediate directories in a remote host
+
+    :param sftp_client: A Paramiko SFTP client.
+    :param remote_directory: Absolute Path of the directory containing the file
+    :return:
+    """
+    if remote_directory == '/':
+        sftp_client.chdir('/')
+        return
+    if remote_directory == '':
+        return
+    try:
+        sftp_client.chdir(remote_directory)
+    except IOError:
+        dirname, basename = os.path.split(remote_directory.rstrip('/'))
+        _make_intermediate_dirs(sftp_client, dirname)
+        sftp_client.mkdir(basename)
+        sftp_client.chdir(basename)
+        return
diff --git a/airflow/contrib/operators/sftp_to_s3_operator.py b/airflow/contrib/operators/sftp_to_s3_operator.py
new file mode 100644
index 0000000000000..cefc838cf12d1
--- /dev/null
+++ b/airflow/contrib/operators/sftp_to_s3_operator.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.models import BaseOperator
+from airflow.hooks.S3_hook import S3Hook
+from airflow.contrib.hooks.ssh_hook import SSHHook
+from tempfile import NamedTemporaryFile
+from urllib.parse import urlparse
+from airflow.utils.decorators import apply_defaults
+
+
+class SFTPToS3Operator(BaseOperator):
+    """
+    This operator enables the transferring of files from a SFTP server to
+    Amazon S3.
+
+    :param sftp_conn_id: The sftp connection id. The name or identifier for
+        establishing a connection to the SFTP server.
+    :type sftp_conn_id: string
+    :param sftp_path: The sftp remote path. This is the specified file path
+        for downloading the file from the SFTP server.
+    :type sftp_path: string
+    :param s3_conn_id: The s3 connection id. The name or identifier for
+        establishing a connection to S3
+    :type s3_conn_id: string
+    :param s3_bucket: The targeted s3 bucket. This is the S3 bucket to where
+        the file is uploaded.
+    :type s3_bucket: string
+    :param s3_key: The targeted s3 key. This is the specified path for
+        uploading the file to S3.
+    :type s3_key: string
+    """
+
+    template_fields = ('s3_key', 'sftp_path')
+
+    @apply_defaults
+    def __init__(self,
+                 s3_bucket,
+                 s3_key,
+                 sftp_path,
+                 sftp_conn_id='ssh_default',
+                 s3_conn_id='aws_default',
+                 *args,
+                 **kwargs):
+        super(SFTPToS3Operator, self).__init__(*args, **kwargs)
+        self.sftp_conn_id = sftp_conn_id
+        self.sftp_path = sftp_path
+        self.s3_bucket = s3_bucket
+        self.s3_key = s3_key
+        self.s3_conn_id = s3_conn_id
+
+    @staticmethod
+    def get_s3_key(s3_key):
+        """This parses the correct format for S3 keys
+            regardless of how the S3 url is passed."""
+
+        parsed_s3_key = urlparse(s3_key)
+        return parsed_s3_key.path.lstrip('/')
+
+    def execute(self, context):
+        self.s3_key = self.get_s3_key(self.s3_key)
+        ssh_hook = SSHHook(ssh_conn_id=self.sftp_conn_id)
+        s3_hook = S3Hook(self.s3_conn_id)
+
+        sftp_client = ssh_hook.get_conn().open_sftp()
+
+        with NamedTemporaryFile("w") as f:
+            sftp_client.get(self.sftp_path, f.name)
+
+            s3_hook.load_file(
+                filename=f.name,
+                key=self.s3_key,
+                bucket_name=self.s3_bucket,
+                replace=True
+            )
diff --git a/airflow/contrib/operators/slack_webhook_operator.py b/airflow/contrib/operators/slack_webhook_operator.py
index bab3e90856d34..8950c69b5a38e 100644
--- a/airflow/contrib/operators/slack_webhook_operator.py
+++ b/airflow/contrib/operators/slack_webhook_operator.py
@@ -31,12 +31,15 @@ class SlackWebhookOperator(SimpleHttpOperator):
     Each Slack webhook token can be pre-configured to use a specific channel, username and
     icon. You can override these defaults in this hook.
 
-    :param conn_id: connection that has Slack webhook token in the extra field
-    :type conn_id: str
+    :param http_conn_id: connection that has Slack webhook token in the extra field
+    :type http_conn_id: str
     :param webhook_token: Slack webhook token
     :type webhook_token: str
     :param message: The message you want to send on Slack
     :type message: str
+    :param attachments: The attachments to send on Slack. Should be a list of
+                        dictionaries representing Slack attachments.
+    :type attachments: list
     :param channel: The channel the message should be posted to
     :type channel: str
     :param username: The username to post to slack with
@@ -55,6 +58,7 @@ def __init__(self,
                  http_conn_id=None,
                  webhook_token=None,
                  message="",
+                 attachments=None,
                  channel=None,
                  username=None,
                  icon_emoji=None,
@@ -68,6 +72,7 @@ def __init__(self,
         self.http_conn_id = http_conn_id
         self.webhook_token = webhook_token
         self.message = message
+        self.attachments = attachments
         self.channel = channel
         self.username = username
         self.icon_emoji = icon_emoji
@@ -77,12 +82,13 @@ def __init__(self,
 
     def execute(self, context):
         """
-        Call the SparkSqlHook to run the provided sql query
+        Call the SlackWebhookHook to post the provided Slack message
         """
         self.hook = SlackWebhookHook(
             self.http_conn_id,
             self.webhook_token,
             self.message,
+            self.attachments,
             self.channel,
             self.username,
             self.icon_emoji,
diff --git a/airflow/contrib/operators/snowflake_operator.py b/airflow/contrib/operators/snowflake_operator.py
index 39d7d496ea740..6c3f403912134 100644
--- a/airflow/contrib/operators/snowflake_operator.py
+++ b/airflow/contrib/operators/snowflake_operator.py
@@ -26,16 +26,16 @@ class SnowflakeOperator(BaseOperator):
     Executes sql code in a Snowflake database
 
     :param snowflake_conn_id: reference to specific snowflake connection id
-    :type snowflake_conn_id: string
+    :type snowflake_conn_id: str
     :param sql: the sql code to be executed. (templated)
     :type sql: Can receive a str representing a sql statement,
         a list of str (sql statements), or reference to a template file.
         Template reference are recognized by str ending in '.sql'
     :param warehouse: name of warehouse which overwrite defined
         one in connection
-    :type warehouse: string
+    :type warehouse: str
     :param database: name of database which overwrite defined one in connection
-    :type database: string
+    :type database: str
     """
 
     template_fields = ('sql',)
diff --git a/airflow/contrib/operators/sns_publish_operator.py b/airflow/contrib/operators/sns_publish_operator.py
new file mode 100644
index 0000000000000..d1a042be55e03
--- /dev/null
+++ b/airflow/contrib/operators/sns_publish_operator.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.aws_sns_hook import AwsSnsHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class SnsPublishOperator(BaseOperator):
+    """
+    Publish a message to Amazon SNS.
+
+    :param aws_conn_id: aws connection to use
+    :type aws_conn_id: str
+    :param target_arn: either a TopicArn or an EndpointArn
+    :type target_arn: str
+    :param message: the default message you want to send (templated)
+    :type message: str
+    """
+    template_fields = ['message']
+    template_ext = ()
+
+    @apply_defaults
+    def __init__(
+            self,
+            target_arn,
+            message,
+            aws_conn_id='aws_default',
+            *args, **kwargs):
+        super(SnsPublishOperator, self).__init__(*args, **kwargs)
+        self.target_arn = target_arn
+        self.message = message
+        self.aws_conn_id = aws_conn_id
+
+    def execute(self, context):
+        sns = AwsSnsHook(aws_conn_id=self.aws_conn_id)
+
+        self.log.info(
+            'Sending SNS notification to {} using {}:\n{}'.format(
+                self.target_arn,
+                self.aws_conn_id,
+                self.message
+            )
+        )
+
+        return sns.publish_to_target(
+            target_arn=self.target_arn,
+            message=self.message
+        )
diff --git a/airflow/contrib/operators/spark_jdbc_operator.py b/airflow/contrib/operators/spark_jdbc_operator.py
index 42f9dd5676f1b..b1c2a138b9ea2 100644
--- a/airflow/contrib/operators/spark_jdbc_operator.py
+++ b/airflow/contrib/operators/spark_jdbc_operator.py
@@ -64,7 +64,7 @@ class SparkJDBCOperator(SparkSubmitOperator):
     :param jdbc_table: The name of the JDBC table
     :type jdbc_table: str
     :param jdbc_conn_id: Connection id used for connection to JDBC database
-    :type: jdbc_conn_id: str
+    :type jdbc_conn_id: str
     :param jdbc_driver: Name of the JDBC driver to use for the JDBC connection. This
                         driver (usually a jar) should be passed in the 'jars' parameter
     :type jdbc_driver: str
diff --git a/airflow/contrib/operators/spark_submit_operator.py b/airflow/contrib/operators/spark_submit_operator.py
index 3c3cab0ceb350..45a70b64ea827 100644
--- a/airflow/contrib/operators/spark_submit_operator.py
+++ b/airflow/contrib/operators/spark_submit_operator.py
@@ -29,31 +29,30 @@ class SparkSubmitOperator(BaseOperator):
     It requires that the "spark-submit" binary is in the PATH or the spark-home is set
     in the extra on the connection.
 
-    :param application: The application that submitted as a job, either jar or
-        py file. (templated)
+    :param application: The application that submitted as a job, either jar or py file. (templated)
     :type application: str
-    :param conf: Arbitrary Spark configuration properties
+    :param conf: Arbitrary Spark configuration properties (templated)
     :type conf: dict
     :param conn_id: The connection id as configured in Airflow administration. When an
                     invalid connection_id is supplied, it will default to yarn.
     :type conn_id: str
     :param files: Upload additional files to the executor running the job, separated by a
                   comma. Files will be placed in the working directory of each executor.
-                  For example, serialized objects.
+                  For example, serialized objects. (templated)
     :type files: str
-    :param py_files: Additional python files used by the job, can be .zip, .egg or .py.
+    :param py_files: Additional python files used by the job, can be .zip, .egg or .py. (templated)
     :type py_files: str
-    :param jars: Submit additional jars to upload and place them in executor classpath.
-    :param driver_classpath: Additional, driver-specific, classpath settings.
-    :type driver_classpath: str
+    :param jars: Submit additional jars to upload and place them in executor classpath. (templated)
     :type jars: str
+    :param driver_class_path: Additional, driver-specific, classpath settings. (templated)
+    :type driver_class_path: str
     :param java_class: the main class of the Java application
     :type java_class: str
     :param packages: Comma-separated list of maven coordinates of jars to include on the
                      driver and executor classpaths. (templated)
     :type packages: str
     :param exclude_packages: Comma-separated list of maven coordinates of jars to exclude
-                             while resolving the dependencies provided in 'packages'
+                             while resolving the dependencies provided in 'packages' (templated)
     :type exclude_packages: str
     :param repositories: Comma-separated list of additional remote repositories to search
                          for the maven coordinates given with 'packages'
@@ -61,30 +60,33 @@ class SparkSubmitOperator(BaseOperator):
     :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors
                                  (Default: all the available cores on the worker)
     :type total_executor_cores: int
-    :param executor_cores: (Standalone & YARN only) Number of cores per executor
-                           (Default: 2)
+    :param executor_cores: (Standalone & YARN only) Number of cores per executor (Default: 2)
     :type executor_cores: int
     :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G)
     :type executor_memory: str
     :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G) (Default: 1G)
     :type driver_memory: str
-    :param keytab: Full path to the file that contains the keytab
+    :param keytab: Full path to the file that contains the keytab (templated)
     :type keytab: str
-    :param principal: The name of the kerberos principal used for keytab
+    :param principal: The name of the kerberos principal used for keytab (templated)
     :type principal: str
     :param name: Name of the job (default airflow-spark). (templated)
     :type name: str
     :param num_executors: Number of executors to launch
     :type num_executors: int
-    :param application_args: Arguments for the application being submitted
+    :param application_args: Arguments for the application being submitted (templated)
     :type application_args: list
-    :param env_vars: Environment variables for spark-submit. It
-                     supports yarn and k8s mode too.
+    :param env_vars: Environment variables for spark-submit. It supports yarn and k8s mode too. (templated)
     :type env_vars: dict
     :param verbose: Whether to pass the verbose flag to spark-submit process for debugging
     :type verbose: bool
+    :param spark_binary: The command to use for spark submit.
+                         Some distros may use spark2-submit.
+    :type spark_binary: string
     """
-    template_fields = ('_name', '_application_args', '_packages')
+    template_fields = ('_application', '_conf', '_files', '_py_files', '_jars', '_driver_class_path',
+                       '_packages', '_exclude_packages', '_keytab', '_principal', '_name',
+                       '_application_args', '_env_vars')
     ui_color = WEB_COLORS['LIGHTORANGE']
 
     @apply_defaults
@@ -94,7 +96,8 @@ def __init__(self,
                  conn_id='spark_default',
                  files=None,
                  py_files=None,
-                 driver_classpath=None,
+                 archives=None,
+                 driver_class_path=None,
                  jars=None,
                  java_class=None,
                  packages=None,
@@ -111,6 +114,7 @@ def __init__(self,
                  application_args=None,
                  env_vars=None,
                  verbose=False,
+                 spark_binary="spark-submit",
                  *args,
                  **kwargs):
         super(SparkSubmitOperator, self).__init__(*args, **kwargs)
@@ -118,7 +122,8 @@ def __init__(self,
         self._conf = conf
         self._files = files
         self._py_files = py_files
-        self._driver_classpath = driver_classpath
+        self._archives = archives
+        self._driver_class_path = driver_class_path
         self._jars = jars
         self._java_class = java_class
         self._packages = packages
@@ -135,6 +140,7 @@ def __init__(self,
         self._application_args = application_args
         self._env_vars = env_vars
         self._verbose = verbose
+        self._spark_binary = spark_binary
         self._hook = None
         self._conn_id = conn_id
 
@@ -147,7 +153,8 @@ def execute(self, context):
             conn_id=self._conn_id,
             files=self._files,
             py_files=self._py_files,
-            driver_classpath=self._driver_classpath,
+            archives=self._archives,
+            driver_class_path=self._driver_class_path,
             jars=self._jars,
             java_class=self._java_class,
             packages=self._packages,
@@ -163,7 +170,8 @@ def execute(self, context):
             num_executors=self._num_executors,
             application_args=self._application_args,
             env_vars=self._env_vars,
-            verbose=self._verbose
+            verbose=self._verbose,
+            spark_binary=self._spark_binary
         )
         self._hook.submit(self._application)
 
diff --git a/airflow/contrib/operators/sqoop_operator.py b/airflow/contrib/operators/sqoop_operator.py
index fa61ca14cac45..c7ebcf4b3a2d4 100644
--- a/airflow/contrib/operators/sqoop_operator.py
+++ b/airflow/contrib/operators/sqoop_operator.py
@@ -34,7 +34,7 @@ class SqoopOperator(BaseOperator):
     """
     Execute a Sqoop job.
     Documentation for Apache Sqoop can be found here:
-        https://sqoop.apache.org/docs/1.4.2/SqoopUserGuide.html.
+    https://sqoop.apache.org/docs/1.4.2/SqoopUserGuide.html
     """
     template_fields = ('conn_id', 'cmd_type', 'table', 'query', 'target_dir',
                        'file_type', 'columns', 'split_by',
diff --git a/airflow/contrib/operators/ssh_operator.py b/airflow/contrib/operators/ssh_operator.py
index d246800953341..5686792b35d62 100644
--- a/airflow/contrib/operators/ssh_operator.py
+++ b/airflow/contrib/operators/ssh_operator.py
@@ -31,11 +31,15 @@ class SSHOperator(BaseOperator):
     """
     SSHOperator to execute commands on given remote host using the ssh_hook.
 
-    :param ssh_hook: predefined ssh_hook to use for remote execution
-    :type ssh_hook: :class:`SSHHook`
-    :param ssh_conn_id: connection id from airflow Connections
+    :param ssh_hook: predefined ssh_hook to use for remote execution.
+        Either `ssh_hook` or `ssh_conn_id` needs to be provided.
+    :type ssh_hook: airflow.contrib.hooks.ssh_hook.SSHHook
+    :param ssh_conn_id: connection id from airflow Connections.
+        `ssh_conn_id` will be ignored if `ssh_hook` is provided.
     :type ssh_conn_id: str
-    :param remote_host: remote host to connect
+    :param remote_host: remote host to connect (templated)
+        Nullable. If provided, it will replace the `remote_host` which was
+        defined in `ssh_hook` or predefined in the connection of `ssh_conn_id`.
     :type remote_host: str
     :param command: command to execute on remote host. (templated)
     :type command: str
@@ -45,7 +49,7 @@ class SSHOperator(BaseOperator):
     :type do_xcom_push: bool
     """
 
-    template_fields = ('command',)
+    template_fields = ('command', 'remote_host')
     template_ext = ('.sh',)
 
     @apply_defaults
@@ -68,88 +72,96 @@ def __init__(self,
 
     def execute(self, context):
         try:
-            if self.ssh_conn_id and not self.ssh_hook:
-                self.ssh_hook = SSHHook(ssh_conn_id=self.ssh_conn_id)
+            if self.ssh_conn_id:
+                if self.ssh_hook and isinstance(self.ssh_hook, SSHHook):
+                    self.log.info("ssh_conn_id is ignored when ssh_hook is provided.")
+                else:
+                    self.log.info("ssh_hook is not provided or invalid. " +
+                                  "Trying ssh_conn_id to create SSHHook.")
+                    self.ssh_hook = SSHHook(ssh_conn_id=self.ssh_conn_id,
+                                            timeout=self.timeout)
 
             if not self.ssh_hook:
-                raise AirflowException("can not operate without ssh_hook or ssh_conn_id")
+                raise AirflowException("Cannot operate without ssh_hook or ssh_conn_id.")
 
             if self.remote_host is not None:
+                self.log.info("remote_host is provided explicitly. " +
+                              "It will replace the remote_host which was defined " +
+                              "in ssh_hook or predefined in connection of ssh_conn_id.")
                 self.ssh_hook.remote_host = self.remote_host
 
-            ssh_client = self.ssh_hook.get_conn()
-
             if not self.command:
-                raise AirflowException("no command specified so nothing to execute here.")
-
-            # Auto apply tty when its required in case of sudo
-            get_pty = False
-            if self.command.startswith('sudo'):
-                get_pty = True
-
-            # set timeout taken as params
-            stdin, stdout, stderr = ssh_client.exec_command(command=self.command,
-                                                            get_pty=get_pty,
-                                                            timeout=self.timeout
-                                                            )
-            # get channels
-            channel = stdout.channel
-
-            # closing stdin
-            stdin.close()
-            channel.shutdown_write()
-
-            agg_stdout = b''
-            agg_stderr = b''
-
-            # capture any initial output in case channel is closed already
-            stdout_buffer_length = len(stdout.channel.in_buffer)
-
-            if stdout_buffer_length > 0:
-                agg_stdout += stdout.channel.recv(stdout_buffer_length)
-
-            # read from both stdout and stderr
-            while not channel.closed or \
-                    channel.recv_ready() or \
-                    channel.recv_stderr_ready():
-                readq, _, _ = select([channel], [], [], self.timeout)
-                for c in readq:
-                    if c.recv_ready():
-                        line = stdout.channel.recv(len(c.in_buffer))
-                        line = line
-                        agg_stdout += line
-                        self.log.info(line.decode('utf-8').strip('\n'))
-                    if c.recv_stderr_ready():
-                        line = stderr.channel.recv_stderr(len(c.in_stderr_buffer))
-                        line = line
-                        agg_stderr += line
-                        self.log.warning(line.decode('utf-8').strip('\n'))
-                if stdout.channel.exit_status_ready()\
-                        and not stderr.channel.recv_stderr_ready()\
-                        and not stdout.channel.recv_ready():
-                    stdout.channel.shutdown_read()
-                    stdout.channel.close()
-                    break
-
-            stdout.close()
-            stderr.close()
-
-            exit_status = stdout.channel.recv_exit_status()
-            if exit_status is 0:
-                # returning output if do_xcom_push is set
-                if self.do_xcom_push:
-                    enable_pickling = configuration.conf.getboolean(
-                        'core', 'enable_xcom_pickling'
-                    )
-                    if enable_pickling:
-                        return agg_stdout
-                    else:
-                        return b64encode(agg_stdout).decode('utf-8')
-
-            else:
-                error_msg = agg_stderr.decode('utf-8')
-                raise AirflowException("error running cmd: {0}, error: {1}"
-                                       .format(self.command, error_msg))
+                raise AirflowException("SSH command not specified. Aborting.")
+
+            with self.ssh_hook.get_conn() as ssh_client:
+                # Auto apply tty when its required in case of sudo
+                get_pty = False
+                if self.command.startswith('sudo'):
+                    get_pty = True
+
+                # set timeout taken as params
+                stdin, stdout, stderr = ssh_client.exec_command(command=self.command,
+                                                                get_pty=get_pty,
+                                                                timeout=self.timeout
+                                                                )
+                # get channels
+                channel = stdout.channel
+
+                # closing stdin
+                stdin.close()
+                channel.shutdown_write()
+
+                agg_stdout = b''
+                agg_stderr = b''
+
+                # capture any initial output in case channel is closed already
+                stdout_buffer_length = len(stdout.channel.in_buffer)
+
+                if stdout_buffer_length > 0:
+                    agg_stdout += stdout.channel.recv(stdout_buffer_length)
+
+                # read from both stdout and stderr
+                while not channel.closed or \
+                        channel.recv_ready() or \
+                        channel.recv_stderr_ready():
+                    readq, _, _ = select([channel], [], [], self.timeout)
+                    for c in readq:
+                        if c.recv_ready():
+                            line = stdout.channel.recv(len(c.in_buffer))
+                            line = line
+                            agg_stdout += line
+                            self.log.info(line.decode('utf-8').strip('\n'))
+                        if c.recv_stderr_ready():
+                            line = stderr.channel.recv_stderr(len(c.in_stderr_buffer))
+                            line = line
+                            agg_stderr += line
+                            self.log.warning(line.decode('utf-8').strip('\n'))
+                    if stdout.channel.exit_status_ready()\
+                            and not stderr.channel.recv_stderr_ready()\
+                            and not stdout.channel.recv_ready():
+                        stdout.channel.shutdown_read()
+                        stdout.channel.close()
+                        break
+
+                stdout.close()
+                stderr.close()
+
+                exit_status = stdout.channel.recv_exit_status()
+                if exit_status == 0:
+                    # returning output if do_xcom_push is set
+                    if self.do_xcom_push:
+                        enable_pickling = configuration.conf.getboolean(
+                            'core', 'enable_xcom_pickling'
+                        )
+                        if enable_pickling:
+                            return agg_stdout
+                        else:
+                            return b64encode(agg_stdout).decode('utf-8')
+
+                else:
+                    error_msg = agg_stderr.decode('utf-8')
+                    raise AirflowException("error running cmd: {0}, error: {1}"
+                                           .format(self.command, error_msg))
 
         except Exception as e:
             raise AirflowException("SSH operator error: {0}".format(str(e)))
diff --git a/airflow/contrib/operators/vertica_operator.py b/airflow/contrib/operators/vertica_operator.py
index 41072ff82bdf0..37ccd134e84ac 100644
--- a/airflow/contrib/operators/vertica_operator.py
+++ b/airflow/contrib/operators/vertica_operator.py
@@ -26,7 +26,7 @@ class VerticaOperator(BaseOperator):
     Executes sql code in a specific Vertica database
 
     :param vertica_conn_id: reference to a specific Vertica database
-    :type vertica_conn_id: string
+    :type vertica_conn_id: str
     :param sql: the sql code to be executed. (templated)
     :type sql: Can receive a str representing a sql statement,
         a list of str (sql statements), or reference to a template file.
diff --git a/airflow/contrib/operators/vertica_to_mysql.py b/airflow/contrib/operators/vertica_to_mysql.py
index 9d3fe721e4cbb..99b55e61bc6e0 100644
--- a/airflow/contrib/operators/vertica_to_mysql.py
+++ b/airflow/contrib/operators/vertica_to_mysql.py
@@ -17,7 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import logging
 import MySQLdb
 
 from airflow.contrib.hooks.vertica_hook import VerticaHook
@@ -103,10 +102,10 @@ def execute(self, context):
                 if self.bulk_load:
                     tmpfile = NamedTemporaryFile("w")
 
-                    logging.info(
-                        "Selecting rows from Vertica to local file " + str(
-                            tmpfile.name) + "...")
-                    logging.info(self.sql)
+                    self.log.info(
+                        "Selecting rows from Vertica to local file %s...",
+                        tmpfile.name)
+                    self.log.info(self.sql)
 
                     csv_writer = csv.writer(tmpfile, delimiter='\t', encoding='utf-8')
                     for row in cursor.iterate():
@@ -115,21 +114,21 @@ def execute(self, context):
 
                     tmpfile.flush()
                 else:
-                    logging.info("Selecting rows from Vertica...")
-                    logging.info(self.sql)
+                    self.log.info("Selecting rows from Vertica...")
+                    self.log.info(self.sql)
 
                     result = cursor.fetchall()
                     count = len(result)
 
-                logging.info("Selected rows from Vertica " + str(count))
+                self.log.info("Selected rows from Vertica %s", count)
 
         if self.mysql_preoperator:
-            logging.info("Running MySQL preoperator...")
+            self.log.info("Running MySQL preoperator...")
             mysql.run(self.mysql_preoperator)
 
         try:
             if self.bulk_load:
-                logging.info("Bulk inserting rows into MySQL...")
+                self.log.info("Bulk inserting rows into MySQL...")
                 with closing(mysql.get_conn()) as conn:
                     with closing(conn.cursor()) as cursor:
                         cursor.execute("LOAD DATA LOCAL INFILE '%s' INTO "
@@ -140,17 +139,17 @@ def execute(self, context):
                         conn.commit()
                 tmpfile.close()
             else:
-                logging.info("Inserting rows into MySQL...")
+                self.log.info("Inserting rows into MySQL...")
                 mysql.insert_rows(table=self.mysql_table,
                                   rows=result,
                                   target_fields=selected_columns)
-            logging.info("Inserted rows into MySQL " + str(count))
+            self.log.info("Inserted rows into MySQL %s", count)
         except (MySQLdb.Error, MySQLdb.Warning):
-            logging.error("Inserted rows into MySQL 0")
+            self.log.info("Inserted rows into MySQL 0")
             raise
 
         if self.mysql_postoperator:
-            logging.info("Running MySQL postoperator...")
+            self.log.info("Running MySQL postoperator...")
             mysql.run(self.mysql_postoperator)
 
-        logging.info("Done")
+        self.log.info("Done")
diff --git a/airflow/contrib/operators/wasb_delete_blob_operator.py b/airflow/contrib/operators/wasb_delete_blob_operator.py
new file mode 100644
index 0000000000000..4173d7e7a8a5d
--- /dev/null
+++ b/airflow/contrib/operators/wasb_delete_blob_operator.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from airflow.contrib.hooks.wasb_hook import WasbHook
+from airflow.models import BaseOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class WasbDeleteBlobOperator(BaseOperator):
+    """
+    Deletes blob(s) on Azure Blob Storage.
+
+    :param container_name: Name of the container. (templated)
+    :type container_name: str
+    :param blob_name: Name of the blob. (templated)
+    :type blob_name: str
+    :param wasb_conn_id: Reference to the wasb connection.
+    :type wasb_conn_id: str
+    :param check_options: Optional keyword arguments that
+        `WasbHook.check_for_blob()` takes.
+    :param is_prefix: If blob_name is a prefix, delete all files matching prefix.
+    :type is_prefix: bool
+    :param ignore_if_missing: if True, then return success even if the
+        blob does not exist.
+    :type ignore_if_missing: bool
+    """
+
+    template_fields = ('container_name', 'blob_name')
+
+    @apply_defaults
+    def __init__(self, container_name, blob_name,
+                 wasb_conn_id='wasb_default', check_options=None,
+                 is_prefix=False, ignore_if_missing=False,
+                 *args,
+                 **kwargs):
+        super(WasbDeleteBlobOperator, self).__init__(*args, **kwargs)
+        if check_options is None:
+            check_options = {}
+        self.wasb_conn_id = wasb_conn_id
+        self.container_name = container_name
+        self.blob_name = blob_name
+        self.check_options = check_options
+        self.is_prefix = is_prefix
+        self.ignore_if_missing = ignore_if_missing
+
+    def execute(self, context):
+        self.log.info(
+            'Deleting blob: {self.blob_name}\n'
+            'in wasb://{self.container_name}'.format(**locals())
+        )
+        hook = WasbHook(wasb_conn_id=self.wasb_conn_id)
+
+        hook.delete_file(self.container_name, self.blob_name,
+                         self.is_prefix, self.ignore_if_missing,
+                         **self.check_options)
diff --git a/airflow/contrib/operators/winrm_operator.py b/airflow/contrib/operators/winrm_operator.py
index fcd2328d9ad3a..e7738a471fd92 100644
--- a/airflow/contrib/operators/winrm_operator.py
+++ b/airflow/contrib/operators/winrm_operator.py
@@ -17,19 +17,29 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from base64 import b64encode
+import logging
+
+from winrm.exceptions import WinRMOperationTimeoutError
+
+from airflow import configuration
 from airflow.contrib.hooks.winrm_hook import WinRMHook
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
 
+# Hide the following error message in urllib3 when making WinRM connections:
+# requests.packages.urllib3.exceptions.HeaderParsingError: [StartBoundaryNotFoundDefect(),
+#   MultipartInvariantViolationDefect()], unparsed data: ''
+logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.CRITICAL)
 
-class WinRMOperator(BaseOperator):
 
+class WinRMOperator(BaseOperator):
     """
     WinRMOperator to execute commands on given remote host using the winrm_hook.
 
     :param winrm_hook: predefined ssh_hook to use for remote execution
-    :type winrm_hook: :class:`WinRMHook`
+    :type winrm_hook: airflow.contrib.hooks.winrm_hook.WinRMHook
     :param ssh_conn_id: connection id from airflow Connections
     :type ssh_conn_id: str
     :param remote_host: remote host to connect
@@ -41,7 +51,6 @@ class WinRMOperator(BaseOperator):
     :param do_xcom_push: return the stdout which also get set in xcom by airflow platform
     :type do_xcom_push: bool
     """
-
     template_fields = ('command',)
 
     @apply_defaults
@@ -63,48 +72,78 @@ def __init__(self,
         self.do_xcom_push = do_xcom_push
 
     def execute(self, context):
-        try:
-            if self.ssh_conn_id and not self.winrm_hook:
-                self.log.info("hook not found, creating")
-                self.winrm_hook = WinRMHook(ssh_conn_id=self.ssh_conn_id)
+        if self.ssh_conn_id and not self.winrm_hook:
+            self.log.info("Hook not found, creating...")
+            self.winrm_hook = WinRMHook(ssh_conn_id=self.ssh_conn_id)
 
-            if not self.winrm_hook:
-                raise AirflowException("can not operate without ssh_hook or ssh_conn_id")
+        if not self.winrm_hook:
+            raise AirflowException("Cannot operate without winrm_hook or ssh_conn_id.")
 
-            if self.remote_host is not None:
-                self.winrm_hook.remote_host = self.remote_host
+        if self.remote_host is not None:
+            self.winrm_hook.remote_host = self.remote_host
 
-            winrm_client = self.winrm_hook.get_conn()
-            self.log.info("Established WinRM connection")
+        if not self.command:
+            raise AirflowException("No command specified so nothing to execute here.")
 
-            if not self.command:
-                raise AirflowException("no command specified so nothing to execute here.")
+        winrm_client = self.winrm_hook.get_conn()
 
-            self.log.info(
-                "Starting command: '{command}' on remote host: {remotehost}".
-                format(command=self.command, remotehost=self.winrm_hook.remote_host)
+        try:
+            self.log.info("Running command: '%s'...", self.command)
+            command_id = self.winrm_hook.winrm_protocol.run_command(
+                winrm_client,
+                self.command
             )
-            command_id = self.winrm_hook.winrm_protocol. \
-                run_command(winrm_client, self.command)
-            std_out, std_err, status_code = self.winrm_hook.winrm_protocol. \
-                get_command_output(winrm_client, command_id)
-
-            self.log.info("std out: " + std_out.decode())
-            self.log.info("std err: " + std_err.decode())
-            self.log.info("exit code: " + str(status_code))
-            self.log.info("Cleaning up WinRM command")
+
+            # See: https://github.com/diyan/pywinrm/blob/master/winrm/protocol.py
+            stdout_buffer = []
+            stderr_buffer = []
+            command_done = False
+            while not command_done:
+                try:
+                    stdout, stderr, return_code, command_done = \
+                        self.winrm_hook.winrm_protocol._raw_get_command_output(
+                            winrm_client,
+                            command_id
+                        )
+
+                    # Only buffer stdout if we need to so that we minimize memory usage.
+                    if self.do_xcom_push:
+                        stdout_buffer.append(stdout)
+                    stderr_buffer.append(stderr)
+
+                    for line in stdout.decode('utf-8').splitlines():
+                        self.log.info(line)
+                    for line in stderr.decode('utf-8').splitlines():
+                        self.log.warning(line)
+                except WinRMOperationTimeoutError:
+                    # this is an expected error when waiting for a
+                    # long-running process, just silently retry
+                    pass
+
             self.winrm_hook.winrm_protocol.cleanup_command(winrm_client, command_id)
-            self.log.info("Cleaning up WinRM protocol shell")
             self.winrm_hook.winrm_protocol.close_shell(winrm_client)
-            if status_code is 0:
-                return std_out.decode()
-
-            else:
-                error_msg = std_err.decode()
-                raise AirflowException("error running cmd: {0}, error: {1}"
-                                       .format(self.command, error_msg))
 
         except Exception as e:
             raise AirflowException("WinRM operator error: {0}".format(str(e)))
 
+        if return_code == 0:
+            # returning output if do_xcom_push is set
+            if self.do_xcom_push:
+                enable_pickling = configuration.conf.getboolean(
+                    'core', 'enable_xcom_pickling'
+                )
+                if enable_pickling:
+                    return stdout_buffer
+                else:
+                    return b64encode(b''.join(stdout_buffer)).decode('utf-8')
+        else:
+            error_msg = "Error running cmd: {0}, return code: {1}, error: {2}".format(
+                self.command,
+                return_code,
+                b''.join(stderr_buffer).decode('utf-8')
+            )
+            raise AirflowException(error_msg)
+
+        self.log.info("Finished!")
+
         return True
diff --git a/airflow/contrib/plugins/metastore_browser/main.py b/airflow/contrib/plugins/metastore_browser/main.py
index 836e53191ac71..33a8ca5f661ce 100644
--- a/airflow/contrib/plugins/metastore_browser/main.py
+++ b/airflow/contrib/plugins/metastore_browser/main.py
@@ -164,7 +164,7 @@ def ddl(self):
 
 v = MetastoreBrowserView(category="Plugins", name="Hive Metadata Browser")
 
-# Creating a flask blueprint to intergrate the templates and static folder
+# Creating a flask blueprint to integrate the templates and static folder
 bp = Blueprint(
     "metastore_browser", __name__,
     template_folder='templates',
diff --git a/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/table.html b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/table.html
index 3a9d7ca671ad2..cbb7acff6da37 100644
--- a/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/table.html
+++ b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/table.html
@@ -28,7 +28,7 @@ <h4>
     <li role="presentation" class="active"><a href="#home" aria-controls="fields" role="tab" data-toggle="tab">Fields</a></li>
     <li role="presentation"><a href="#data" aria-controls="data" role="tab" data-toggle="tab">Sample Data</a></li>
     <li role="presentation"><a href="#partitions" aria-controls="partitions" role="tab" data-toggle="tab">Partitions</a></li>
-    <li role="presentation"><a href="#attributes" aria-controls="attributes" role="tab" data-toggle="tab">Atributes</a></li>
+    <li role="presentation"><a href="#attributes" aria-controls="attributes" role="tab" data-toggle="tab">Attributes</a></li>
     <li role="presentation"><a href="#parameters" aria-controls="parameters" role="tab" data-toggle="tab">Parameters</a></li>
     <li role="presentation"><a href="#ddl" aria-controls="ddl" role="tab" data-toggle="tab">DDL</a></li>
 </ul>
diff --git a/airflow/contrib/sensors/aws_athena_sensor.py b/airflow/contrib/sensors/aws_athena_sensor.py
new file mode 100644
index 0000000000000..af758b93c1335
--- /dev/null
+++ b/airflow/contrib/sensors/aws_athena_sensor.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from airflow.exceptions import AirflowException
+from airflow.utils.decorators import apply_defaults
+from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+
+
+class AthenaSensor(BaseSensorOperator):
+    """
+    Asks for the state of the Query until it reaches a failure state or success state.
+    If it fails, failing the task.
+
+    :param query_execution_id: query_execution_id to check the state of
+    :type query_execution_id: str
+    :param max_retires: Number of times to poll for query state before
+        returning the current state, defaults to None
+    :type max_retires: int
+    :param aws_conn_id: aws connection to use, defaults to 'aws_default'
+    :type aws_conn_id: str
+    :param sleep_time: Time to wait between two consecutive call to
+        check query status on athena, defaults to 10
+    :type sleep_time: int
+    """
+
+    INTERMEDIATE_STATES = ('QUEUED', 'RUNNING',)
+    FAILURE_STATES = ('FAILED', 'CANCELLED',)
+    SUCCESS_STATES = ('SUCCEEDED',)
+
+    template_fields = ['query_execution_id']
+    template_ext = ()
+    ui_color = '#66c3ff'
+
+    @apply_defaults
+    def __init__(self,
+                 query_execution_id,
+                 max_retires=None,
+                 aws_conn_id='aws_default',
+                 sleep_time=10,
+                 *args, **kwargs):
+        super(AthenaSensor, self).__init__(*args, **kwargs)
+        self.aws_conn_id = aws_conn_id
+        self.query_execution_id = query_execution_id
+        self.hook = None
+        self.sleep_time = sleep_time
+        self.max_retires = max_retires
+
+    def poke(self, context):
+        self.hook = self.get_hook()
+        self.hook.get_conn()
+        state = self.hook.poll_query_status(self.query_execution_id, self.max_retires)
+
+        if state in self.FAILURE_STATES:
+            raise AirflowException('Athena sensor failed')
+
+        if state in self.INTERMEDIATE_STATES:
+            return False
+        return True
+
+    def get_hook(self):
+        return AWSAthenaHook(self.aws_conn_id, self.sleep_time)
diff --git a/airflow/contrib/sensors/aws_glue_catalog_partition_sensor.py b/airflow/contrib/sensors/aws_glue_catalog_partition_sensor.py
new file mode 100644
index 0000000000000..d4c1d3a884e72
--- /dev/null
+++ b/airflow/contrib/sensors/aws_glue_catalog_partition_sensor.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class AwsGlueCatalogPartitionSensor(BaseSensorOperator):
+    """
+    Waits for a partition to show up in AWS Glue Catalog.
+
+    :param table_name: The name of the table to wait for, supports the dot
+        notation (my_database.my_table)
+    :type table_name: str
+    :param expression: The partition clause to wait for. This is passed as
+        is to the AWS Glue Catalog API's get_partitions function,
+        and supports SQL like notation as in ``ds='2015-01-01'
+        AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"``.
+        See https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-partitions.html
+        #aws-glue-api-catalog-partitions-GetPartitions
+    :type expression: str
+    :param aws_conn_id: ID of the Airflow connection where
+        credentials and extra configuration are stored
+    :type aws_conn_id: str
+    :param region_name: Optional aws region name (example: us-east-1). Uses region from connection
+        if not specified.
+    :type region_name: str
+    :param database_name: The name of the catalog database where the partitions reside.
+    :type database_name: str
+    :param poke_interval: Time in seconds that the job should wait in
+        between each tries
+    :type poke_interval: int
+    """
+    template_fields = ('database_name', 'table_name', 'expression',)
+    ui_color = '#C5CAE9'
+
+    @apply_defaults
+    def __init__(self,
+                 table_name, expression="ds='{{ ds }}'",
+                 aws_conn_id='aws_default',
+                 region_name=None,
+                 database_name='default',
+                 poke_interval=60 * 3,
+                 *args,
+                 **kwargs):
+        super(AwsGlueCatalogPartitionSensor, self).__init__(
+            poke_interval=poke_interval, *args, **kwargs)
+        self.aws_conn_id = aws_conn_id
+        self.region_name = region_name
+        self.table_name = table_name
+        self.expression = expression
+        self.database_name = database_name
+
+    def poke(self, context):
+        """
+        Checks for existence of the partition in the AWS Glue Catalog table
+        """
+        if '.' in self.table_name:
+            self.database_name, self.table_name = self.table_name.split('.')
+        self.log.info(
+            'Poking for table %s. %s, expression %s', self.database_name, self.table_name, self.expression
+        )
+
+        return self.get_hook().check_for_partition(
+            self.database_name, self.table_name, self.expression)
+
+    def get_hook(self):
+        """
+        Gets the AwsGlueCatalogHook
+        """
+        if not hasattr(self, 'hook'):
+            from airflow.contrib.hooks.aws_glue_catalog_hook import AwsGlueCatalogHook
+            self.hook = AwsGlueCatalogHook(
+                aws_conn_id=self.aws_conn_id,
+                region_name=self.region_name)
+
+        return self.hook
diff --git a/airflow/contrib/sensors/azure_cosmos_sensor.py b/airflow/contrib/sensors/azure_cosmos_sensor.py
new file mode 100644
index 0000000000000..78b340d4efe53
--- /dev/null
+++ b/airflow/contrib/sensors/azure_cosmos_sensor.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from airflow.contrib.hooks.azure_cosmos_hook import AzureCosmosDBHook
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class AzureCosmosDocumentSensor(BaseSensorOperator):
+    """
+    Checks for the existence of a document which
+    matches the given query in CosmosDB. Example:
+
+    >>> azure_cosmos_sensor = AzureCosmosDocumentSensor(database_name="somedatabase_name",
+    ...                            collection_name="somecollection_name",
+    ...                            document_id="unique-doc-id",
+    ...                            azure_cosmos_conn_id="azure_cosmos_default",
+    ...                            task_id="azure_cosmos_sensor")
+    """
+    template_fields = ('database_name', 'collection_name', 'document_id')
+
+    @apply_defaults
+    def __init__(
+            self,
+            database_name,
+            collection_name,
+            document_id,
+            azure_cosmos_conn_id="azure_cosmos_default",
+            *args,
+            **kwargs):
+        """
+        Create a new AzureCosmosDocumentSensor
+
+        :param database_name: Target CosmosDB database_name.
+        :type database_name: str
+        :param collection_name: Target CosmosDB collection_name.
+        :type collection_name: str
+        :param document_id: The ID of the target document.
+        :type query: str
+        :param azure_cosmos_conn_id: Reference to the Azure CosmosDB connection.
+        :type azure_cosmos_conn_id: str
+        """
+        super(AzureCosmosDocumentSensor, self).__init__(*args, **kwargs)
+        self.azure_cosmos_conn_id = azure_cosmos_conn_id
+        self.database_name = database_name
+        self.collection_name = collection_name
+        self.document_id = document_id
+
+    def poke(self, context):
+        self.log.info("*** Intering poke")
+        hook = AzureCosmosDBHook(self.azure_cosmos_conn_id)
+        return hook.get_document(self.document_id, self.database_name, self.collection_name) is not None
diff --git a/airflow/contrib/sensors/bash_sensor.py b/airflow/contrib/sensors/bash_sensor.py
index 26fbb06c18d81..fec48c5778ef1 100644
--- a/airflow/contrib/sensors/bash_sensor.py
+++ b/airflow/contrib/sensors/bash_sensor.py
@@ -33,7 +33,7 @@ class BashSensor(BaseSensorOperator):
 
     :param bash_command: The command, set of commands or reference to a
         bash script (must be '.sh') to be executed.
-    :type bash_command: string
+    :type bash_command: str
 
     :param env: If env is not None, it must be a mapping that defines the
         environment variables for the new process; these are used instead
@@ -41,7 +41,7 @@ class BashSensor(BaseSensorOperator):
         behavior. (templated)
     :type env: dict
     :param output_encoding: output encoding of bash command.
-    :type output_encoding: string
+    :type output_encoding: str
     """
 
     template_fields = ('bash_command', 'env')
@@ -70,10 +70,7 @@ def poke(self, context):
                 f.flush()
                 fname = f.name
                 script_location = tmp_dir + "/" + fname
-                self.log.info(
-                    "Temporary script location: %s",
-                    script_location
-                )
+                self.log.info("Temporary script location: %s", script_location)
                 self.log.info("Running command: %s", bash_command)
                 sp = Popen(
                     ['bash', fname],
@@ -89,7 +86,6 @@ def poke(self, context):
                     line = line.decode(self.output_encoding).strip()
                     self.log.info(line)
                 sp.wait()
-                self.log.info("Command exited with "
-                              "return code {0}".format(sp.returncode))
+                self.log.info("Command exited with return code %s", sp.returncode)
 
                 return not sp.returncode
diff --git a/airflow/contrib/sensors/bigquery_sensor.py b/airflow/contrib/sensors/bigquery_sensor.py
index 2e496f68897dd..fe8bd2ed6acc9 100644
--- a/airflow/contrib/sensors/bigquery_sensor.py
+++ b/airflow/contrib/sensors/bigquery_sensor.py
@@ -25,22 +25,22 @@ class BigQueryTableSensor(BaseSensorOperator):
     """
     Checks for the existence of a table in Google Bigquery.
 
-        :param project_id: The Google cloud project in which to look for the table.
-            The connection supplied to the hook must provide
-            access to the specified project.
-        :type project_id: string
-        :param dataset_id: The name of the dataset in which to look for the table.
-            storage bucket.
-        :type dataset_id: string
-        :param table_id: The name of the table to check the existence of.
-        :type table_id: string
-        :param bigquery_conn_id: The connection ID to use when connecting to
-            Google BigQuery.
-        :type bigquery_conn_id: string
-        :param delegate_to: The account to impersonate, if any.
-            For this to work, the service account making the request must
-            have domain-wide delegation enabled.
-        :type delegate_to: string
+    :param project_id: The Google cloud project in which to look for the table.
+        The connection supplied to the hook must provide
+        access to the specified project.
+    :type project_id: str
+    :param dataset_id: The name of the dataset in which to look for the table.
+        storage bucket.
+    :type dataset_id: str
+    :param table_id: The name of the table to check the existence of.
+    :type table_id: str
+    :param bigquery_conn_id: The connection ID to use when connecting to
+        Google BigQuery.
+    :type bigquery_conn_id: str
+    :param delegate_to: The account to impersonate, if any.
+        For this to work, the service account making the request must
+        have domain-wide delegation enabled.
+    :type delegate_to: str
     """
     template_fields = ('project_id', 'dataset_id', 'table_id',)
     ui_color = '#f0eee4'
diff --git a/airflow/contrib/sensors/cassandra_sensor.py b/airflow/contrib/sensors/cassandra_record_sensor.py
similarity index 85%
rename from airflow/contrib/sensors/cassandra_sensor.py
rename to airflow/contrib/sensors/cassandra_record_sensor.py
index aef66122e90c2..bbb2f5099030e 100644
--- a/airflow/contrib/sensors/cassandra_sensor.py
+++ b/airflow/contrib/sensors/cassandra_record_sensor.py
@@ -29,9 +29,10 @@ class CassandraRecordSensor(BaseSensorOperator):
     primary keys 'p1' and 'p2' to be populated in keyspace 'k' and table 't',
     instantiate it as follows:
 
-    >>> CassandraRecordSensor(table="k.t", keys={"p1": "v1", "p2": "v2"},
-    ...     cassandra_conn_id="cassandra_default", task_id="cassandra_sensor")
-    <Task(CassandraRecordSensor): cassandra_sensor>
+    >>> cassandra_sensor = CassandraRecordSensor(table="k.t",
+    ...                                          keys={"p1": "v1", "p2": "v2"},
+    ...                                          cassandra_conn_id="cassandra_default",
+    ...                                          task_id="cassandra_sensor")
     """
     template_fields = ('table', 'keys')
 
@@ -42,12 +43,12 @@ def __init__(self, table, keys, cassandra_conn_id, *args, **kwargs):
 
         :param table: Target Cassandra table.
                       Use dot notation to target a specific keyspace.
-        :type table: string
+        :type table: str
         :param keys: The keys and their values to be monitored
         :type keys: dict
         :param cassandra_conn_id: The connection ID to use
                                   when connecting to Cassandra cluster
-        :type cassandra_conn_id: string
+        :type cassandra_conn_id: str
         """
         super(CassandraRecordSensor, self).__init__(*args, **kwargs)
         self.cassandra_conn_id = cassandra_conn_id
diff --git a/airflow/contrib/sensors/cassandra_table_sensor.py b/airflow/contrib/sensors/cassandra_table_sensor.py
new file mode 100644
index 0000000000000..088c82bdd3060
--- /dev/null
+++ b/airflow/contrib/sensors/cassandra_table_sensor.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from airflow.contrib.hooks.cassandra_hook import CassandraHook
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class CassandraTableSensor(BaseSensorOperator):
+    """
+    Checks for the existence of a table in a Cassandra cluster.
+
+    For example, if you want to wait for a table called 't' to be created
+    in a keyspace 'k', instantiate it as follows:
+
+    >>> cassandra_sensor = CassandraTableSensor(table="k.t",
+    ...                                         cassandra_conn_id="cassandra_default",
+    ...                                         task_id="cassandra_sensor")
+    """
+    template_fields = ('table',)
+
+    @apply_defaults
+    def __init__(self, table, cassandra_conn_id, *args, **kwargs):
+        """
+        Create a new CassandraTableSensor
+
+        :param table: Target Cassandra table.
+                      Use dot notation to target a specific keyspace.
+        :type table: str
+        :param cassandra_conn_id: The connection ID to use
+                                  when connecting to Cassandra cluster
+        :type cassandra_conn_id: str
+        """
+        super(CassandraTableSensor, self).__init__(*args, **kwargs)
+        self.cassandra_conn_id = cassandra_conn_id
+        self.table = table
+
+    def poke(self, context):
+        self.log.info('Sensor check existence of table: %s', self.table)
+        hook = CassandraHook(self.cassandra_conn_id)
+        return hook.table_exists(self.table)
diff --git a/airflow/contrib/sensors/datadog_sensor.py b/airflow/contrib/sensors/datadog_sensor.py
index fb455d6cd77eb..df0fb0976fb36 100644
--- a/airflow/contrib/sensors/datadog_sensor.py
+++ b/airflow/contrib/sensors/datadog_sensor.py
@@ -32,7 +32,7 @@ class DatadogSensor(BaseSensorOperator):
     Airflow runs.
 
     :param datadog_conn_id: The connection to datadog, containing metadata for api keys.
-    :param datadog_conn_id: string
+    :param datadog_conn_id: str
     """
     ui_color = '#66c3dd'
 
diff --git a/airflow/contrib/sensors/emr_job_flow_sensor.py b/airflow/contrib/sensors/emr_job_flow_sensor.py
index 31d16a066b241..2c7514b3926f3 100644
--- a/airflow/contrib/sensors/emr_job_flow_sensor.py
+++ b/airflow/contrib/sensors/emr_job_flow_sensor.py
@@ -27,7 +27,7 @@ class EmrJobFlowSensor(EmrBaseSensor):
     If it fails the sensor errors, failing the task.
 
     :param job_flow_id: job_flow_id to check the state of
-    :type job_flow_id: string
+    :type job_flow_id: str
     """
 
     NON_TERMINAL_STATES = ['STARTING', 'BOOTSTRAPPING', 'RUNNING',
@@ -50,5 +50,6 @@ def get_emr_response(self):
         self.log.info('Poking cluster %s', self.job_flow_id)
         return emr.describe_cluster(ClusterId=self.job_flow_id)
 
-    def state_from_response(self, response):
+    @staticmethod
+    def state_from_response(response):
         return response['Cluster']['Status']['State']
diff --git a/airflow/contrib/sensors/emr_step_sensor.py b/airflow/contrib/sensors/emr_step_sensor.py
index 3dddf01bd83a5..8440acbbb314a 100644
--- a/airflow/contrib/sensors/emr_step_sensor.py
+++ b/airflow/contrib/sensors/emr_step_sensor.py
@@ -27,13 +27,13 @@ class EmrStepSensor(EmrBaseSensor):
     If it fails the sensor errors, failing the task.
 
     :param job_flow_id: job_flow_id which contains the step check the state of
-    :type job_flow_id: string
+    :type job_flow_id: str
     :param step_id: step to check the state of
-    :type step_id: string
+    :type step_id: str
     """
 
-    NON_TERMINAL_STATES = ['PENDING', 'RUNNING', 'CONTINUE']
-    FAILED_STATE = ['CANCELLED', 'FAILED']
+    NON_TERMINAL_STATES = ['PENDING', 'RUNNING', 'CONTINUE', 'CANCEL_PENDING']
+    FAILED_STATE = ['CANCELLED', 'FAILED', 'INTERRUPTED']
     template_fields = ['job_flow_id', 'step_id']
     template_ext = ()
 
@@ -53,5 +53,6 @@ def get_emr_response(self):
         self.log.info('Poking step %s on cluster %s', self.step_id, self.job_flow_id)
         return emr.describe_step(ClusterId=self.job_flow_id, StepId=self.step_id)
 
-    def state_from_response(self, response):
+    @staticmethod
+    def state_from_response(response):
         return response['Step']['Status']['State']
diff --git a/airflow/contrib/sensors/file_sensor.py b/airflow/contrib/sensors/file_sensor.py
index 3f7bb24e0837f..570faac86a610 100644
--- a/airflow/contrib/sensors/file_sensor.py
+++ b/airflow/contrib/sensors/file_sensor.py
@@ -35,10 +35,10 @@ class FileSensor(BaseSensorOperator):
 
     :param fs_conn_id: reference to the File (path)
         connection id
-    :type fs_conn_id: string
+    :type fs_conn_id: str
     :param filepath: File or folder name (relative to
         the base path set within the connection)
-    :type fs_conn_id: string
+    :type fs_conn_id: str
     """
     template_fields = ('filepath',)
     ui_color = '#91818a'
diff --git a/airflow/contrib/sensors/ftp_sensor.py b/airflow/contrib/sensors/ftp_sensor.py
index f0a49285d031a..4773b15989299 100644
--- a/airflow/contrib/sensors/ftp_sensor.py
+++ b/airflow/contrib/sensors/ftp_sensor.py
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import ftplib
+import re
 
 from airflow.contrib.hooks.ftp_hook import FTPHook, FTPSHook
 from airflow.sensors.base_sensor_operator import BaseSensorOperator
@@ -26,33 +27,65 @@
 class FTPSensor(BaseSensorOperator):
     """
     Waits for a file or directory to be present on FTP.
-
-    :param path: Remote file or directory path
-    :type path: str
-    :param ftp_conn_id: The connection to run the sensor against
-    :type ftp_conn_id: str
     """
+
     template_fields = ('path',)
 
+    """Errors that are transient in nature, and where action can be retried"""
+    transient_errors = [421, 425, 426, 434, 450, 451, 452]
+
+    error_code_pattern = re.compile(r"([\d]+)")
+
     @apply_defaults
-    def __init__(self, path, ftp_conn_id='ftp_default', *args, **kwargs):
+    def __init__(
+            self,
+            path,
+            ftp_conn_id='ftp_default',
+            fail_on_transient_errors=True,
+            *args,
+            **kwargs):
+        """
+        Create a new FTP sensor
+
+        :param path: Remote file or directory path
+        :type path: str
+        :param fail_on_transient_errors: Fail on all errors,
+            including 4xx transient errors. Default True.
+        :type fail_on_transient_errors: bool
+        :param ftp_conn_id: The connection to run the sensor against
+        :type ftp_conn_id: str
+        """
+
         super(FTPSensor, self).__init__(*args, **kwargs)
 
         self.path = path
         self.ftp_conn_id = ftp_conn_id
+        self.fail_on_transient_errors = fail_on_transient_errors
 
     def _create_hook(self):
         """Return connection hook."""
         return FTPHook(ftp_conn_id=self.ftp_conn_id)
 
+    def _get_error_code(self, e):
+        """Extract error code from ftp exception"""
+        try:
+            matches = self.error_code_pattern.match(str(e))
+            code = int(matches.group(0))
+            return code
+        except ValueError:
+            return e
+
     def poke(self, context):
         with self._create_hook() as hook:
             self.log.info('Poking for %s', self.path)
             try:
                 hook.get_mod_time(self.path)
             except ftplib.error_perm as e:
-                error = str(e).split(None, 1)
-                if error[1] != "Can't check for file existence":
+                self.log.info('Ftp error encountered: %s', str(e))
+                error_code = self._get_error_code(e)
+                if ((error_code != 550) and
+                        (self.fail_on_transient_errors or
+                            (error_code not in self.transient_errors))):
                     raise e
 
                 return False
diff --git a/airflow/contrib/sensors/gcp_transfer_sensor.py b/airflow/contrib/sensors/gcp_transfer_sensor.py
new file mode 100644
index 0000000000000..a6a1c73c39dea
--- /dev/null
+++ b/airflow/contrib/sensors/gcp_transfer_sensor.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import six
+
+from airflow.contrib.hooks.gcp_transfer_hook import GCPTransferServiceHook
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class GCPTransferServiceWaitForJobStatusSensor(BaseSensorOperator):
+    """
+    Waits for at least one operation belonging to the job to have the
+    expected status.
+
+    :param job_name: The name of the transfer job
+    :type job_name: str
+    :param expected_statuses: The expected state of the operation.
+        See:
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations#Status
+    :type expected_statuses: set[str] or string
+    :param project_id: (Optional) the ID of the project that owns the Transfer
+        Job. If set to None or missing, the default project_id from the GCP
+        connection is used.
+    :type project_id: str
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud
+        Platform.
+    :type gcp_conn_id: str
+    """
+
+    # [START gcp_transfer_job_sensor_template_fields]
+    template_fields = ('job_name',)
+    # [END gcp_transfer_job_sensor_template_fields]
+
+    @apply_defaults
+    def __init__(
+        self,
+        job_name,
+        expected_statuses,
+        project_id=None,
+        gcp_conn_id='google_cloud_default',
+        *args,
+        **kwargs
+    ):
+        super(GCPTransferServiceWaitForJobStatusSensor, self).__init__(*args, **kwargs)
+        self.job_name = job_name
+        self.expected_statuses = (
+            {expected_statuses} if isinstance(expected_statuses, six.string_types) else expected_statuses
+        )
+        self.project_id = project_id
+        self.gcp_cloud_conn_id = gcp_conn_id
+
+    def poke(self, context):
+        hook = GCPTransferServiceHook(gcp_conn_id=self.gcp_cloud_conn_id)
+        operations = hook.list_transfer_operations(
+            filter={'project_id': self.project_id, 'job_names': [self.job_name]}
+        )
+
+        check = GCPTransferServiceHook.operations_contain_expected_statuses(
+            operations=operations, expected_statuses=self.expected_statuses
+        )
+        if check:
+            self.xcom_push(key="sensed_operations", value=operations, context=context)
+
+        return check
diff --git a/airflow/contrib/sensors/gcs_sensor.py b/airflow/contrib/sensors/gcs_sensor.py
index 23cd760a41f19..5a4f73e61e822 100644
--- a/airflow/contrib/sensors/gcs_sensor.py
+++ b/airflow/contrib/sensors/gcs_sensor.py
@@ -27,17 +27,17 @@ class GoogleCloudStorageObjectSensor(BaseSensorOperator):
     Create a new GoogleCloudStorageObjectSensor.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to check in the Google cloud
             storage bucket.
-        :type object: string
+        :type object: str
         :param google_cloud_storage_conn_id: The connection ID to use when
             connecting to Google cloud storage.
-        :type google_cloud_storage_conn_id: string
+        :type google_cloud_storage_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request must have
             domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
     """
     template_fields = ('bucket', 'object')
     ui_color = '#f0eee4'
@@ -79,21 +79,21 @@ class GoogleCloudStorageObjectUpdatedSensor(BaseSensorOperator):
     Create a new GoogleCloudStorageObjectUpdatedSensor.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param object: The name of the object to download in the Google cloud
             storage bucket.
-        :type object: string
+        :type object: str
         :param ts_func: Callback for defining the update condition. The default callback
             returns execution_date + schedule_interval. The callback takes the context
             as parameter.
         :type ts_func: function
         :param google_cloud_storage_conn_id: The connection ID to use when
             connecting to Google cloud storage.
-        :type google_cloud_storage_conn_id: string
+        :type google_cloud_storage_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request must have domain-wide
             delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
     """
     template_fields = ('bucket', 'object')
     template_ext = ('.sql',)
@@ -129,17 +129,17 @@ class GoogleCloudStoragePrefixSensor(BaseSensorOperator):
     Create a new GoogleCloudStorageObjectSensor.
 
         :param bucket: The Google cloud storage bucket where the object is.
-        :type bucket: string
+        :type bucket: str
         :param prefix: The name of the prefix to check in the Google cloud
             storage bucket.
-        :type prefix: string
+        :type prefix: str
         :param google_cloud_storage_conn_id: The connection ID to use when
             connecting to Google cloud storage.
-        :type google_cloud_storage_conn_id: string
+        :type google_cloud_storage_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request must have
             domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
     """
     template_fields = ('bucket', 'prefix')
     ui_color = '#f0eee4'
diff --git a/airflow/contrib/sensors/hdfs_sensor.py b/airflow/contrib/sensors/hdfs_sensor.py
index 832b81b8e5f25..fe9e5cfc6f4df 100644
--- a/airflow/contrib/sensors/hdfs_sensor.py
+++ b/airflow/contrib/sensors/hdfs_sensor.py
@@ -35,9 +35,7 @@ def poke(self, context):
         """
         sb = self.hook(self.hdfs_conn_id).get_conn()
         self.log.info(
-            'Poking for {self.filepath} to be a directory '
-            'with files matching {self.regex.pattern}'.
-            format(**locals())
+            'Poking for %s to be a directory with files matching %s', self.filepath, self.regex.pattern
         )
         result = [f for f in sb.ls([self.filepath], include_toplevel=False) if
                   f['file_type'] == 'f' and
@@ -68,11 +66,9 @@ def poke(self, context):
                                              self.ignore_copying)
         result = self.filter_for_filesize(result, self.file_size)
         if self.be_empty:
-            self.log.info('Poking for filepath {self.filepath} to a empty directory'
-                          .format(**locals()))
+            self.log.info('Poking for filepath %s to a empty directory', self.filepath)
             return len(result) == 1 and result[0]['path'] == self.filepath
         else:
-            self.log.info('Poking for filepath {self.filepath} to a non empty directory'
-                          .format(**locals()))
+            self.log.info('Poking for filepath %s to a non empty directory', self.filepath)
             result.pop(0)
             return bool(result) and result[0]['file_type'] == 'f'
diff --git a/airflow/contrib/sensors/imap_attachment_sensor.py b/airflow/contrib/sensors/imap_attachment_sensor.py
new file mode 100644
index 0000000000000..c0eb9b6cd22c3
--- /dev/null
+++ b/airflow/contrib/sensors/imap_attachment_sensor.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.imap_hook import ImapHook
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class ImapAttachmentSensor(BaseSensorOperator):
+    """
+    Waits for a specific attachment on a mail server.
+
+    :param attachment_name: The name of the attachment that will be checked.
+    :type attachment_name: str
+    :param check_regex: If set to True the attachment's name will be parsed as regular expression.
+                        Through this you can get a broader set of attachments
+                        that it will look for than just only the equality of the attachment name.
+                        The default value is False.
+    :type check_regex: bool
+    :param mail_folder: The mail folder in where to search for the attachment.
+                        The default value is 'INBOX'.
+    :type mail_folder: str
+    :param conn_id: The connection to run the sensor against.
+                    The default value is 'imap_default'.
+    :type conn_id: str
+    """
+    template_fields = ('attachment_name',)
+
+    @apply_defaults
+    def __init__(self,
+                 attachment_name,
+                 mail_folder='INBOX',
+                 check_regex=False,
+                 conn_id='imap_default',
+                 *args,
+                 **kwargs):
+        super(ImapAttachmentSensor, self).__init__(*args, **kwargs)
+
+        self.attachment_name = attachment_name
+        self.mail_folder = mail_folder
+        self.check_regex = check_regex
+        self.conn_id = conn_id
+
+    def poke(self, context):
+        """
+        Pokes for a mail attachment on the mail server.
+
+        :param context: The context that is being provided when poking.
+        :type context: dict
+        :return: True if attachment with the given name is present and False if not.
+        :rtype: bool
+        """
+        self.log.info('Poking for %s', self.attachment_name)
+
+        with ImapHook(imap_conn_id=self.conn_id) as imap_hook:
+            return imap_hook.has_mail_attachment(
+                name=self.attachment_name,
+                mail_folder=self.mail_folder,
+                check_regex=self.check_regex
+            )
diff --git a/airflow/contrib/sensors/mongo_sensor.py b/airflow/contrib/sensors/mongo_sensor.py
new file mode 100644
index 0000000000000..8fd32a0775feb
--- /dev/null
+++ b/airflow/contrib/sensors/mongo_sensor.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from airflow.contrib.hooks.mongo_hook import MongoHook
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class MongoSensor(BaseSensorOperator):
+    """
+    Checks for the existence of a document which
+    matches the given query in MongoDB. Example:
+
+    >>> mongo_sensor = MongoSensor(collection="coll",
+    ...                            query={"key": "value"},
+    ...                            mongo_conn_id="mongo_default",
+    ...                            task_id="mongo_sensor")
+    """
+    template_fields = ('collection', 'query')
+
+    @apply_defaults
+    def __init__(self, collection, query, mongo_conn_id="mongo_default", *args, **kwargs):
+        """
+        Create a new MongoSensor
+
+        :param collection: Target MongoDB collection.
+        :type collection: str
+        :param query: The query to find the target document.
+        :type query: dict
+        :param mongo_conn_id: The connection ID to use
+                              when connecting to MongoDB.
+        :type mongo_conn_id: str
+        """
+        super(MongoSensor, self).__init__(*args, **kwargs)
+        self.mongo_conn_id = mongo_conn_id
+        self.collection = collection
+        self.query = query
+
+    def poke(self, context):
+        self.log.info("Sensor check existence of the document "
+                      "that matches the following query: %s", self.query)
+        hook = MongoHook(self.mongo_conn_id)
+        return hook.find(self.collection, self.query, find_one=True) is not None
diff --git a/airflow/contrib/sensors/pubsub_sensor.py b/airflow/contrib/sensors/pubsub_sensor.py
index 7d1721570dea4..8104fb56997b2 100644
--- a/airflow/contrib/sensors/pubsub_sensor.py
+++ b/airflow/contrib/sensors/pubsub_sensor.py
@@ -54,10 +54,10 @@ def __init__(
             **kwargs):
         """
         :param project: the GCP project ID for the subscription (templated)
-        :type project: string
+        :type project: str
         :param subscription: the Pub/Sub subscription name. Do not include the
             full subscription path.
-        :type subscription: string
+        :type subscription: str
         :param max_messages: The maximum number of messages to retrieve per
             PubSub pull request
         :type max_messages: int
@@ -69,11 +69,11 @@ def __init__(
         :type ack_messages: bool
         :param gcp_conn_id: The connection ID to use connecting to
             Google Cloud Platform.
-        :type gcp_conn_id: string
+        :type gcp_conn_id: str
         :param delegate_to: The account to impersonate, if any.
             For this to work, the service account making the request
             must have domain-wide delegation enabled.
-        :type delegate_to: string
+        :type delegate_to: str
         """
         super(PubSubPullSensor, self).__init__(*args, **kwargs)
 
diff --git a/airflow/contrib/sensors/python_sensor.py b/airflow/contrib/sensors/python_sensor.py
new file mode 100644
index 0000000000000..7c4579a891fbb
--- /dev/null
+++ b/airflow/contrib/sensors/python_sensor.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+
+
+class PythonSensor(BaseSensorOperator):
+    """
+    Waits for a Python callable to return True.
+
+    User could put input argument in templates_dict
+    e.g ``templates_dict = {'start_ds': 1970}``
+    and access the argument by calling ``kwargs['templates_dict']['start_ds']``
+    in the the callable
+
+    :param python_callable: A reference to an object that is callable
+    :type python_callable: python callable
+    :param op_kwargs: a dictionary of keyword arguments that will get unpacked
+        in your function
+    :type op_kwargs: dict
+    :param op_args: a list of positional arguments that will get unpacked when
+        calling your callable
+    :type op_args: list
+    :param provide_context: if set to true, Airflow will pass a set of
+        keyword arguments that can be used in your function. This set of
+        kwargs correspond exactly to what you can use in your jinja
+        templates. For this to work, you need to define `**kwargs` in your
+        function header.
+    :type provide_context: bool
+    :param templates_dict: a dictionary where the values are templates that
+        will get templated by the Airflow engine sometime between
+        ``__init__`` and ``execute`` takes place and are made available
+        in your callable's context after the template has been applied.
+    :type templates_dict: dict of str
+    """
+
+    template_fields = ('templates_dict',)
+
+    @apply_defaults
+    def __init__(
+            self,
+            python_callable,
+            op_args=None,
+            op_kwargs=None,
+            provide_context=False,
+            templates_dict=None,
+            *args, **kwargs):
+        super(PythonSensor, self).__init__(*args, **kwargs)
+        self.python_callable = python_callable
+        self.op_args = op_args or []
+        self.op_kwargs = op_kwargs or {}
+        self.provide_context = provide_context
+        self.templates_dict = templates_dict
+
+    def poke(self, context):
+        if self.provide_context:
+            context.update(self.op_kwargs)
+            context['templates_dict'] = self.templates_dict
+            self.op_kwargs = context
+
+        self.log.info("Poking callable: %s", str(self.python_callable))
+        return_value = self.python_callable(*self.op_args, **self.op_kwargs)
+        return bool(return_value)
diff --git a/airflow/contrib/sensors/qubole_sensor.py b/airflow/contrib/sensors/qubole_sensor.py
index d67fa8557e7ad..08e4290570fe0 100644
--- a/airflow/contrib/sensors/qubole_sensor.py
+++ b/airflow/contrib/sensors/qubole_sensor.py
@@ -29,14 +29,6 @@
 class QuboleSensor(BaseSensorOperator):
     """
     Base class for all Qubole Sensors
-
-    :param qubole_conn_id: The qubole connection to run the sensor against
-    :type qubole_conn_id: string
-    :param data: a JSON object containing payload, whose presence needs to be checked
-    :type data: a JSON object
-
-    .. note:: Both ``data`` and ``qubole_conn_id`` fields are template-supported. You can
-    also use ``.txt`` files for template driven use cases.
     """
 
     template_fields = ('data', 'qubole_conn_id')
@@ -75,6 +67,22 @@ def poke(self, context):
 
 
 class QuboleFileSensor(QuboleSensor):
+    """
+    Wait for a file or folder to be present in cloud storage
+    and check for its presence via QDS APIs
+
+    :param qubole_conn_id: Connection id which consists of qds auth_token
+    :type qubole_conn_id: str
+    :param data: a JSON object containing payload, whose presence needs to be checked
+        Check this `example <https://github.com/apache/airflow/blob/master\
+        /airflow/contrib/example_dags/example_qubole_sensor.py>`_ for sample payload
+        structure.
+    :type data: a JSON object
+
+    .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can
+        also use ``.txt`` files for template-driven use cases.
+    """
+
     @apply_defaults
     def __init__(self, *args, **kwargs):
         self.sensor_class = FileSensor
@@ -82,6 +90,22 @@ def __init__(self, *args, **kwargs):
 
 
 class QubolePartitionSensor(QuboleSensor):
+    """
+    Wait for a Hive partition to show up in QHS (Qubole Hive Service)
+    and check for its presence via QDS APIs
+
+    :param qubole_conn_id: Connection id which consists of qds auth_token
+    :type qubole_conn_id: str
+    :param data: a JSON object containing payload, whose presence needs to be checked.
+        Check this `example <https://github.com/apache/airflow/blob/master\
+        /airflow/contrib/example_dags/example_qubole_sensor.py>`_ for sample payload
+        structure.
+    :type data: a JSON object
+
+    .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can
+        also use ``.txt`` files for template-driven use cases.
+    """
+
     @apply_defaults
     def __init__(self, *args, **kwargs):
         self.sensor_class = PartitionSensor
diff --git a/airflow/contrib/sensors/redis_key_sensor.py b/airflow/contrib/sensors/redis_key_sensor.py
index baf3e161f5365..4c0ac68840b50 100644
--- a/airflow/contrib/sensors/redis_key_sensor.py
+++ b/airflow/contrib/sensors/redis_key_sensor.py
@@ -23,25 +23,17 @@
 
 class RedisKeySensor(BaseSensorOperator):
     """
-    Checks for the existence of a key in a Redis database
+    Checks for the existence of a key in a Redis
     """
     template_fields = ('key',)
     ui_color = '#f0eee4'
 
     @apply_defaults
     def __init__(self, key, redis_conn_id, *args, **kwargs):
-        """
-        Create a new RedisKeySensor
-
-        :param key: The key to be monitored
-        :type key: string
-        :param redis_conn_id: The connection ID to use when connecting to Redis DB.
-        :type redis_conn_id: string
-        """
         super(RedisKeySensor, self).__init__(*args, **kwargs)
         self.redis_conn_id = redis_conn_id
         self.key = key
 
     def poke(self, context):
-        self.log.info('Sensor check existence of key: %s', self.key)
-        return RedisHook(self.redis_conn_id).key_exists(self.key)
+        self.log.info('Sensor checks for existence of key: %s', self.key)
+        return RedisHook(self.redis_conn_id).get_conn().exists(self.key)
diff --git a/airflow/contrib/sensors/redis_pub_sub_sensor.py b/airflow/contrib/sensors/redis_pub_sub_sensor.py
new file mode 100644
index 0000000000000..474a8d21d61ef
--- /dev/null
+++ b/airflow/contrib/sensors/redis_pub_sub_sensor.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.contrib.hooks.redis_hook import RedisHook
+
+
+class RedisPubSubSensor(BaseSensorOperator):
+
+    """
+    Redis sensor for reading a message from pub sub channels
+    """
+    template_fields = ('channels',)
+    ui_color = '#f0eee4'
+
+    @apply_defaults
+    def __init__(self, channels, redis_conn_id, *args, **kwargs):
+        """
+        Create a new RedisPubSubSensor and subscribe to the channels
+
+        :param channels: The channels to be subscribed to (templated)
+        :type channels: str or list of str
+        :param redis_conn_id: the redis connection id
+        :type redis_conn_id: str
+        """
+
+        super(RedisPubSubSensor, self).__init__(*args, **kwargs)
+        self.channels = channels
+        self.redis_conn_id = redis_conn_id
+        self.pubsub = RedisHook(redis_conn_id=self.redis_conn_id).get_conn().pubsub()
+        self.pubsub.subscribe(self.channels)
+
+    def poke(self, context):
+        """
+        Check for message on subscribed channels and write to xcom the message with key ``message``
+
+        An example of message ``{'type': 'message', 'pattern': None, 'channel': b'test', 'data': b'hello'}``
+
+        :param context: the context object
+        :type context: dict
+        :return: ``True`` if message (with type 'message') is available or ``False`` if not
+        """
+        self.log.info('RedisPubSubSensor checking for message on channels: %s', self.channels)
+
+        message = self.pubsub.get_message()
+        self.log.info('Message %s from channel %s', message, self.channels)
+
+        # Process only message types
+        if message and message['type'] == 'message':
+
+            context['ti'].xcom_push(key='message', value=message)
+            self.pubsub.unsubscribe(self.channels)
+
+            return True
+
+        return False
diff --git a/airflow/contrib/sensors/sagemaker_base_sensor.py b/airflow/contrib/sensors/sagemaker_base_sensor.py
new file mode 100644
index 0000000000000..10dd6b2357a66
--- /dev/null
+++ b/airflow/contrib/sensors/sagemaker_base_sensor.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils.decorators import apply_defaults
+from airflow.exceptions import AirflowException
+
+
+class SageMakerBaseSensor(BaseSensorOperator):
+    """
+    Contains general sensor behavior for SageMaker.
+    Subclasses should implement get_sagemaker_response()
+    and state_from_response() methods.
+    Subclasses should also implement NON_TERMINAL_STATES and FAILED_STATE methods.
+    """
+    ui_color = '#ededed'
+
+    @apply_defaults
+    def __init__(
+            self,
+            aws_conn_id='aws_default',
+            *args, **kwargs):
+        super(SageMakerBaseSensor, self).__init__(*args, **kwargs)
+        self.aws_conn_id = aws_conn_id
+
+    def poke(self, context):
+        response = self.get_sagemaker_response()
+
+        if not response['ResponseMetadata']['HTTPStatusCode'] == 200:
+            self.log.info('Bad HTTP response: %s', response)
+            return False
+
+        state = self.state_from_response(response)
+
+        self.log.info('Job currently %s', state)
+
+        if state in self.non_terminal_states():
+            return False
+
+        if state in self.failed_states():
+            failed_reason = self.get_failed_reason_from_response(response)
+            raise AirflowException('Sagemaker job failed for the following reason: %s'
+                                   % failed_reason)
+        return True
+
+    def non_terminal_states(self):
+        raise NotImplementedError('Please implement non_terminal_states() in subclass')
+
+    def failed_states(self):
+        raise NotImplementedError('Please implement failed_states() in subclass')
+
+    def get_sagemaker_response(self):
+        raise NotImplementedError('Please implement get_sagemaker_response() in subclass')
+
+    def get_failed_reason_from_response(self, response):
+        return 'Unknown'
+
+    def state_from_response(self, response):
+        raise NotImplementedError('Please implement state_from_response() in subclass')
diff --git a/airflow/contrib/sensors/sagemaker_endpoint_sensor.py b/airflow/contrib/sensors/sagemaker_endpoint_sensor.py
new file mode 100644
index 0000000000000..ceed9c10097d9
--- /dev/null
+++ b/airflow/contrib/sensors/sagemaker_endpoint_sensor.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.sensors.sagemaker_base_sensor import SageMakerBaseSensor
+from airflow.utils.decorators import apply_defaults
+
+
+class SageMakerEndpointSensor(SageMakerBaseSensor):
+    """
+    Asks for the state of the endpoint state until it reaches a terminal state.
+    If it fails the sensor errors, the task fails.
+
+    :param job_name: job_name of the endpoint instance to check the state of
+    :type job_name: str
+    """
+
+    template_fields = ['endpoint_name']
+    template_ext = ()
+
+    @apply_defaults
+    def __init__(self,
+                 endpoint_name,
+                 *args,
+                 **kwargs):
+        super(SageMakerEndpointSensor, self).__init__(*args, **kwargs)
+        self.endpoint_name = endpoint_name
+
+    def non_terminal_states(self):
+        return SageMakerHook.endpoint_non_terminal_states
+
+    def failed_states(self):
+        return SageMakerHook.failed_states
+
+    def get_sagemaker_response(self):
+        sagemaker = SageMakerHook(aws_conn_id=self.aws_conn_id)
+
+        self.log.info('Poking Sagemaker Endpoint %s', self.endpoint_name)
+        return sagemaker.describe_endpoint(self.endpoint_name)
+
+    def get_failed_reason_from_response(self, response):
+        return response['FailureReason']
+
+    def state_from_response(self, response):
+        return response['EndpointStatus']
diff --git a/airflow/contrib/sensors/sagemaker_training_sensor.py b/airflow/contrib/sensors/sagemaker_training_sensor.py
new file mode 100644
index 0000000000000..d550a6c434bfb
--- /dev/null
+++ b/airflow/contrib/sensors/sagemaker_training_sensor.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import time
+
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook, LogState
+from airflow.contrib.sensors.sagemaker_base_sensor import SageMakerBaseSensor
+from airflow.utils.decorators import apply_defaults
+
+
+class SageMakerTrainingSensor(SageMakerBaseSensor):
+    """
+    Asks for the state of the training state until it reaches a terminal state.
+    If it fails the sensor errors, failing the task.
+
+    :param job_name: name of the SageMaker training job to check the state of
+    :type job_name: str
+    :param print_log: if the operator should print the cloudwatch log
+    :type print_log: bool
+    """
+
+    template_fields = ['job_name']
+    template_ext = ()
+
+    @apply_defaults
+    def __init__(self,
+                 job_name,
+                 print_log=True,
+                 *args,
+                 **kwargs):
+        super(SageMakerTrainingSensor, self).__init__(*args, **kwargs)
+        self.job_name = job_name
+        self.print_log = print_log
+        self.positions = {}
+        self.stream_names = []
+        self.instance_count = None
+        self.state = None
+        self.last_description = None
+        self.last_describe_job_call = None
+        self.log_resource_inited = False
+
+    def init_log_resource(self, hook):
+        description = hook.describe_training_job(self.job_name)
+        self.instance_count = description['ResourceConfig']['InstanceCount']
+
+        status = description['TrainingJobStatus']
+        job_already_completed = status not in self.non_terminal_states()
+        self.state = LogState.TAILING if not job_already_completed else LogState.COMPLETE
+        self.last_description = description
+        self.last_describe_job_call = time.time()
+        self.log_resource_inited = True
+
+    def non_terminal_states(self):
+        return SageMakerHook.non_terminal_states
+
+    def failed_states(self):
+        return SageMakerHook.failed_states
+
+    def get_sagemaker_response(self):
+        sagemaker_hook = SageMakerHook(aws_conn_id=self.aws_conn_id)
+        if self.print_log:
+            if not self.log_resource_inited:
+                self.init_log_resource(sagemaker_hook)
+            self.state, self.last_description, self.last_describe_job_call = \
+                sagemaker_hook.describe_training_job_with_log(self.job_name,
+                                                              self.positions, self.stream_names,
+                                                              self.instance_count, self.state,
+                                                              self.last_description,
+                                                              self.last_describe_job_call)
+        else:
+            self.last_description = sagemaker_hook.describe_training_job(self.job_name)
+
+        status = self.state_from_response(self.last_description)
+        if status not in self.non_terminal_states() and status not in self.failed_states():
+            billable_time = \
+                (self.last_description['TrainingEndTime'] - self.last_description['TrainingStartTime']) * \
+                self.last_description['ResourceConfig']['InstanceCount']
+            self.log.info('Billable seconds: %s', int(billable_time.total_seconds()) + 1)
+
+        return self.last_description
+
+    def get_failed_reason_from_response(self, response):
+        return response['FailureReason']
+
+    def state_from_response(self, response):
+        return response['TrainingJobStatus']
diff --git a/airflow/contrib/sensors/sagemaker_transform_sensor.py b/airflow/contrib/sensors/sagemaker_transform_sensor.py
new file mode 100644
index 0000000000000..f64724bde9b24
--- /dev/null
+++ b/airflow/contrib/sensors/sagemaker_transform_sensor.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.sensors.sagemaker_base_sensor import SageMakerBaseSensor
+from airflow.utils.decorators import apply_defaults
+
+
+class SageMakerTransformSensor(SageMakerBaseSensor):
+    """
+    Asks for the state of the transform state until it reaches a terminal state.
+    The sensor will error if the job errors, throwing a AirflowException
+    containing the failure reason.
+
+    :param job_name: job_name of the transform job instance to check the state of
+    :type job_name: string
+    """
+
+    template_fields = ['job_name']
+    template_ext = ()
+
+    @apply_defaults
+    def __init__(self,
+                 job_name,
+                 *args,
+                 **kwargs):
+        super(SageMakerTransformSensor, self).__init__(*args, **kwargs)
+        self.job_name = job_name
+
+    def non_terminal_states(self):
+        return SageMakerHook.non_terminal_states
+
+    def failed_states(self):
+        return SageMakerHook.failed_states
+
+    def get_sagemaker_response(self):
+        sagemaker = SageMakerHook(aws_conn_id=self.aws_conn_id)
+
+        self.log.info('Poking Sagemaker Transform Job %s', self.job_name)
+        return sagemaker.describe_transform_job(self.job_name)
+
+    def get_failed_reason_from_response(self, response):
+        return response['FailureReason']
+
+    def state_from_response(self, response):
+        return response['TransformJobStatus']
diff --git a/airflow/contrib/sensors/sagemaker_tuning_sensor.py b/airflow/contrib/sensors/sagemaker_tuning_sensor.py
new file mode 100644
index 0000000000000..8c835216d6bb3
--- /dev/null
+++ b/airflow/contrib/sensors/sagemaker_tuning_sensor.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.sensors.sagemaker_base_sensor import SageMakerBaseSensor
+from airflow.utils.decorators import apply_defaults
+
+
+class SageMakerTuningSensor(SageMakerBaseSensor):
+    """
+    Asks for the state of the tuning state until it reaches a terminal state.
+    The sensor will error if the job errors, throwing a AirflowException
+    containing the failure reason.
+
+    :param job_name: job_name of the tuning instance to check the state of
+    :type job_name: str
+    """
+
+    template_fields = ['job_name']
+    template_ext = ()
+
+    @apply_defaults
+    def __init__(self,
+                 job_name,
+                 *args,
+                 **kwargs):
+        super(SageMakerTuningSensor, self).__init__(*args, **kwargs)
+        self.job_name = job_name
+
+    def non_terminal_states(self):
+        return SageMakerHook.non_terminal_states
+
+    def failed_states(self):
+        return SageMakerHook.failed_states
+
+    def get_sagemaker_response(self):
+        sagemaker = SageMakerHook(aws_conn_id=self.aws_conn_id)
+
+        self.log.info('Poking Sagemaker Tuning Job %s', self.job_name)
+        return sagemaker.describe_tuning_job(self.job_name)
+
+    def get_failed_reason_from_response(self, response):
+        return response['FailureReason']
+
+    def state_from_response(self, response):
+        return response['HyperParameterTuningJobStatus']
diff --git a/airflow/contrib/sensors/sftp_sensor.py b/airflow/contrib/sensors/sftp_sensor.py
index 51ad11ddbf68b..0b61c801514cc 100644
--- a/airflow/contrib/sensors/sftp_sensor.py
+++ b/airflow/contrib/sensors/sftp_sensor.py
@@ -17,7 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import logging
 from paramiko import SFTP_NO_SUCH_FILE
 from airflow.contrib.hooks.sftp_hook import SFTPHook
 from airflow.operators.sensors import BaseSensorOperator
@@ -27,6 +26,7 @@
 class SFTPSensor(BaseSensorOperator):
     """
     Waits for a file or directory to be present on SFTP.
+
     :param path: Remote file or directory path
     :type path: str
     :param sftp_conn_id: The connection to run the sensor against
@@ -41,7 +41,7 @@ def __init__(self, path, sftp_conn_id='sftp_default', *args, **kwargs):
         self.hook = SFTPHook(sftp_conn_id)
 
     def poke(self, context):
-        logging.info('Poking for %s', self.path)
+        self.log.info('Poking for %s', self.path)
         try:
             self.hook.get_mod_time(self.path)
         except IOError as e:
diff --git a/airflow/contrib/sensors/wasb_sensor.py b/airflow/contrib/sensors/wasb_sensor.py
index ec6a63bf3d633..8b32d3fa221d9 100644
--- a/airflow/contrib/sensors/wasb_sensor.py
+++ b/airflow/contrib/sensors/wasb_sensor.py
@@ -90,10 +90,7 @@ def __init__(self, container_name, prefix, wasb_conn_id='wasb_default',
         self.check_options = check_options
 
     def poke(self, context):
-        self.log.info(
-            'Poking for prefix: {self.prefix}\n'
-            'in wasb://{self.container_name}'.format(**locals())
-        )
+        self.log.info('Poking for prefix: %s in wasb://%s', self.prefix, self.container_name)
         hook = WasbHook(wasb_conn_id=self.wasb_conn_id)
         return hook.check_for_prefix(self.container_name, self.prefix,
                                      **self.check_options)
diff --git a/airflow/contrib/sensors/weekday_sensor.py b/airflow/contrib/sensors/weekday_sensor.py
new file mode 100644
index 0000000000000..8d99e11fb459f
--- /dev/null
+++ b/airflow/contrib/sensors/weekday_sensor.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import six
+from airflow.contrib.utils.weekday import WeekDay
+from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.utils import timezone
+from airflow.utils.decorators import apply_defaults
+
+
+class DayOfWeekSensor(BaseSensorOperator):
+    """
+    Waits until the first specified day of the week. For example, if the execution
+    day of the task is '2018-12-22' (Saturday) and you pass 'FRIDAY', the task will wait
+    until next Friday.
+
+    **Example** (with single day): ::
+
+        weekend_check = DayOfWeekSensor(
+            task_id='weekend_check',
+            week_day='Saturday',
+            use_task_execution_day=True,
+            dag=dag)
+
+    **Example** (with multiple day using set): ::
+
+        weekend_check = DayOfWeekSensor(
+            task_id='weekend_check',
+            week_day={'Saturday', 'Sunday'},
+            use_task_execution_day=True,
+            dag=dag)
+
+    **Example** (with :class:`~airflow.contrib.utils.weekday.WeekDay` enum): ::
+
+        # import WeekDay Enum
+        from airflow.contrib.utils.weekday import WeekDay
+
+        weekend_check = DayOfWeekSensor(
+            task_id='weekend_check',
+            week_day={WeekDay.SATURDAY, WeekDay.SUNDAY},
+            use_task_execution_day=True,
+            dag=dag)
+
+    :param week_day: Day of the week to check (full name). Optionally, a set
+        of days can also be provided using a set.
+        Example values:
+
+            * ``"MONDAY"``,
+            * ``{"Saturday", "Sunday"}``
+            * ``{WeekDay.TUESDAY}``
+            * ``{WeekDay.SATURDAY, WeekDay.SUNDAY}``
+
+    :type week_day: set or str or airflow.contrib.utils.weekday.WeekDay
+    :param use_task_execution_day: If ``True``, uses task's execution day to compare
+        with week_day. Execution Date is Useful for backfilling.
+        If ``False``, uses system's day of the week. Useful when you
+        don't want to run anything on weekdays on the system.
+    :type use_task_execution_day: bool
+    """
+
+    @apply_defaults
+    def __init__(self, week_day,
+                 use_task_execution_day=False,
+                 *args, **kwargs):
+        super(DayOfWeekSensor, self).__init__(*args, **kwargs)
+        self.week_day = week_day
+        self.use_task_execution_day = use_task_execution_day
+        if isinstance(self.week_day, six.string_types):
+            self._week_day_num = {WeekDay.get_weekday_number(week_day_str=self.week_day)}
+        elif isinstance(self.week_day, WeekDay):
+            self._week_day_num = {self.week_day}
+        elif isinstance(self.week_day, set):
+            if all(isinstance(day, six.string_types) for day in self.week_day):
+                self._week_day_num = {WeekDay.get_weekday_number(day) for day in week_day}
+            elif all(isinstance(day, WeekDay) for day in self.week_day):
+                self._week_day_num = self.week_day
+        else:
+            raise TypeError(
+                'Unsupported Type for week_day parameter: {}. It should be one of str'
+                ', set or Weekday enum type'.format(type(week_day)))
+
+    def poke(self, context):
+        self.log.info('Poking until weekday is in %s, Today is %s',
+                      self.week_day,
+                      WeekDay(timezone.utcnow().isoweekday()).name)
+        if self.use_task_execution_day:
+            return context['execution_date'].isoweekday() in self._week_day_num
+        else:
+            return timezone.utcnow().isoweekday() in self._week_day_num
diff --git a/airflow/contrib/task_runner/cgroup_task_runner.py b/airflow/contrib/task_runner/cgroup_task_runner.py
index a97eac2af88ef..4662b0fe82f5a 100644
--- a/airflow/contrib/task_runner/cgroup_task_runner.py
+++ b/airflow/contrib/task_runner/cgroup_task_runner.py
@@ -117,13 +117,13 @@ def start(self):
                 "creating another one",
                 cgroups.get("cpu"), cgroups.get("memory")
             )
-            self.process = self.run_command(['bash', '-c'], join_args=True)
+            self.process = self.run_command()
             return
 
         # Create a unique cgroup name
         cgroup_name = "airflow/{}/{}".format(datetime.datetime.utcnow().
                                              strftime("%Y-%m-%d"),
-                                             str(uuid.uuid1()))
+                                             str(uuid.uuid4()))
 
         self.mem_cgroup_name = "memory/{}".format(cgroup_name)
         self.cpu_cgroup_name = "cpu/{}".format(cgroup_name)
@@ -193,7 +193,8 @@ def on_finish(self):
         if self._created_cpu_cgroup:
             self._delete_cgroup(self.cpu_cgroup_name)
 
-    def _get_cgroup_names(self):
+    @staticmethod
+    def _get_cgroup_names():
         """
         :return: a mapping between the subsystem name to the cgroup name
         :rtype: dict[str, str]
diff --git a/airflow/contrib/utils/gcp_field_sanitizer.py b/airflow/contrib/utils/gcp_field_sanitizer.py
new file mode 100644
index 0000000000000..8103e6a7f0644
--- /dev/null
+++ b/airflow/contrib/utils/gcp_field_sanitizer.py
@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Sanitizer for body fields sent via GCP API.
+
+The sanitizer removes fields specified from the body.
+
+Context
+-------
+In some cases where GCP operation requires modification of existing resources (such
+as instances or instance templates) we need to sanitize body of the resources returned
+via GCP APIs. This is in the case when we retrieve information from GCP first,
+modify the body and either update the existing resource or create a new one with the
+modified body. Usually when you retrieve resource from GCP you get some extra fields which
+are Output-only, and we need to delete those fields if we want to use
+the body as input for subsequent create/insert type operation.
+
+
+Field specification
+-------------------
+
+Specification of fields is an array of strings which denote names of fields to be removed.
+The field can be either direct field name to remove from the body or the full
+specification of the path you should delete - separated with '.'
+
+
+>>> FIELDS_TO_SANITIZE = [
+>>>    "kind",
+>>>    "properties.disks.kind",
+>>>    "properties.metadata.kind",
+>>>]
+>>> body = {
+>>>     "kind": "compute#instanceTemplate",
+>>>     "name": "instance",
+>>>     "properties": {
+>>>         "disks": [
+>>>             {
+>>>                 "name": "a",
+>>>                 "kind": "compute#attachedDisk",
+>>>                 "type": "PERSISTENT",
+>>>                 "mode": "READ_WRITE",
+>>>             },
+>>>             {
+>>>                 "name": "b",
+>>>                 "kind": "compute#attachedDisk",
+>>>                 "type": "PERSISTENT",
+>>>                 "mode": "READ_WRITE",
+>>>             }
+>>>         ],
+>>>         "metadata": {
+>>>             "kind": "compute#metadata",
+>>>             "fingerprint": "GDPUYxlwHe4="
+>>>         },
+>>>     }
+>>> }
+>>> sanitizer=GcpBodyFieldSanitizer(FIELDS_TO_SANITIZE)
+>>> SANITIZED_BODY = sanitizer.sanitize(body)
+>>> json.dumps(SANITIZED_BODY, indent=2)
+{
+    "name":  "instance",
+    "properties": {
+        "disks": [
+            {
+                "name": "a",
+                "type": "PERSISTENT",
+                "mode": "READ_WRITE",
+            },
+            {
+                "name": "b",
+                "type": "PERSISTENT",
+                "mode": "READ_WRITE",
+            }
+        ],
+        "metadata": {
+            "fingerprint": "GDPUYxlwHe4="
+        },
+    }
+}
+
+Note that the components of the path can be either dictionaries or arrays of dictionaries.
+In case  they are dictionaries, subsequent component names key of the field, in case of
+arrays - the sanitizer iterates through all dictionaries in the array and searches
+components in all elements of the array.
+"""
+
+from typing import List
+
+from airflow import LoggingMixin, AirflowException
+
+
+class GcpFieldSanitizerException(AirflowException):
+    """Thrown when sanitizer finds unexpected field type in the path
+    (other than dict or array).
+    """
+
+    def __init__(self, message):
+        super(GcpFieldSanitizerException, self).__init__(message)
+
+
+class GcpBodyFieldSanitizer(LoggingMixin):
+    """Sanitizes the body according to specification.
+
+    :param sanitize_specs: array of strings that specifies which fields to remove
+    :type sanitize_specs: list[str]
+
+    """
+    def __init__(self, sanitize_specs):
+        # type: (List[str]) -> None
+        super(GcpBodyFieldSanitizer, self).__init__()
+        self._sanitize_specs = sanitize_specs
+
+    def _sanitize(self, dictionary, remaining_field_spec, current_path):
+        field_split = remaining_field_spec.split(".", 1)
+        if len(field_split) == 1:
+            field_name = field_split[0]
+            if field_name in dictionary:
+                self.log.info("Deleted %s [%s]", field_name, current_path)
+                del dictionary[field_name]
+            else:
+                self.log.debug(
+                    "The field %s is missing in %s at the path %s.", field_name, dictionary, current_path
+                )
+        else:
+            field_name = field_split[0]
+            remaining_path = field_split[1]
+            child = dictionary.get(field_name)
+            if child is None:
+                self.log.debug(
+                    "The field %s is missing in %s at the path %s. ", field_name, dictionary, current_path
+                )
+            elif isinstance(child, dict):
+                self._sanitize(child, remaining_path, "{}.{}".format(
+                    current_path, field_name))
+            elif isinstance(child, list):
+                for index, elem in enumerate(child):
+                    if not isinstance(elem, dict):
+                        self.log.warn(
+                            "The field %s element at index %s is of wrong type. "
+                            "It should be dict and is %s. Skipping it.",
+                            current_path, index, elem)
+                    self._sanitize(elem, remaining_path, "{}.{}[{}]".format(
+                        current_path, field_name, index))
+            else:
+                self.log.warn(
+                    "The field %s is of wrong type. It should be dict or list and it is %s. Skipping it.",
+                    current_path, child
+                )
+
+    def sanitize(self, body):
+        for elem in self._sanitize_specs:
+            self._sanitize(body, elem, "")
diff --git a/airflow/contrib/utils/gcp_field_validator.py b/airflow/contrib/utils/gcp_field_validator.py
new file mode 100644
index 0000000000000..73e37f3e41be9
--- /dev/null
+++ b/airflow/contrib/utils/gcp_field_validator.py
@@ -0,0 +1,444 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Validator for body fields sent via GCP API.
+
+The validator performs validation of the body (being dictionary of fields) that
+is sent in the API request to Google Cloud (via googleclient API usually).
+
+Context
+-------
+The specification mostly focuses on helping Airflow DAG developers in the development
+phase. You can build your own GCP operator (such as GcfDeployOperator for example) which
+can have built-in validation specification for the particular API. It's super helpful
+when developer plays with different fields and their values at the initial phase of
+DAG development. Most of the Google Cloud APIs perform their own validation on the
+server side, but most of the requests are asynchronous and you need to wait for result
+of the operation. This takes precious times and slows
+down iteration over the API. BodyFieldValidator is meant to be used on the client side
+and it should therefore provide an instant feedback to the developer on misspelled or
+wrong type of parameters.
+
+The validation should be performed in "execute()" method call in order to allow
+template parameters to be expanded before validation is performed.
+
+Types of fields
+---------------
+
+Specification is an array of dictionaries - each dictionary describes field, its type,
+validation, optionality, api_version supported and nested fields (for unions and dicts).
+
+Typically (for clarity and in order to aid syntax highlighting) the array of
+dicts should be defined as series of dict() executions. Fragment of example
+specification might look as follows::
+
+    SPECIFICATION =[
+       dict(name="an_union", type="union", optional=True, fields=[
+           dict(name="variant_1", type="dict"),
+           dict(name="variant_2", regexp=r'^.+$', api_version='v1beta2'),
+       ),
+       dict(name="an_union", type="dict", fields=[
+           dict(name="field_1", type="dict"),
+           dict(name="field_2", regexp=r'^.+$'),
+       ),
+       ...
+    ]
+
+
+Each field should have key = "name" indicating field name. The field can be of one of the
+following types:
+
+* Dict fields: (key = "type", value="dict"):
+  Field of this type should contain nested fields in form of an array of dicts.
+  Each of the fields in the array is then expected (unless marked as optional)
+  and validated recursively. If an extra field is present in the dictionary, warning is
+  printed in log file (but the validation succeeds - see the Forward-compatibility notes)
+* List fields: (key = "type", value="list"):
+  Field of this type should be a list. Only the type correctness is validated.
+  The contents of a list are not subject to validation.
+* Union fields (key = "type", value="union"): field of this type should contain nested
+  fields in form of an array of dicts. One of the fields (and only one) should be
+  present (unless the union is marked as optional). If more than one union field is
+  present, FieldValidationException is raised. If none of the union fields is
+  present - warning is printed in the log (see below Forward-compatibility notes).
+* Fields validated for non-emptiness: (key = "allow_empty") - this applies only to
+  fields the value of which is a string, and it allows to check for non-emptiness of
+  the field (allow_empty=False).
+* Regexp-validated fields: (key = "regexp") - fields of this type are assumed to be
+  strings and they are validated with the regexp specified. Remember that the regexps
+  should ideally contain ^ at the beginning and $ at the end to make sure that
+  the whole field content is validated. Typically such regexp
+  validations should be used carefully and sparingly (see Forward-compatibility
+  notes below).
+* Custom-validated fields: (key = "custom_validation") - fields of this type are validated
+  using method specified via custom_validation field. Any exception thrown in the custom
+  validation will be turned into FieldValidationException and will cause validation to
+  fail. Such custom validations might be used to check numeric fields (including
+  ranges of values), booleans or any other types of fields.
+* API version: (key="api_version") if API version is specified, then the field will only
+  be validated when api_version used at field validator initialization matches exactly the
+  the version specified. If you want to declare fields that are available in several
+  versions of the APIs, you should specify the field as many times as many API versions
+  should be supported (each time with different API version).
+* if none of the keys ("type", "regexp", "custom_validation" - the field is not validated
+
+You can see some of the field examples in EXAMPLE_VALIDATION_SPECIFICATION.
+
+
+Forward-compatibility notes
+---------------------------
+Certain decisions are crucial to allow the client APIs to work also with future API
+versions. Since body attached is passed to the API’s call, this is entirely
+possible to pass-through any new fields in the body (for future API versions) -
+albeit without validation on the client side - they can and will still be validated
+on the server side usually.
+
+Here are the guidelines that you should follow to make validation forward-compatible:
+
+* most of the fields are not validated for their content. It's possible to use regexp
+  in some specific cases that are guaranteed not to change in the future, but for most
+  fields regexp validation should be r'^.+$' indicating check for non-emptiness
+* api_version is not validated - user can pass any future version of the api here. The API
+  version is only used to filter parameters that are marked as present in this api version
+  any new (not present in the specification) fields in the body are allowed (not verified)
+  For dictionaries, new fields can be added to dictionaries by future calls. However if an
+  unknown field in dictionary is added, a warning is logged by the client (but validation
+  remains successful). This is very nice feature to protect against typos in names.
+* For unions, newly added union variants can be added by future calls and they will
+  pass validation, however the content or presence of those fields will not be validated.
+  This means that it’s possible to send a new non-validated union field together with an
+  old validated field and this problem will not be detected by the client. In such case
+  warning will be printed.
+* When you add validator to an operator, you should also add ``validate_body`` parameter
+  (default = True) to __init__ of such operators - when it is set to False,
+  no validation should be performed. This is a safeguard for totally unpredicted and
+  backwards-incompatible changes that might sometimes occur in the APIs.
+
+"""
+
+import re
+from typing import Sequence, Dict, Callable
+
+from airflow import LoggingMixin, AirflowException
+
+COMPOSITE_FIELD_TYPES = ['union', 'dict', 'list']
+
+
+class GcpFieldValidationException(AirflowException):
+    """Thrown when validation finds dictionary field not valid according to specification.
+    """
+
+    def __init__(self, message):
+        super(GcpFieldValidationException, self).__init__(message)
+
+
+class GcpValidationSpecificationException(AirflowException):
+    """Thrown when validation specification is wrong.
+
+    This should only happen during development as ideally
+     specification itself should not be invalid ;) .
+    """
+
+    def __init__(self, message):
+        super(GcpValidationSpecificationException, self).__init__(message)
+
+
+def _int_greater_than_zero(value):
+    if int(value) <= 0:
+        raise GcpFieldValidationException("The available memory has to be greater than 0")
+
+
+EXAMPLE_VALIDATION_SPECIFICATION = [
+    dict(name="name", allow_empty=False),
+    dict(name="description", allow_empty=False, optional=True),
+    dict(name="availableMemoryMb", custom_validation=_int_greater_than_zero,
+         optional=True),
+    dict(name="labels", optional=True, type="dict"),
+    dict(name="an_union", type="union", fields=[
+        dict(name="variant_1", regexp=r'^.+$'),
+        dict(name="variant_2", regexp=r'^.+$', api_version='v1beta2'),
+        dict(name="variant_3", type="dict", fields=[
+            dict(name="url", regexp=r'^.+$')
+        ]),
+        dict(name="variant_4")
+    ]),
+]
+
+
+class GcpBodyFieldValidator(LoggingMixin):
+    """Validates correctness of request body according to specification.
+
+    The specification can describe various type of
+    fields including custom validation, and union of fields. This validator is
+    to be reusable by various operators. See the EXAMPLE_VALIDATION_SPECIFICATION
+    for some examples and explanations of how to create specification.
+
+    :param validation_specs: dictionary describing validation specification
+    :type validation_specs: list[dict]
+    :param api_version: Version of the api used (for example v1)
+    :type api_version: str
+
+    """
+    def __init__(self, validation_specs, api_version):
+        # type: (Sequence[Dict], str) -> None
+        super(GcpBodyFieldValidator, self).__init__()
+        self._validation_specs = validation_specs
+        self._api_version = api_version
+
+    @staticmethod
+    def _get_field_name_with_parent(field_name, parent):
+        if parent:
+            return parent + '.' + field_name
+        return field_name
+
+    @staticmethod
+    def _sanity_checks(children_validation_specs, field_type, full_field_path,
+                       regexp, allow_empty, custom_validation, value):
+        # type: (dict, str, str, str, Callable, object) -> None
+        if value is None and field_type != 'union':
+            raise GcpFieldValidationException(
+                "The required body field '{}' is missing. Please add it.".
+                format(full_field_path))
+        if regexp and field_type:
+            raise GcpValidationSpecificationException(
+                "The validation specification entry '{}' has both type and regexp. "
+                "The regexp is only allowed without type (i.e. assume type is 'str' "
+                "that can be validated with regexp)".format(full_field_path))
+        if allow_empty is not None and field_type:
+            raise GcpValidationSpecificationException(
+                "The validation specification entry '{}' has both type and allow_empty. "
+                "The allow_empty is only allowed without type (i.e. assume type is 'str' "
+                "that can be validated with allow_empty)".format(full_field_path))
+        if children_validation_specs and field_type not in COMPOSITE_FIELD_TYPES:
+            raise GcpValidationSpecificationException(
+                "Nested fields are specified in field '{}' of type '{}'. "
+                "Nested fields are only allowed for fields of those types: ('{}').".
+                format(full_field_path, field_type, COMPOSITE_FIELD_TYPES))
+        if custom_validation and field_type:
+            raise GcpValidationSpecificationException(
+                "The validation specification field '{}' has both type and "
+                "custom_validation. Custom validation is only allowed without type.".
+                format(full_field_path))
+
+    @staticmethod
+    def _validate_regexp(full_field_path, regexp, value):
+        # type: (str, str, str) -> None
+        if not re.match(regexp, value):
+            # Note matching of only the beginning as we assume the regexps all-or-nothing
+            raise GcpFieldValidationException(
+                "The body field '{}' of value '{}' does not match the field "
+                "specification regexp: '{}'.".
+                format(full_field_path, value, regexp))
+
+    @staticmethod
+    def _validate_is_empty(full_field_path, value):
+        # type: (str, str) -> None
+        if not value:
+            raise GcpFieldValidationException(
+                "The body field '{}' can't be empty. Please provide a value."
+                .format(full_field_path, value))
+
+    def _validate_dict(self, children_validation_specs, full_field_path, value):
+        # type: (dict, str, dict) -> None
+        for child_validation_spec in children_validation_specs:
+            self._validate_field(validation_spec=child_validation_spec,
+                                 dictionary_to_validate=value,
+                                 parent=full_field_path)
+        all_dict_keys = [spec['name'] for spec in children_validation_specs]
+        for field_name in value.keys():
+            if field_name not in all_dict_keys:
+                self.log.warning(
+                    "The field '%s' is in the body, but is not specified in the "
+                    "validation specification '%s'. "
+                    "This might be because you are using newer API version and "
+                    "new field names defined for that version. Then the warning "
+                    "can be safely ignored, or you might want to upgrade the operator"
+                    "to the version that supports the new API version.",
+                    self._get_field_name_with_parent(field_name, full_field_path),
+                    children_validation_specs)
+
+    def _validate_union(self, children_validation_specs, full_field_path,
+                        dictionary_to_validate):
+        # type: (dict, str, dict) -> None
+        field_found = False
+        found_field_name = None
+        for child_validation_spec in children_validation_specs:
+            # Forcing optional so that we do not have to type optional = True
+            # in specification for all union fields
+            new_field_found = self._validate_field(
+                validation_spec=child_validation_spec,
+                dictionary_to_validate=dictionary_to_validate,
+                parent=full_field_path,
+                force_optional=True)
+            field_name = child_validation_spec['name']
+            if new_field_found and field_found:
+                raise GcpFieldValidationException(
+                    "The mutually exclusive fields '{}' and '{}' belonging to the "
+                    "union '{}' are both present. Please remove one".
+                    format(field_name, found_field_name, full_field_path))
+            if new_field_found:
+                field_found = True
+                found_field_name = field_name
+        if not field_found:
+            self.log.warning(
+                "There is no '%s' union defined in the body %s. "
+                "Validation expected one of '%s' but could not find any. It's possible "
+                "that you are using newer API version and there is another union variant "
+                "defined for that version. Then the warning can be safely ignored, "
+                "or you might want to upgrade the operator to the version that "
+                "supports the new API version.",
+                full_field_path, dictionary_to_validate,
+                [field['name'] for field in children_validation_specs])
+
+    def _validate_field(self, validation_spec, dictionary_to_validate, parent=None,
+                        force_optional=False):
+        """
+        Validates if field is OK.
+
+        :param validation_spec: specification of the field
+        :type validation_spec: dict
+        :param dictionary_to_validate: dictionary where the field should be present
+        :type dictionary_to_validate: dict
+        :param parent: full path of parent field
+        :type parent: str
+        :param force_optional: forces the field to be optional
+            (all union fields have force_optional set to True)
+        :type force_optional: bool
+        :return: True if the field is present
+        """
+        field_name = validation_spec['name']
+        field_type = validation_spec.get('type')
+        optional = validation_spec.get('optional')
+        regexp = validation_spec.get('regexp')
+        allow_empty = validation_spec.get('allow_empty')
+        children_validation_specs = validation_spec.get('fields')
+        required_api_version = validation_spec.get('api_version')
+        custom_validation = validation_spec.get('custom_validation')
+
+        full_field_path = self._get_field_name_with_parent(field_name=field_name,
+                                                           parent=parent)
+        if required_api_version and required_api_version != self._api_version:
+            self.log.debug(
+                "Skipping validation of the field '%s' for API version '%s' "
+                "as it is only valid for API version '%s'",
+                field_name, self._api_version, required_api_version)
+            return False
+        value = dictionary_to_validate.get(field_name)
+
+        if (optional or force_optional) and value is None:
+            self.log.debug("The optional field '%s' is missing. That's perfectly OK.", full_field_path)
+            return False
+
+        # Certainly down from here the field is present (value is not None)
+        # so we should only return True from now on
+
+        self._sanity_checks(children_validation_specs=children_validation_specs,
+                            field_type=field_type,
+                            full_field_path=full_field_path,
+                            regexp=regexp,
+                            allow_empty=allow_empty,
+                            custom_validation=custom_validation,
+                            value=value)
+
+        if allow_empty is False:
+            self._validate_is_empty(full_field_path, value)
+        if regexp:
+            self._validate_regexp(full_field_path, regexp, value)
+        elif field_type == 'dict':
+            if not isinstance(value, dict):
+                raise GcpFieldValidationException(
+                    "The field '{}' should be of dictionary type according to the "
+                    "specification '{}' but it is '{}'".
+                    format(full_field_path, validation_spec, value))
+            if children_validation_specs is None:
+                self.log.debug(
+                    "The dict field '%s' has no nested fields defined in the "
+                    "specification '%s'. That's perfectly ok - it's content will "
+                    "not be validated.", full_field_path, validation_spec)
+            else:
+                self._validate_dict(children_validation_specs, full_field_path, value)
+        elif field_type == 'union':
+            if not children_validation_specs:
+                raise GcpValidationSpecificationException(
+                    "The union field '%s' has no nested fields "
+                    "defined in specification '%s'. Unions should have at least one "
+                    "nested field defined.", full_field_path, validation_spec)
+            self._validate_union(children_validation_specs, full_field_path,
+                                 dictionary_to_validate)
+        elif field_type == 'list':
+            if not isinstance(value, list):
+                raise GcpFieldValidationException(
+                    "The field '{}' should be of list type according to the "
+                    "specification '{}' but it is '{}'".
+                    format(full_field_path, validation_spec, value))
+        elif custom_validation:
+            try:
+                custom_validation(value)
+            except Exception as e:
+                raise GcpFieldValidationException(
+                    "Error while validating custom field '{}' specified by '{}': '{}'".
+                    format(full_field_path, validation_spec, e))
+        elif field_type is None:
+            self.log.debug("The type of field '%s' is not specified in '%s'. "
+                           "Not validating its content.", full_field_path, validation_spec)
+        else:
+            raise GcpValidationSpecificationException(
+                "The field '{}' is of type '{}' in specification '{}'."
+                "This type is unknown to validation!".format(
+                    full_field_path, field_type, validation_spec))
+        return True
+
+    def validate(self, body_to_validate):
+        """
+        Validates if the body (dictionary) follows specification that the validator was
+        instantiated with. Raises ValidationSpecificationException or
+        ValidationFieldException in case of problems with specification or the
+        body not conforming to the specification respectively.
+
+        :param body_to_validate: body that must follow the specification
+        :type body_to_validate: dict
+        :return: None
+        """
+        try:
+            for validation_spec in self._validation_specs:
+                self._validate_field(validation_spec=validation_spec,
+                                     dictionary_to_validate=body_to_validate)
+        except GcpFieldValidationException as e:
+            raise GcpFieldValidationException(
+                "There was an error when validating: body '{}': '{}'".
+                format(body_to_validate, e))
+        all_field_names = [spec['name'] for spec in self._validation_specs
+                           if spec.get('type') != 'union' and
+                           spec.get('api_version') != self._api_version]
+        all_union_fields = [spec for spec in self._validation_specs
+                            if spec.get('type') == 'union']
+        for union_field in all_union_fields:
+            all_field_names.extend(
+                [nested_union_spec['name'] for nested_union_spec in union_field['fields']
+                 if nested_union_spec.get('type') != 'union' and
+                 nested_union_spec.get('api_version') != self._api_version])
+        for field_name in body_to_validate.keys():
+            if field_name not in all_field_names:
+                self.log.warning(
+                    "The field '%s' is in the body, but is not specified in the "
+                    "validation specification '%s'. "
+                    "This might be because you are using newer API version and "
+                    "new field names defined for that version. Then the warning "
+                    "can be safely ignored, or you might want to upgrade the operator"
+                    "to the version that supports the new API version.",
+                    field_name, self._validation_specs)
diff --git a/airflow/contrib/operators/mlengine_operator_utils.py b/airflow/contrib/utils/mlengine_operator_utils.py
similarity index 95%
rename from airflow/contrib/operators/mlengine_operator_utils.py
rename to airflow/contrib/utils/mlengine_operator_utils.py
index 7ce784ebb4ae3..c3ca8530dee6e 100644
--- a/airflow/contrib/operators/mlengine_operator_utils.py
+++ b/airflow/contrib/utils/mlengine_operator_utils.py
@@ -108,22 +108,22 @@ def validate_err_and_count(summary):
     :param task_prefix: a prefix for the tasks. Only alphanumeric characters and
         hyphen are allowed (no underscores), since this will be used as dataflow
         job name, which doesn't allow other characters.
-    :type task_prefix: string
+    :type task_prefix: str
 
     :param data_format: either of 'TEXT', 'TF_RECORD', 'TF_RECORD_GZIP'
-    :type data_format: string
+    :type data_format: str
 
     :param input_paths: a list of input paths to be sent to BatchPrediction.
-    :type input_paths: list of strings
+    :type input_paths: list[str]
 
     :param prediction_path: GCS path to put the prediction results in.
-    :type prediction_path: string
+    :type prediction_path: str
 
     :param metric_fn_and_keys: a tuple of metric_fn and metric_keys:
         - metric_fn is a function that accepts a dictionary (for an instance),
           and returns a tuple of metric(s) that it calculates.
         - metric_keys is a list of strings to denote the key of each metric.
-    :type metric_fn_and_keys: tuple of a function and a list of strings
+    :type metric_fn_and_keys: tuple of a function and a list[str]
 
     :param validate_fn: a function to validate whether the averaged metric(s) is
         good enough to push the model.
@@ -132,17 +132,17 @@ def validate_err_and_count(summary):
     :param batch_prediction_job_id: the id to use for the Cloud ML Batch
         prediction job. Passed directly to the MLEngineBatchPredictionOperator as
         the job_id argument.
-    :type batch_prediction_job_id: string
+    :type batch_prediction_job_id: str
 
     :param project_id: the Google Cloud Platform project id in which to execute
         Cloud ML Batch Prediction and Dataflow jobs. If None, then the `dag`'s
         `default_args['project_id']` will be used.
-    :type project_id: string
+    :type project_id: str
 
     :param region: the Google Cloud Platform region in which to execute Cloud ML
         Batch Prediction and Dataflow jobs. If None, then the `dag`'s
         `default_args['region']` will be used.
-    :type region: string
+    :type region: str
 
     :param dataflow_options: options to run Dataflow jobs. If None, then the
         `dag`'s `default_args['dataflow_default_options']` will be used.
@@ -152,22 +152,22 @@ def validate_err_and_count(summary):
         tensorflow.estimator.export_savedmodel(). It cannot be used with
         model_name or version_name below. See MLEngineBatchPredictionOperator for
         more detail.
-    :type model_uri: string
+    :type model_uri: str
 
     :param model_name: Used to indicate a model to use for prediction. Can be
         used in combination with version_name, but cannot be used together with
         model_uri. See MLEngineBatchPredictionOperator for more detail. If None,
         then the `dag`'s `default_args['model_name']` will be used.
-    :type model_name: string
+    :type model_name: str
 
-    :param version_name: Used to indicate a model version to use for prediciton,
+    :param version_name: Used to indicate a model version to use for prediction,
         in combination with model_name. Cannot be used together with model_uri.
         See MLEngineBatchPredictionOperator for more detail. If None, then the
         `dag`'s `default_args['version_name']` will be used.
-    :type version_name: string
+    :type version_name: str
 
     :param dag: The `DAG` to use for all Operators.
-    :type dag: airflow.DAG
+    :type dag: airflow.models.DAG
 
     :returns: a tuple of three operators, (prediction, summary, validation)
     :rtype: tuple(DataFlowPythonOperator, DataFlowPythonOperator,
@@ -213,7 +213,7 @@ def validate_err_and_count(summary):
     evaluate_summary = DataFlowPythonOperator(
         task_id=(task_prefix + "-summary"),
         py_options=["-m"],
-        py_file="airflow.contrib.operators.mlengine_prediction_summary",
+        py_file="airflow.contrib.utils.mlengine_prediction_summary",
         dataflow_default_options=dataflow_options,
         options={
             "prediction_path": prediction_path,
diff --git a/airflow/contrib/operators/mlengine_prediction_summary.py b/airflow/contrib/utils/mlengine_prediction_summary.py
similarity index 93%
rename from airflow/contrib/operators/mlengine_prediction_summary.py
rename to airflow/contrib/utils/mlengine_prediction_summary.py
index 17fc2c090379e..def793c1be001 100644
--- a/airflow/contrib/operators/mlengine_prediction_summary.py
+++ b/airflow/contrib/utils/mlengine_prediction_summary.py
@@ -1,3 +1,4 @@
+# flake8: noqa: F841
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -102,24 +103,26 @@ def metric_fn(inst):
 
 
 class JsonCoder(object):
-    def encode(self, x):
+    @staticmethod
+    def encode(x):
         return json.dumps(x)
 
-    def decode(self, x):
+    @staticmethod
+    def decode(x):
         return json.loads(x)
 
 
 @beam.ptransform_fn
 def MakeSummary(pcoll, metric_fn, metric_keys):  # pylint: disable=invalid-name
     return (
-        pcoll
-        | "ApplyMetricFnPerInstance" >> beam.Map(metric_fn)
-        | "PairWith1" >> beam.Map(lambda tup: tup + (1,))
-        | "SumTuple" >> beam.CombineGlobally(beam.combiners.TupleCombineFn(
-            *([sum] * (len(metric_keys) + 1))))
-        | "AverageAndMakeDict" >> beam.Map(
+        pcoll |
+        "ApplyMetricFnPerInstance" >> beam.Map(metric_fn) |
+        "PairWith1" >> beam.Map(lambda tup: tup + (1,)) |
+        "SumTuple" >> beam.CombineGlobally(beam.combiners.TupleCombineFn(
+            *([sum] * (len(metric_keys) + 1)))) |
+        "AverageAndMakeDict" >> beam.Map(
             lambda tup: dict(
-                [(name, tup[i]/tup[-1]) for i, name in enumerate(metric_keys)] +
+                [(name, tup[i] / tup[-1]) for i, name in enumerate(metric_keys)] +
                 [("count", tup[-1])])))
 
 
diff --git a/airflow/contrib/utils/sendgrid.py b/airflow/contrib/utils/sendgrid.py
index 9055c97879e17..5082c7a3328bf 100644
--- a/airflow/contrib/utils/sendgrid.py
+++ b/airflow/contrib/utils/sendgrid.py
@@ -42,7 +42,7 @@ def send_email(to, subject, html_content, files=None,
 
     To use this plugin:
     0. include sendgrid subpackage as part of your Airflow installation, e.g.,
-    pip install airflow[sendgrid]
+    pip install 'apache-airflow[sendgrid]'
     1. update [email] backend in airflow.cfg, i.e.,
     [email]
     email_backend = airflow.contrib.utils.sendgrid.send_email
@@ -88,7 +88,7 @@ def send_email(to, subject, html_content, files=None,
         basename = os.path.basename(fname)
         attachment = Attachment()
         with open(fname, "rb") as f:
-            attachment.content = base64.b64encode(f.read())
+            attachment.content = str(base64.b64encode(f.read()), 'utf-8')
             attachment.type = mimetypes.guess_type(basename)[0]
             attachment.filename = basename
             attachment.disposition = "attachment"
diff --git a/airflow/contrib/utils/weekday.py b/airflow/contrib/utils/weekday.py
new file mode 100644
index 0000000000000..8630e5a3dca7c
--- /dev/null
+++ b/airflow/contrib/utils/weekday.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import enum
+
+
+@enum.unique
+class WeekDay(enum.IntEnum):
+    """
+    Python Enum containing Days of the Week
+    """
+    MONDAY = 1
+    TUESDAY = 2
+    WEDNESDAY = 3
+    THURSDAY = 4
+    FRIDAY = 5
+    SATURDAY = 6
+    SUNDAY = 7
+
+    @classmethod
+    def get_weekday_number(cls, week_day_str):
+        """
+        Return the ISO Week Day Number for a Week Day
+
+        :param week_day_str: Full Name of the Week Day. Example: "Sunday"
+        :type week_day_str: str
+        :return: ISO Week Day Number corresponding to the provided Weekday
+        """
+        sanitized_week_day_str = week_day_str.upper()
+
+        if sanitized_week_day_str not in cls.__members__:
+            raise AttributeError(
+                'Invalid Week Day passed: "{}"'.format(week_day_str)
+            )
+
+        return cls[sanitized_week_day_str]
diff --git a/airflow/dag/base_dag.py b/airflow/dag/base_dag.py
index 5719f572e9b4c..0e65775d41424 100644
--- a/airflow/dag/base_dag.py
+++ b/airflow/dag/base_dag.py
@@ -96,6 +96,6 @@ def dag_ids(self):
     def get_dag(self, dag_id):
         """
         :return: whether the task exists in this bag
-        :rtype: BaseDag
+        :rtype: airflow.dag.base_dag.BaseDag
         """
         raise NotImplementedError()
diff --git a/airflow/default_login.py b/airflow/default_login.py
index d44dbf39ea9b3..e423199fe01a9 100644
--- a/airflow/default_login.py
+++ b/airflow/default_login.py
@@ -25,11 +25,11 @@
 """
 
 import flask_login
-from flask_login import login_required, current_user, logout_user
+from flask_login import login_required, current_user, logout_user  # noqa: F401
 
 from flask import url_for, redirect
 
-from airflow import settings
+from airflow import settings  # noqa: F401
 from airflow import models
 from airflow.utils.db import provide_session
 
@@ -44,14 +44,17 @@ class DefaultUser(object):
     def __init__(self, user):
         self.user = user
 
+    @property
     def is_active(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_authenticated(self):
         """Required by flask_login"""
         return True
 
+    @property
     def is_anonymous(self):
         """Required by flask_login"""
         return False
@@ -64,9 +67,6 @@ def is_superuser(self):
         """Access all the things"""
         return True
 
-# models.User = User  # hack!
-# del User
-
 
 @login_manager.user_loader
 @provide_session
diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py
index b2d9d14610dfc..68accc6317bbf 100644
--- a/airflow/example_dags/example_bash_operator.py
+++ b/airflow/example_dags/example_bash_operator.py
@@ -17,44 +17,57 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import airflow
 from builtins import range
-from airflow.operators.bash_operator import BashOperator
-from airflow.operators.dummy_operator import DummyOperator
-from airflow.models import DAG
 from datetime import timedelta
 
+import airflow
+from airflow.models import DAG
+from airflow.operators.bash_operator import BashOperator
+from airflow.operators.dummy_operator import DummyOperator
 
 args = {
     'owner': 'airflow',
-    'start_date': airflow.utils.dates.days_ago(2)
+    'start_date': airflow.utils.dates.days_ago(2),
 }
 
 dag = DAG(
-    dag_id='example_bash_operator', default_args=args,
+    dag_id='example_bash_operator',
+    default_args=args,
     schedule_interval='0 0 * * *',
-    dagrun_timeout=timedelta(minutes=60))
+    dagrun_timeout=timedelta(minutes=60),
+)
 
-cmd = 'ls -l'
-run_this_last = DummyOperator(task_id='run_this_last', dag=dag)
+run_this_last = DummyOperator(
+    task_id='run_this_last',
+    dag=dag,
+)
 
+# [START howto_operator_bash]
 run_this = BashOperator(
-    task_id='run_after_loop', bash_command='echo 1', dag=dag)
-run_this.set_downstream(run_this_last)
+    task_id='run_after_loop',
+    bash_command='echo 1',
+    dag=dag,
+)
+# [END howto_operator_bash]
+
+run_this >> run_this_last
 
 for i in range(3):
-    i = str(i)
     task = BashOperator(
-        task_id='runme_' + i,
+        task_id='runme_' + str(i),
         bash_command='echo "{{ task_instance_key_str }}" && sleep 1',
-        dag=dag)
-    task.set_downstream(run_this)
+        dag=dag,
+    )
+    task >> run_this
 
-task = BashOperator(
+# [START howto_operator_bash_template]
+also_run_this = BashOperator(
     task_id='also_run_this',
     bash_command='echo "run_id={{ run_id }} | dag_run={{ dag_run }}"',
-    dag=dag)
-task.set_downstream(run_this_last)
+    dag=dag,
+)
+# [END howto_operator_bash_template]
+also_run_this >> run_this_last
 
 if __name__ == "__main__":
     dag.cli()
diff --git a/airflow/example_dags/example_branch_operator.py b/airflow/example_dags/example_branch_operator.py
index 45bf11f30139c..197d7d7a73bad 100644
--- a/airflow/example_dags/example_branch_operator.py
+++ b/airflow/example_dags/example_branch_operator.py
@@ -17,43 +17,53 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import airflow
-from airflow.operators.python_operator import BranchPythonOperator
-from airflow.operators.dummy_operator import DummyOperator
-from airflow.models import DAG
 import random
 
+import airflow
+from airflow.models import DAG
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.operators.python_operator import BranchPythonOperator
 
 args = {
     'owner': 'airflow',
-    'start_date': airflow.utils.dates.days_ago(2)
+    'start_date': airflow.utils.dates.days_ago(2),
 }
 
 dag = DAG(
     dag_id='example_branch_operator',
     default_args=args,
-    schedule_interval="@daily")
+    schedule_interval="@daily",
+)
 
-cmd = 'ls -l'
-run_this_first = DummyOperator(task_id='run_this_first', dag=dag)
+run_this_first = DummyOperator(
+    task_id='run_this_first',
+    dag=dag,
+)
 
 options = ['branch_a', 'branch_b', 'branch_c', 'branch_d']
 
 branching = BranchPythonOperator(
     task_id='branching',
     python_callable=lambda: random.choice(options),
-    dag=dag)
-branching.set_upstream(run_this_first)
+    dag=dag,
+)
+run_this_first >> branching
 
 join = DummyOperator(
     task_id='join',
     trigger_rule='one_success',
-    dag=dag
+    dag=dag,
 )
 
 for option in options:
-    t = DummyOperator(task_id=option, dag=dag)
-    t.set_upstream(branching)
-    dummy_follow = DummyOperator(task_id='follow_' + option, dag=dag)
-    t.set_downstream(dummy_follow)
-    dummy_follow.set_downstream(join)
+    t = DummyOperator(
+        task_id=option,
+        dag=dag,
+    )
+
+    dummy_follow = DummyOperator(
+        task_id='follow_' + option,
+        dag=dag,
+    )
+
+    branching >> t >> dummy_follow >> join
diff --git a/airflow/example_dags/example_branch_python_dop_operator_3.py b/airflow/example_dags/example_branch_python_dop_operator_3.py
index 7be55a5f36915..36edb27317344 100644
--- a/airflow/example_dags/example_branch_python_dop_operator_3.py
+++ b/airflow/example_dags/example_branch_python_dop_operator_3.py
@@ -18,9 +18,9 @@
 # under the License.
 
 import airflow
-from airflow.operators.python_operator import BranchPythonOperator
-from airflow.operators.dummy_operator import DummyOperator
 from airflow.models import DAG
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.operators.python_operator import BranchPythonOperator
 
 args = {
     'owner': 'airflow',
@@ -31,32 +31,29 @@
 # BranchPython operator that depends on past
 # and where tasks may run or be skipped on
 # alternating runs
-dag = DAG(dag_id='example_branch_dop_operator_v3',
-          schedule_interval='*/1 * * * *', default_args=args)
-
+dag = DAG(
+    dag_id='example_branch_dop_operator_v3',
+    schedule_interval='*/1 * * * *',
+    default_args=args,
+)
 
-def should_run(ds, **kwargs):
 
+def should_run(**kwargs):
     print('------------- exec dttm = {} and minute = {}'.
           format(kwargs['execution_date'], kwargs['execution_date'].minute))
     if kwargs['execution_date'].minute % 2 == 0:
-        return "oper_1"
+        return "dummy_task_1"
     else:
-        return "oper_2"
+        return "dummy_task_2"
 
 
 cond = BranchPythonOperator(
     task_id='condition',
     provide_context=True,
     python_callable=should_run,
-    dag=dag)
-
-oper_1 = DummyOperator(
-    task_id='oper_1',
-    dag=dag)
-oper_1.set_upstream(cond)
+    dag=dag,
+)
 
-oper_2 = DummyOperator(
-    task_id='oper_2',
-    dag=dag)
-oper_2.set_upstream(cond)
+dummy_task_1 = DummyOperator(task_id='dummy_task_1', dag=dag)
+dummy_task_2 = DummyOperator(task_id='dummy_task_2', dag=dag)
+cond >> [dummy_task_1, dummy_task_2]
diff --git a/airflow/example_dags/example_http_operator.py b/airflow/example_dags/example_http_operator.py
index da7ea3f2187e3..4a469795a21f7 100644
--- a/airflow/example_dags/example_http_operator.py
+++ b/airflow/example_dags/example_http_operator.py
@@ -49,14 +49,16 @@
     data=json.dumps({"priority": 5}),
     headers={"Content-Type": "application/json"},
     response_check=lambda response: True if len(response.json()) == 0 else False,
-    dag=dag)
+    dag=dag,
+)
 
 t5 = SimpleHttpOperator(
     task_id='post_op_formenc',
     endpoint='nodes/url',
     data="name=Joe",
     headers={"Content-Type": "application/x-www-form-urlencoded"},
-    dag=dag)
+    dag=dag,
+)
 
 t2 = SimpleHttpOperator(
     task_id='get_op',
@@ -64,7 +66,8 @@
     endpoint='api/v1.0/nodes',
     data={"param1": "value1", "param2": "value2"},
     headers={},
-    dag=dag)
+    dag=dag,
+)
 
 t3 = SimpleHttpOperator(
     task_id='put_op',
@@ -72,7 +75,8 @@
     endpoint='api/v1.0/nodes',
     data=json.dumps({"priority": 5}),
     headers={"Content-Type": "application/json"},
-    dag=dag)
+    dag=dag,
+)
 
 t4 = SimpleHttpOperator(
     task_id='del_op',
@@ -80,19 +84,17 @@
     endpoint='api/v1.0/nodes',
     data="some=data",
     headers={"Content-Type": "application/x-www-form-urlencoded"},
-    dag=dag)
+    dag=dag,
+)
 
 sensor = HttpSensor(
     task_id='http_sensor_check',
     http_conn_id='http_default',
     endpoint='',
     request_params={},
-    response_check=lambda response: True if "Google" in response.content else False,
+    response_check=lambda response: True if "Google" in response.text else False,
     poke_interval=5,
-    dag=dag)
+    dag=dag,
+)
 
-t1.set_upstream(sensor)
-t2.set_upstream(t1)
-t3.set_upstream(t2)
-t4.set_upstream(t3)
-t5.set_upstream(t4)
+sensor >> t1 >> t2 >> t3 >> t4 >> t5
diff --git a/airflow/example_dags/example_latest_only.py b/airflow/example_dags/example_latest_only.py
index fdb2dca490fbf..635a7641983c0 100644
--- a/airflow/example_dags/example_latest_only.py
+++ b/airflow/example_dags/example_latest_only.py
@@ -33,6 +33,6 @@
 )
 
 latest_only = LatestOnlyOperator(task_id='latest_only', dag=dag)
-
 task1 = DummyOperator(task_id='task1', dag=dag)
-task1.set_upstream(latest_only)
+
+latest_only >> task1
diff --git a/airflow/example_dags/example_latest_only_with_trigger.py b/airflow/example_dags/example_latest_only_with_trigger.py
index b8f4811c1a529..3559afb0c85b9 100644
--- a/airflow/example_dags/example_latest_only_with_trigger.py
+++ b/airflow/example_dags/example_latest_only_with_trigger.py
@@ -34,15 +34,10 @@
 )
 
 latest_only = LatestOnlyOperator(task_id='latest_only', dag=dag)
-
 task1 = DummyOperator(task_id='task1', dag=dag)
-task1.set_upstream(latest_only)
-
 task2 = DummyOperator(task_id='task2', dag=dag)
-
 task3 = DummyOperator(task_id='task3', dag=dag)
-task3.set_upstream([task1, task2])
+task4 = DummyOperator(task_id='task4', dag=dag, trigger_rule=TriggerRule.ALL_DONE)
 
-task4 = DummyOperator(task_id='task4', dag=dag,
-                      trigger_rule=TriggerRule.ALL_DONE)
-task4.set_upstream([task1, task2])
+latest_only >> task1 >> [task3, task4]
+task2 >> [task3, task4]
diff --git a/airflow/example_dags/example_passing_params_via_test_command.py b/airflow/example_dags/example_passing_params_via_test_command.py
index 7efca2f3b0ef2..2aef593abfd8b 100644
--- a/airflow/example_dags/example_passing_params_via_test_command.py
+++ b/airflow/example_dags/example_passing_params_via_test_command.py
@@ -18,18 +18,21 @@
 # under the License.
 
 from datetime import timedelta
+
 import airflow
 from airflow import DAG
 from airflow.operators.bash_operator import BashOperator
 from airflow.operators.python_operator import PythonOperator
 
-
-dag = DAG("example_passing_params_via_test_command",
-          default_args={"owner": "airflow",
-                        "start_date": airflow.utils.dates.days_ago(1)},
-          schedule_interval='*/1 * * * *',
-          dagrun_timeout=timedelta(minutes=4)
-          )
+dag = DAG(
+    "example_passing_params_via_test_command",
+    default_args={
+        "owner": "airflow",
+        "start_date": airflow.utils.dates.days_ago(1),
+    },
+    schedule_interval='*/1 * * * *',
+    dagrun_timeout=timedelta(minutes=4),
+)
 
 
 def my_py_command(ds, **kwargs):
@@ -54,12 +57,14 @@ def my_py_command(ds, **kwargs):
     provide_context=True,
     python_callable=my_py_command,
     params={"miff": "agg"},
-    dag=dag)
-
+    dag=dag,
+)
 
 also_run_this = BashOperator(
     task_id='also_run_this',
     bash_command=my_templated_command,
     params={"miff": "agg"},
-    dag=dag)
-also_run_this.set_upstream(run_this)
+    dag=dag,
+)
+
+run_this >> also_run_this
diff --git a/airflow/example_dags/example_python_operator.py b/airflow/example_dags/example_python_operator.py
index 0ecf099e7b01c..84705181a8b01 100644
--- a/airflow/example_dags/example_python_operator.py
+++ b/airflow/example_dags/example_python_operator.py
@@ -18,29 +18,28 @@
 # under the License.
 
 from __future__ import print_function
-from builtins import range
-import airflow
-from airflow.operators.python_operator import PythonOperator
-from airflow.models import DAG
 
 import time
+from builtins import range
 from pprint import pprint
 
+import airflow
+from airflow.models import DAG
+from airflow.operators.python_operator import PythonOperator
+
 args = {
     'owner': 'airflow',
-    'start_date': airflow.utils.dates.days_ago(2)
+    'start_date': airflow.utils.dates.days_ago(2),
 }
 
 dag = DAG(
-    dag_id='example_python_operator', default_args=args,
-    schedule_interval=None)
-
-
-def my_sleeping_function(random_base):
-    """This is a function that will run within the DAG execution"""
-    time.sleep(random_base)
+    dag_id='example_python_operator',
+    default_args=args,
+    schedule_interval=None,
+)
 
 
+# [START howto_operator_python]
 def print_context(ds, **kwargs):
     pprint(kwargs)
     print(ds)
@@ -51,14 +50,25 @@ def print_context(ds, **kwargs):
     task_id='print_the_context',
     provide_context=True,
     python_callable=print_context,
-    dag=dag)
+    dag=dag,
+)
+# [END howto_operator_python]
+
+
+# [START howto_operator_python_kwargs]
+def my_sleeping_function(random_base):
+    """This is a function that will run within the DAG execution"""
+    time.sleep(random_base)
+
 
-# Generate 10 sleeping tasks, sleeping from 0 to 4 seconds respectively
+# Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
 for i in range(5):
     task = PythonOperator(
         task_id='sleep_for_' + str(i),
         python_callable=my_sleeping_function,
         op_kwargs={'random_base': float(i) / 10},
-        dag=dag)
+        dag=dag,
+    )
 
-    task.set_upstream(run_this)
+    run_this >> task
+# [END howto_operator_python_kwargs]
diff --git a/airflow/example_dags/example_short_circuit_operator.py b/airflow/example_dags/example_short_circuit_operator.py
index 8e9565df917da..1093dab616779 100644
--- a/airflow/example_dags/example_short_circuit_operator.py
+++ b/airflow/example_dags/example_short_circuit_operator.py
@@ -17,25 +17,29 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import airflow
-from airflow.operators.python_operator import ShortCircuitOperator
-from airflow.operators.dummy_operator import DummyOperator
-from airflow.models import DAG
 import airflow.utils.helpers
-
+from airflow.models import DAG
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.operators.python_operator import ShortCircuitOperator
 
 args = {
     'owner': 'airflow',
-    'start_date': airflow.utils.dates.days_ago(2)
+    'start_date': airflow.utils.dates.days_ago(2),
 }
 
 dag = DAG(dag_id='example_short_circuit_operator', default_args=args)
 
 cond_true = ShortCircuitOperator(
-    task_id='condition_is_True', python_callable=lambda: True, dag=dag)
+    task_id='condition_is_True',
+    python_callable=lambda: True,
+    dag=dag,
+)
 
 cond_false = ShortCircuitOperator(
-    task_id='condition_is_False', python_callable=lambda: False, dag=dag)
+    task_id='condition_is_False',
+    python_callable=lambda: False,
+    dag=dag,
+)
 
 ds_true = [DummyOperator(task_id='true_' + str(i), dag=dag) for i in [1, 2]]
 ds_false = [DummyOperator(task_id='false_' + str(i), dag=dag) for i in [1, 2]]
diff --git a/airflow/example_dags/example_skip_dag.py b/airflow/example_dags/example_skip_dag.py
index f11ca59338a0a..456eb911dcaa1 100644
--- a/airflow/example_dags/example_skip_dag.py
+++ b/airflow/example_dags/example_skip_dag.py
@@ -18,14 +18,13 @@
 # under the License.
 
 import airflow
-from airflow.operators.dummy_operator import DummyOperator
-from airflow.models import DAG
 from airflow.exceptions import AirflowSkipException
-
+from airflow.models import DAG
+from airflow.operators.dummy_operator import DummyOperator
 
 args = {
     'owner': 'airflow',
-    'start_date': airflow.utils.dates.days_ago(2)
+    'start_date': airflow.utils.dates.days_ago(2),
 }
 
 
@@ -37,23 +36,17 @@ def execute(self, context):
         raise AirflowSkipException
 
 
-dag = DAG(dag_id='example_skip_dag', default_args=args)
-
-
 def create_test_pipeline(suffix, trigger_rule, dag):
-
     skip_operator = DummySkipOperator(task_id='skip_operator_{}'.format(suffix), dag=dag)
-
     always_true = DummyOperator(task_id='always_true_{}'.format(suffix), dag=dag)
-
     join = DummyOperator(task_id=trigger_rule, dag=dag, trigger_rule=trigger_rule)
-
-    join.set_upstream(skip_operator)
-    join.set_upstream(always_true)
-
     final = DummyOperator(task_id='final_{}'.format(suffix), dag=dag)
-    final.set_upstream(join)
+
+    skip_operator >> join
+    always_true >> join
+    join >> final
 
 
+dag = DAG(dag_id='example_skip_dag', default_args=args)
 create_test_pipeline('1', 'all_success', dag)
 create_test_pipeline('2', 'one_success', dag)
diff --git a/airflow/example_dags/example_subdag_operator.py b/airflow/example_dags/example_subdag_operator.py
index ffd254b19a845..98386ba4545e2 100644
--- a/airflow/example_dags/example_subdag_operator.py
+++ b/airflow/example_dags/example_subdag_operator.py
@@ -18,14 +18,11 @@
 # under the License.
 
 import airflow
-
+from airflow.example_dags.subdags.subdag import subdag
 from airflow.models import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.subdag_operator import SubDagOperator
 
-from airflow.example_dags.subdags.subdag import subdag
-
-
 DAG_NAME = 'example_subdag_operator'
 
 args = {
@@ -71,7 +68,4 @@
     dag=dag,
 )
 
-start.set_downstream(section_1)
-section_1.set_downstream(some_other_task)
-some_other_task.set_downstream(section_2)
-section_2.set_downstream(end)
+start >> section_1 >> some_other_task >> section_2 >> end
diff --git a/airflow/example_dags/example_trigger_controller_dag.py b/airflow/example_dags/example_trigger_controller_dag.py
index f5c7218239150..35e7184f76cf0 100644
--- a/airflow/example_dags/example_trigger_controller_dag.py
+++ b/airflow/example_dags/example_trigger_controller_dag.py
@@ -33,11 +33,11 @@
 2. A Target DAG : c.f. example_trigger_target_dag.py
 """
 
-from airflow import DAG
-from airflow.operators.dagrun_operator import TriggerDagRunOperator
+import pprint
 from datetime import datetime
 
-import pprint
+from airflow import DAG
+from airflow.operators.dagrun_operator import TriggerDagRunOperator
 
 pp = pprint.PrettyPrinter(indent=4)
 
@@ -53,16 +53,20 @@ def conditionally_trigger(context, dag_run_obj):
 
 
 # Define the DAG
-dag = DAG(dag_id='example_trigger_controller_dag',
-          default_args={"owner": "airflow",
-                        "start_date": datetime.utcnow()},
-          schedule_interval='@once')
-
+dag = DAG(
+    dag_id='example_trigger_controller_dag',
+    default_args={
+        "owner": "airflow",
+        "start_date": datetime.utcnow(),
+    },
+    schedule_interval='@once',
+)
 
 # Define the single task in this controller example DAG
-trigger = TriggerDagRunOperator(task_id='test_trigger_dagrun',
-                                trigger_dag_id="example_trigger_target_dag",
-                                python_callable=conditionally_trigger,
-                                params={'condition_param': True,
-                                        'message': 'Hello World'},
-                                dag=dag)
+trigger = TriggerDagRunOperator(
+    task_id='test_trigger_dagrun',
+    trigger_dag_id="example_trigger_target_dag",
+    python_callable=conditionally_trigger,
+    params={'condition_param': True, 'message': 'Hello World'},
+    dag=dag,
+)
diff --git a/airflow/example_dags/example_trigger_target_dag.py b/airflow/example_dags/example_trigger_target_dag.py
index 7a656f285964e..c1403a60e1c00 100644
--- a/airflow/example_dags/example_trigger_target_dag.py
+++ b/airflow/example_dags/example_trigger_target_dag.py
@@ -17,12 +17,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import pprint
+from datetime import datetime
+
+from airflow.models import DAG
 from airflow.operators.bash_operator import BashOperator
 from airflow.operators.python_operator import PythonOperator
-from airflow.models import DAG
-from datetime import datetime
 
-import pprint
 pp = pprint.PrettyPrinter(indent=4)
 
 # This example illustrates the use of the TriggerDagRunOperator. There are 2
@@ -50,7 +51,8 @@
 dag = DAG(
     dag_id='example_trigger_target_dag',
     default_args=args,
-    schedule_interval=None)
+    schedule_interval=None,
+)
 
 
 def run_this_func(ds, **kwargs):
@@ -62,12 +64,13 @@ def run_this_func(ds, **kwargs):
     task_id='run_this',
     provide_context=True,
     python_callable=run_this_func,
-    dag=dag)
-
+    dag=dag,
+)
 
 # You can also access the DagRun object in templates
 bash_task = BashOperator(
     task_id="bash_task",
     bash_command='echo "Here is the message: '
                  '{{ dag_run.conf["message"] if dag_run else "" }}" ',
-    dag=dag)
+    dag=dag,
+)
diff --git a/airflow/example_dags/example_xcom.py b/airflow/example_dags/example_xcom.py
index 66bec9a780285..f2b7627aca2f7 100644
--- a/airflow/example_dags/example_xcom.py
+++ b/airflow/example_dags/example_xcom.py
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 from __future__ import print_function
+
 import airflow
 from airflow import DAG
 from airflow.operators.python_operator import PythonOperator
@@ -24,25 +25,22 @@
 args = {
     'owner': 'airflow',
     'start_date': airflow.utils.dates.days_ago(2),
-    'provide_context': True
+    'provide_context': True,
 }
 
-dag = DAG(
-    'example_xcom',
-    schedule_interval="@once",
-    default_args=args)
+dag = DAG('example_xcom', schedule_interval="@once", default_args=args)
 
 value_1 = [1, 2, 3]
 value_2 = {'a': 'b'}
 
 
 def push(**kwargs):
-    # pushes an XCom without a specific target
+    """Pushes an XCom without a specific target"""
     kwargs['ti'].xcom_push(key='value from pusher 1', value=value_1)
 
 
 def push_by_returning(**kwargs):
-    # pushes an XCom without a specific target, just by returning it
+    """Pushes an XCom without a specific target, just by returning it"""
     return value_2
 
 
@@ -63,12 +61,21 @@ def puller(**kwargs):
 
 
 push1 = PythonOperator(
-    task_id='push', dag=dag, python_callable=push)
+    task_id='push',
+    dag=dag,
+    python_callable=push,
+)
 
 push2 = PythonOperator(
-    task_id='push_by_returning', dag=dag, python_callable=push_by_returning)
+    task_id='push_by_returning',
+    dag=dag,
+    python_callable=push_by_returning,
+)
 
 pull = PythonOperator(
-    task_id='puller', dag=dag, python_callable=puller)
+    task_id='puller',
+    dag=dag,
+    python_callable=puller,
+)
 
-pull.set_upstream([push1, push2])
+pull << [push1, push2]
diff --git a/airflow/example_dags/test_utils.py b/airflow/example_dags/test_utils.py
index fb8792a1bf2bc..3fc8af1dada06 100644
--- a/airflow/example_dags/test_utils.py
+++ b/airflow/example_dags/test_utils.py
@@ -18,17 +18,15 @@
 # under the License.
 """Used for unit tests"""
 import airflow
-from airflow.operators.bash_operator import BashOperator
 from airflow.models import DAG
+from airflow.operators.bash_operator import BashOperator
 
-dag = DAG(
-    dag_id='test_utils',
-    schedule_interval=None,
-)
+dag = DAG(dag_id='test_utils', schedule_interval=None)
 
 task = BashOperator(
     task_id='sleeps_forever',
     dag=dag,
     bash_command="sleep 10000000000",
     start_date=airflow.utils.dates.days_ago(2),
-    owner='airflow')
+    owner='airflow',
+)
diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py
index ad817338ef2aa..f87df8b2bca95 100644
--- a/airflow/example_dags/tutorial.py
+++ b/airflow/example_dags/tutorial.py
@@ -20,16 +20,16 @@
 """
 ### Tutorial Documentation
 Documentation that goes along with the Airflow tutorial located
-[here](https://airflow.incubator.apache.org/tutorial.html)
+[here](https://airflow.apache.org/tutorial.html)
 """
+from datetime import timedelta
+
 import airflow
 from airflow import DAG
 from airflow.operators.bash_operator import BashOperator
-from datetime import timedelta
-
 
-# these args will get passed on to each operator
-# you can override them on a per-task basis during operator initialization
+# These args will get passed on to each operator
+# You can override them on a per-task basis during operator initialization
 default_args = {
     'owner': 'airflow',
     'depends_on_past': False,
@@ -45,7 +45,6 @@
     # 'end_date': datetime(2016, 1, 1),
     # 'wait_for_downstream': False,
     # 'dag': dag,
-    # 'adhoc':False,
     # 'sla': timedelta(hours=2),
     # 'execution_timeout': timedelta(seconds=300),
     # 'on_failure_callback': some_function,
@@ -58,13 +57,15 @@
     'tutorial',
     default_args=default_args,
     description='A simple tutorial DAG',
-    schedule_interval=timedelta(days=1))
+    schedule_interval=timedelta(days=1),
+)
 
 # t1, t2 and t3 are examples of tasks created by instantiating operators
 t1 = BashOperator(
     task_id='print_date',
     bash_command='date',
-    dag=dag)
+    dag=dag,
+)
 
 t1.doc_md = """\
 #### Task Documentation
@@ -80,7 +81,8 @@
     task_id='sleep',
     depends_on_past=False,
     bash_command='sleep 5',
-    dag=dag)
+    dag=dag,
+)
 
 templated_command = """
 {% for i in range(5) %}
@@ -95,7 +97,7 @@
     depends_on_past=False,
     bash_command=templated_command,
     params={'my_param': 'Parameter I passed in'},
-    dag=dag)
+    dag=dag,
+)
 
-t2.set_upstream(t1)
-t3.set_upstream(t1)
+t1 >> [t2, t3]
diff --git a/airflow/exceptions.py b/airflow/exceptions.py
index 89f3d0e048da2..41f0a3dda8e38 100644
--- a/airflow/exceptions.py
+++ b/airflow/exceptions.py
@@ -47,6 +47,17 @@ class AirflowSensorTimeout(AirflowException):
     pass
 
 
+class AirflowRescheduleException(AirflowException):
+    """
+    Raise when the task should be re-scheduled at a later time.
+
+    :param reschedule_date: The date when the task should be rescheduled
+    :type reschedule: datetime.datetime
+    """
+    def __init__(self, reschedule_date):
+        self.reschedule_date = reschedule_date
+
+
 class AirflowTaskTimeout(AirflowException):
     pass
 
diff --git a/airflow/executors/base_executor.py b/airflow/executors/base_executor.py
index 04c90884012f7..f88f34591ec4b 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -18,12 +18,15 @@
 # under the License.
 
 from builtins import range
+from collections import OrderedDict
 
+# To avoid circular imports
+import airflow.utils.dag_processing
 from airflow import configuration
+from airflow.settings import Stats
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import State
 
-
 PARALLELISM = configuration.conf.getint('core', 'PARALLELISM')
 
 
@@ -39,7 +42,7 @@ def __init__(self, parallelism=PARALLELISM):
         :type parallelism: int
         """
         self.parallelism = parallelism
-        self.queued_tasks = {}
+        self.queued_tasks = OrderedDict()
         self.running = {}
         self.event_buffer = {}
 
@@ -50,13 +53,13 @@ def start(self):  # pragma: no cover
         """
         pass
 
-    def queue_command(self, task_instance, command, priority=1, queue=None):
-        key = task_instance.key
+    def queue_command(self, simple_task_instance, command, priority=1, queue=None):
+        key = simple_task_instance.key
         if key not in self.queued_tasks and key not in self.running:
             self.log.info("Adding to queue: %s", command)
-            self.queued_tasks[key] = (command, priority, queue, task_instance)
+            self.queued_tasks[key] = (command, priority, queue, simple_task_instance)
         else:
-            self.log.info("could not queue task {}".format(key))
+            self.log.info("could not queue task %s", key)
 
     def queue_task_instance(
             self,
@@ -75,7 +78,7 @@ def queue_task_instance(
         # cfg_path is needed to propagate the config values if using impersonation
         # (run_as_user), given that there are different code paths running tasks.
         # For a long term solution we need to address AIRFLOW-1986
-        command = task_instance.command(
+        command = task_instance.command_as_list(
             local=True,
             mark_success=mark_success,
             ignore_all_deps=ignore_all_deps,
@@ -86,7 +89,7 @@ def queue_task_instance(
             pickle_id=pickle_id,
             cfg_path=cfg_path)
         self.queue_command(
-            task_instance,
+            airflow.utils.dag_processing.SimpleTaskInstance(task_instance),
             command,
             priority=task_instance.task.priority_weight_total,
             queue=task_instance.task.queue)
@@ -115,43 +118,37 @@ def heartbeat(self):
         else:
             open_slots = self.parallelism - len(self.running)
 
-        self.log.debug("%s running task instances", len(self.running))
-        self.log.debug("%s in queue", len(self.queued_tasks))
+        num_running_tasks = len(self.running)
+        num_queued_tasks = len(self.queued_tasks)
+
+        self.log.debug("%s running task instances", num_running_tasks)
+        self.log.debug("%s in queue", num_queued_tasks)
         self.log.debug("%s open slots", open_slots)
 
+        Stats.gauge('executor.open_slots', open_slots)
+        Stats.gauge('executor.queued_tasks', num_queued_tasks)
+        Stats.gauge('executor.running_tasks', num_running_tasks)
+
         sorted_queue = sorted(
             [(k, v) for k, v in self.queued_tasks.items()],
             key=lambda x: x[1][1],
             reverse=True)
         for i in range(min((open_slots, len(self.queued_tasks)))):
-            key, (command, _, queue, ti) = sorted_queue.pop(0)
-            # TODO(jlowin) without a way to know what Job ran which tasks,
-            # there is a danger that another Job started running a task
-            # that was also queued to this executor. This is the last chance
-            # to check if that happened. The most probable way is that a
-            # Scheduler tried to run a task that was originally queued by a
-            # Backfill. This fix reduces the probability of a collision but
-            # does NOT eliminate it.
+            key, (command, _, queue, simple_ti) = sorted_queue.pop(0)
             self.queued_tasks.pop(key)
-            ti.refresh_from_db()
-            if ti.state != State.RUNNING:
-                self.running[key] = command
-                self.execute_async(key=key,
-                                   command=command,
-                                   queue=queue,
-                                   executor_config=ti.executor_config)
-            else:
-                self.logger.info(
-                    'Task is already running, not sending to '
-                    'executor: {}'.format(key))
+            self.running[key] = command
+            self.execute_async(key=key,
+                               command=command,
+                               queue=queue,
+                               executor_config=simple_ti.executor_config)
 
         # Calling child class sync method
         self.log.debug("Calling the %s sync method", self.__class__)
         self.sync()
 
     def change_state(self, key, state):
-        print("popping: {}".format(key))
-        self.running.pop(key)
+        self.log.debug("Changing state: %s", key)
+        self.running.pop(key, None)
         self.event_buffer[key] = state
 
     def fail(self, key):
@@ -175,7 +172,7 @@ def get_event_buffer(self, dag_ids=None):
             self.event_buffer = dict()
         else:
             for key in list(self.event_buffer.keys()):
-                dag_id, _, _ = key
+                dag_id, _, _, _ = key
                 if dag_id in dag_ids:
                     cleared_events[key] = self.event_buffer.pop(key)
 
diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py
index 6cfd2d3769893..09ed425547e12 100644
--- a/airflow/executors/celery_executor.py
+++ b/airflow/executors/celery_executor.py
@@ -17,21 +17,28 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import math
+import os
 import subprocess
 import time
-import os
+import traceback
+from multiprocessing import Pool, cpu_count
 
 from celery import Celery
 from celery import states as celery_states
 
+from airflow import configuration
 from airflow.config_templates.default_celery import DEFAULT_CELERY_CONFIG
 from airflow.exceptions import AirflowException
 from airflow.executors.base_executor import BaseExecutor
-from airflow import configuration
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.module_loading import import_string
+from airflow.utils.timeout import timeout
 
-PARALLELISM = configuration.conf.get('core', 'PARALLELISM')
+# Make it constant for unit test.
+CELERY_FETCH_ERR_MSG_HEADER = 'Error fetching Celery task state'
+
+CELERY_SEND_ERR_MSG_HEADER = 'Error sending Celery task'
 
 '''
 To start the celery worker, run the command:
@@ -51,12 +58,12 @@
 
 
 @app.task
-def execute_command(command):
+def execute_command(command_to_exec):
     log = LoggingMixin().log
-    log.info("Executing command in Celery: %s", command)
+    log.info("Executing command in Celery: %s", command_to_exec)
     env = os.environ.copy()
     try:
-        subprocess.check_call(command, shell=True, stderr=subprocess.STDOUT,
+        subprocess.check_call(command_to_exec, stderr=subprocess.STDOUT,
                               close_fds=True, env=env)
     except subprocess.CalledProcessError as e:
         log.exception('execute_command encountered a CalledProcessError')
@@ -65,6 +72,58 @@ def execute_command(command):
         raise AirflowException('Celery command failed')
 
 
+class ExceptionWithTraceback(object):
+    """
+    Wrapper class used to propagate exceptions to parent processes from subprocesses.
+
+    :param exception: The exception to wrap
+    :type exception: Exception
+    :param exception_traceback: The stacktrace to wrap
+    :type exception_traceback: str
+    """
+
+    def __init__(self, exception, exception_traceback):
+        self.exception = exception
+        self.traceback = exception_traceback
+
+
+def fetch_celery_task_state(celery_task):
+    """
+    Fetch and return the state of the given celery task. The scope of this function is
+    global so that it can be called by subprocesses in the pool.
+
+    :param celery_task: a tuple of the Celery task key and the async Celery object used
+        to fetch the task's state
+    :type celery_task: tuple(str, celery.result.AsyncResult)
+    :return: a tuple of the Celery task key and the Celery state of the task
+    :rtype: tuple[str, str]
+    """
+
+    try:
+        with timeout(seconds=2):
+            # Accessing state property of celery task will make actual network request
+            # to get the current state of the task.
+            res = (celery_task[0], celery_task[1].state)
+    except Exception as e:
+        exception_traceback = "Celery Task ID: {}\n{}".format(celery_task[0],
+                                                              traceback.format_exc())
+        res = ExceptionWithTraceback(e, exception_traceback)
+    return res
+
+
+def send_task_to_executor(task_tuple):
+    key, simple_ti, command, queue, task = task_tuple
+    try:
+        with timeout(seconds=2):
+            result = task.apply_async(args=[command], queue=queue)
+    except Exception as e:
+        exception_traceback = "Celery Task ID: {}\n{}".format(key,
+                                                              traceback.format_exc())
+        result = ExceptionWithTraceback(e, exception_traceback)
+
+    return key, command, result
+
+
 class CeleryExecutor(BaseExecutor):
     """
     CeleryExecutor is recommended for production use of Airflow. It allows
@@ -74,24 +133,147 @@ class CeleryExecutor(BaseExecutor):
     vast amounts of messages, while providing operations with the tools
     required to maintain such a system.
     """
-    def start(self):
+
+    def __init__(self):
+        super(CeleryExecutor, self).__init__()
+
+        # Celery doesn't support querying the state of multiple tasks in parallel
+        # (which can become a bottleneck on bigger clusters) so we use
+        # a multiprocessing pool to speed this up.
+        # How many worker processes are created for checking celery task state.
+        self._sync_parallelism = configuration.getint('celery', 'SYNC_PARALLELISM')
+        if self._sync_parallelism == 0:
+            self._sync_parallelism = max(1, cpu_count() - 1)
+
+        self._sync_pool = None
         self.tasks = {}
         self.last_state = {}
 
-    def execute_async(self, key, command,
-                      queue=DEFAULT_CELERY_CONFIG['task_default_queue'],
-                      executor_config=None):
-        self.log.info("[celery] queuing {key} through celery, "
-                      "queue={queue}".format(**locals()))
-        self.tasks[key] = execute_command.apply_async(
-            args=[command], queue=queue)
-        self.last_state[key] = celery_states.PENDING
+    def start(self):
+        self.log.debug(
+            'Starting Celery Executor using %s processes for syncing',
+            self._sync_parallelism
+        )
+
+    def _num_tasks_per_send_process(self, to_send_count):
+        """
+        How many Celery tasks should each worker process send.
+
+        :return: Number of tasks that should be sent per process
+        :rtype: int
+        """
+        return max(1,
+                   int(math.ceil(1.0 * to_send_count / self._sync_parallelism)))
+
+    def _num_tasks_per_fetch_process(self):
+        """
+        How many Celery tasks should be sent to each worker process.
+
+        :return: Number of tasks that should be used per process
+        :rtype: int
+        """
+        return max(1,
+                   int(math.ceil(1.0 * len(self.tasks) / self._sync_parallelism)))
+
+    def heartbeat(self):
+        # Triggering new jobs
+        if not self.parallelism:
+            open_slots = len(self.queued_tasks)
+        else:
+            open_slots = self.parallelism - len(self.running)
+
+        self.log.debug("%s running task instances", len(self.running))
+        self.log.debug("%s in queue", len(self.queued_tasks))
+        self.log.debug("%s open slots", open_slots)
+
+        sorted_queue = sorted(
+            [(k, v) for k, v in self.queued_tasks.items()],
+            key=lambda x: x[1][1],
+            reverse=True)
+
+        task_tuples_to_send = []
+
+        for i in range(min((open_slots, len(self.queued_tasks)))):
+            key, (command, _, queue, simple_ti) = sorted_queue.pop(0)
+            task_tuples_to_send.append((key, simple_ti, command, queue,
+                                        execute_command))
+
+        cached_celery_backend = None
+        if task_tuples_to_send:
+            tasks = [t[4] for t in task_tuples_to_send]
+
+            # Celery state queries will stuck if we do not use one same backend
+            # for all tasks.
+            cached_celery_backend = tasks[0].backend
+
+        if task_tuples_to_send:
+            # Use chunking instead of a work queue to reduce context switching
+            # since tasks are roughly uniform in size
+            chunksize = self._num_tasks_per_send_process(len(task_tuples_to_send))
+            num_processes = min(len(task_tuples_to_send), self._sync_parallelism)
+
+            send_pool = Pool(processes=num_processes)
+            key_and_async_results = send_pool.map(
+                send_task_to_executor,
+                task_tuples_to_send,
+                chunksize=chunksize)
+
+            send_pool.close()
+            send_pool.join()
+            self.log.debug('Sent all tasks.')
+
+            for key, command, result in key_and_async_results:
+                if isinstance(result, ExceptionWithTraceback):
+                    self.log.error(
+                        CELERY_SEND_ERR_MSG_HEADER + ":%s\n%s\n", result.exception, result.traceback
+                    )
+                elif result is not None:
+                    # Only pops when enqueued successfully, otherwise keep it
+                    # and expect scheduler loop to deal with it.
+                    self.queued_tasks.pop(key)
+                    result.backend = cached_celery_backend
+                    self.running[key] = command
+                    self.tasks[key] = result
+                    self.last_state[key] = celery_states.PENDING
+
+        # Calling child class sync method
+        self.log.debug("Calling the %s sync method", self.__class__)
+        self.sync()
 
     def sync(self):
-        self.log.debug("Inquiring about %s celery task(s)", len(self.tasks))
-        for key, async in list(self.tasks.items()):
+        num_processes = min(len(self.tasks), self._sync_parallelism)
+        if num_processes == 0:
+            self.log.debug("No task to query celery, skipping sync")
+            return
+
+        self.log.debug("Inquiring about %s celery task(s) using %s processes",
+                       len(self.tasks), num_processes)
+
+        # Recreate the process pool each sync in case processes in the pool die
+        self._sync_pool = Pool(processes=num_processes)
+
+        # Use chunking instead of a work queue to reduce context switching since tasks are
+        # roughly uniform in size
+        chunksize = self._num_tasks_per_fetch_process()
+
+        self.log.debug("Waiting for inquiries to complete...")
+        task_keys_to_states = self._sync_pool.map(
+            fetch_celery_task_state,
+            self.tasks.items(),
+            chunksize=chunksize)
+        self._sync_pool.close()
+        self._sync_pool.join()
+        self.log.debug("Inquiries completed.")
+
+        for key_and_state in task_keys_to_states:
+            if isinstance(key_and_state, ExceptionWithTraceback):
+                self.log.error(
+                    CELERY_FETCH_ERR_MSG_HEADER + ", ignoring it:%s\n%s\n",
+                    repr(key_and_state.exception), key_and_state.traceback
+                )
+                continue
+            key, state = key_and_state
             try:
-                state = async.state
                 if self.last_state[key] != state:
                     if state == celery_states.SUCCESS:
                         self.success(key)
@@ -106,16 +288,15 @@ def sync(self):
                         del self.tasks[key]
                         del self.last_state[key]
                     else:
-                        self.log.info("Unexpected state: %s", async.state)
-                    self.last_state[key] = async.state
-            except Exception as e:
-                self.log.error("Error syncing the celery executor, ignoring it:")
-                self.log.exception(e)
+                        self.log.info("Unexpected state: %s", state)
+                        self.last_state[key] = state
+            except Exception:
+                self.log.exception("Error syncing the Celery executor, ignoring it.")
 
     def end(self, synchronous=False):
         if synchronous:
             while any([
-                    async.state not in celery_states.READY_STATES
-                    for async in self.tasks.values()]):
+                    task.state not in celery_states.READY_STATES
+                    for task in self.tasks.values()]):
                 time.sleep(5)
         self.sync()
diff --git a/airflow/executors/dask_executor.py b/airflow/executors/dask_executor.py
index a6ba677f8bd7c..80527a2512545 100644
--- a/airflow/executors/dask_executor.py
+++ b/airflow/executors/dask_executor.py
@@ -43,12 +43,13 @@ def __init__(self, cluster_address=None):
         super(DaskExecutor, self).__init__(parallelism=0)
 
     def start(self):
-        if (self.tls_ca) or (self.tls_key) or (self.tls_cert):
+        if self.tls_ca or self.tls_key or self.tls_cert:
             from distributed.security import Security
             security = Security(
                 tls_client_key=self.tls_key,
                 tls_client_cert=self.tls_cert,
                 tls_ca_file=self.tls_ca,
+                require_encryption=True,
             )
         else:
             security = None
diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py
index 0c85262324283..c9454a677f832 100644
--- a/airflow/executors/local_executor.py
+++ b/airflow/executors/local_executor.py
@@ -75,16 +75,15 @@ def execute_work(self, key, command):
         """
         Executes command received and stores result state in queue.
         :param key: the key to identify the TI
-        :type key: Tuple(dag_id, task_id, execution_date)
+        :type key: tuple(dag_id, task_id, execution_date)
         :param command: the command to execute
-        :type command: string
+        :type command: str
         """
         if key is None:
             return
         self.log.info("%s running %s", self.__class__.__name__, command)
-        command = "exec bash -c '{0}'".format(command)
         try:
-            subprocess.check_call(command, shell=True, close_fds=True)
+            subprocess.check_call(command, close_fds=True)
             state = State.SUCCESS
         except subprocess.CalledProcessError as e:
             state = State.FAILED
@@ -145,9 +144,9 @@ def start(self):
         def execute_async(self, key, command):
             """
             :param key: the key to identify the TI
-            :type key: Tuple(dag_id, task_id, execution_date)
+            :type key: tuple(dag_id, task_id, execution_date)
             :param command: the command to execute
-            :type command: string
+            :type command: str
             """
             local_worker = LocalWorker(self.executor.result_queue)
             local_worker.key = key
@@ -190,9 +189,9 @@ def start(self):
         def execute_async(self, key, command):
             """
             :param key: the key to identify the TI
-            :type key: Tuple(dag_id, task_id, execution_date)
+            :type key: tuple(dag_id, task_id, execution_date)
             :param command: the command to execute
-            :type command: string
+            :type command: str
             """
             self.executor.queue.put((key, command))
 
diff --git a/airflow/executors/sequential_executor.py b/airflow/executors/sequential_executor.py
index 9c0d8ecf0ca6f..1542e3318eb28 100644
--- a/airflow/executors/sequential_executor.py
+++ b/airflow/executors/sequential_executor.py
@@ -45,7 +45,7 @@ def sync(self):
             self.log.info("Executing command: %s", command)
 
             try:
-                subprocess.check_call(command, shell=True, close_fds=True)
+                subprocess.check_call(command, close_fds=True)
                 self.change_state(key, State.SUCCESS)
             except subprocess.CalledProcessError as e:
                 self.change_state(key, State.FAILED)
diff --git a/airflow/hooks/S3_hook.py b/airflow/hooks/S3_hook.py
index b4f3ac3b030e8..a8e4d865e649a 100644
--- a/airflow/hooks/S3_hook.py
+++ b/airflow/hooks/S3_hook.py
@@ -16,6 +16,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from botocore.exceptions import ClientError
 
 from airflow.exceptions import AirflowException
 from airflow.contrib.hooks.aws_hook import AwsHook
@@ -42,7 +43,7 @@ def parse_s3_url(s3url):
         else:
             bucket_name = parsed_url.netloc
             key = parsed_url.path.strip('/')
-            return (bucket_name, key)
+            return bucket_name, key
 
     def check_for_bucket(self, bucket_name):
         """
@@ -54,7 +55,8 @@ def check_for_bucket(self, bucket_name):
         try:
             self.get_conn().head_bucket(Bucket=bucket_name)
             return True
-        except:
+        except ClientError as e:
+            self.log.info(e.response["Error"]["Message"])
             return False
 
     def get_bucket(self, bucket_name):
@@ -67,9 +69,36 @@ def get_bucket(self, bucket_name):
         s3 = self.get_resource_type('s3')
         return s3.Bucket(bucket_name)
 
+    def create_bucket(self, bucket_name, region_name=None):
+        """
+        Creates an Amazon S3 bucket.
+
+        :param bucket_name: The name of the bucket
+        :type bucket_name: str
+        :param region_name: The name of the aws region in which to create the bucket.
+        :type region_name: str
+        """
+        s3_conn = self.get_conn()
+        if not region_name:
+            region_name = s3_conn.meta.region_name
+        if region_name == 'us-east-1':
+            self.get_conn().create_bucket(Bucket=bucket_name)
+        else:
+            self.get_conn().create_bucket(Bucket=bucket_name,
+                                          CreateBucketConfiguration={
+                                              'LocationConstraint': region_name
+                                          })
+
     def check_for_prefix(self, bucket_name, prefix, delimiter):
         """
         Checks that a prefix exists in a bucket
+
+        :param bucket_name: the name of the bucket
+        :type bucket_name: str
+        :param prefix: a key prefix
+        :type prefix: str
+        :param delimiter: the delimiter marks key hierarchy.
+        :type delimiter: str
         """
         prefix = prefix + delimiter if prefix[-1] != delimiter else prefix
         prefix_split = re.split(r'(\w+[{d}])$'.format(d=delimiter), prefix, 1)
@@ -168,7 +197,8 @@ def check_for_key(self, key, bucket_name=None):
         try:
             self.get_conn().head_object(Bucket=bucket_name, Key=key)
             return True
-        except:
+        except ClientError as e:
+            self.log.info(e.response["Error"]["Message"])
             return False
 
     def get_key(self, key, bucket_name=None):
@@ -203,8 +233,8 @@ def read_key(self, key, bucket_name=None):
     def select_key(self, key, bucket_name=None,
                    expression='SELECT * FROM S3Object',
                    expression_type='SQL',
-                   input_serialization={'CSV': {}},
-                   output_serialization={'CSV': {}}):
+                   input_serialization=None,
+                   output_serialization=None):
         """
         Reads a key with S3 Select.
 
@@ -227,6 +257,10 @@ def select_key(self, key, bucket_name=None,
             For more details about S3 Select parameters:
             http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.select_object_content
         """
+        if input_serialization is None:
+            input_serialization = {'CSV': {}}
+        if output_serialization is None:
+            output_serialization = {'CSV': {}}
         if not bucket_name:
             (bucket_name, key) = self.parse_s3_url(key)
 
@@ -238,7 +272,7 @@ def select_key(self, key, bucket_name=None,
             InputSerialization=input_serialization,
             OutputSerialization=output_serialization)
 
-        return ''.join(event['Records']['Payload']
+        return ''.join(event['Records']['Payload'].decode('utf-8')
                        for event in response['Payload']
                        if 'Records' in event)
 
@@ -246,6 +280,13 @@ def check_for_wildcard_key(self,
                                wildcard_key, bucket_name=None, delimiter=''):
         """
         Checks that a key matching a wildcard expression exists in a bucket
+
+        :param wildcard_key: the path to the key
+        :type wildcard_key: str
+        :param bucket_name: the name of the bucket
+        :type bucket_name: str
+        :param delimiter: the delimiter marks key hierarchy
+        :type delimiter: str
         """
         return self.get_wildcard_key(wildcard_key=wildcard_key,
                                      bucket_name=bucket_name,
@@ -259,6 +300,8 @@ def get_wildcard_key(self, wildcard_key, bucket_name=None, delimiter=''):
         :type wildcard_key: str
         :param bucket_name: the name of the bucket
         :type bucket_name: str
+        :param delimiter: the delimiter marks key hierarchy
+        :type delimiter: str
         """
         if not bucket_name:
             (bucket_name, wildcard_key) = self.parse_s3_url(wildcard_key)
@@ -319,7 +362,7 @@ def load_string(self,
         This is provided as a convenience to drop a string in S3. It uses the
         boto infrastructure to ship a file to s3.
 
-        :param string_data: string to set as content for the key.
+        :param string_data: str to set as content for the key.
         :type string_data: str
         :param key: S3 key that will point to the file
         :type key: str
@@ -377,3 +420,124 @@ def load_bytes(self,
 
         client = self.get_conn()
         client.upload_fileobj(filelike_buffer, bucket_name, key, ExtraArgs=extra_args)
+
+    def load_file_obj(self,
+                      file_obj,
+                      key,
+                      bucket_name=None,
+                      replace=False,
+                      encrypt=False):
+        """
+        Loads a file object to S3
+
+        :param file_obj: The file-like object to set as the content for the S3 key.
+        :type file_obj: file-like object
+        :param key: S3 key that will point to the file
+        :type key: str
+        :param bucket_name: Name of the bucket in which to store the file
+        :type bucket_name: str
+        :param replace: A flag that indicates whether to overwrite the key
+            if it already exists.
+        :type replace: bool
+        :param encrypt: If True, S3 encrypts the file on the server,
+            and the file is stored in encrypted form at rest in S3.
+        :type encrypt: bool
+        """
+        if not bucket_name:
+            (bucket_name, key) = self.parse_s3_url(key)
+
+        if not replace and self.check_for_key(key, bucket_name):
+            raise ValueError("The key {key} already exists.".format(key=key))
+
+        extra_args = {}
+        if encrypt:
+            extra_args['ServerSideEncryption'] = "AES256"
+
+        client = self.get_conn()
+        client.upload_fileobj(file_obj, bucket_name, key, ExtraArgs=extra_args)
+
+    def copy_object(self,
+                    source_bucket_key,
+                    dest_bucket_key,
+                    source_bucket_name=None,
+                    dest_bucket_name=None,
+                    source_version_id=None):
+        """
+        Creates a copy of an object that is already stored in S3.
+
+        Note: the S3 connection used here needs to have access to both
+        source and destination bucket/key.
+
+        :param source_bucket_key: The key of the source object.
+
+            It can be either full s3:// style url or relative path from root level.
+
+            When it's specified as a full s3:// url, please omit source_bucket_name.
+        :type source_bucket_key: str
+        :param dest_bucket_key: The key of the object to copy to.
+
+            The convention to specify `dest_bucket_key` is the same
+            as `source_bucket_key`.
+        :type dest_bucket_key: str
+        :param source_bucket_name: Name of the S3 bucket where the source object is in.
+
+            It should be omitted when `source_bucket_key` is provided as a full s3:// url.
+        :type source_bucket_name: str
+        :param dest_bucket_name: Name of the S3 bucket to where the object is copied.
+
+            It should be omitted when `dest_bucket_key` is provided as a full s3:// url.
+        :type dest_bucket_name: str
+        :param source_version_id: Version ID of the source object (OPTIONAL)
+        :type source_version_id: str
+        """
+
+        if dest_bucket_name is None:
+            dest_bucket_name, dest_bucket_key = self.parse_s3_url(dest_bucket_key)
+        else:
+            parsed_url = urlparse(dest_bucket_key)
+            if parsed_url.scheme != '' or parsed_url.netloc != '':
+                raise AirflowException('If dest_bucket_name is provided, ' +
+                                       'dest_bucket_key should be relative path ' +
+                                       'from root level, rather than a full s3:// url')
+
+        if source_bucket_name is None:
+            source_bucket_name, source_bucket_key = self.parse_s3_url(source_bucket_key)
+        else:
+            parsed_url = urlparse(source_bucket_key)
+            if parsed_url.scheme != '' or parsed_url.netloc != '':
+                raise AirflowException('If source_bucket_name is provided, ' +
+                                       'source_bucket_key should be relative path ' +
+                                       'from root level, rather than a full s3:// url')
+
+        CopySource = {'Bucket': source_bucket_name,
+                      'Key': source_bucket_key,
+                      'VersionId': source_version_id}
+        response = self.get_conn().copy_object(Bucket=dest_bucket_name,
+                                               Key=dest_bucket_key,
+                                               CopySource=CopySource)
+        return response
+
+    def delete_objects(self,
+                       bucket,
+                       keys):
+        """
+        :param bucket: Name of the bucket in which you are going to delete object(s)
+        :type bucket: str
+        :param keys: The key(s) to delete from S3 bucket.
+
+            When ``keys`` is a string, it's supposed to be the key name of
+            the single object to delete.
+
+            When ``keys`` is a list, it's supposed to be the list of the
+            keys to delete.
+        :type keys: str or list
+        """
+        if isinstance(keys, list):
+            keys = keys
+        else:
+            keys = [keys]
+
+        delete_dict = {"Objects": [{"Key": k} for k in keys]}
+        response = self.get_conn().delete_objects(Bucket=bucket,
+                                                  Delete=delete_dict)
+        return response
diff --git a/airflow/hooks/base_hook.py b/airflow/hooks/base_hook.py
index 103fa6260b564..c1283e3fb4ceb 100644
--- a/airflow/hooks/base_hook.py
+++ b/airflow/hooks/base_hook.py
@@ -25,7 +25,7 @@
 import os
 import random
 
-from airflow.models import Connection
+from airflow.models.connection import Connection
 from airflow.exceptions import AirflowException
 from airflow.utils.db import provide_session
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -80,7 +80,7 @@ def get_connection(cls, conn_id):
         conn = random.choice(cls.get_connections(conn_id))
         if conn.host:
             log = LoggingMixin().log
-            log.info("Using connection to: %s", conn.host)
+            log.info("Using connection to: %s", conn.debug_info())
         return conn
 
     @classmethod
diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py
index 5b50ade34e486..9f09b6ccc477a 100644
--- a/airflow/hooks/dbapi_hook.py
+++ b/airflow/hooks/dbapi_hook.py
@@ -22,6 +22,7 @@
 from datetime import datetime
 from contextlib import closing
 import sys
+from typing import Optional
 
 from sqlalchemy import create_engine
 
@@ -34,7 +35,7 @@ class DbApiHook(BaseHook):
     Abstract base class for sql hooks.
     """
     # Override to provide the connection name.
-    conn_name_attr = None
+    conn_name_attr = None  # type: Optional[str]
     # Override to have a default connection id for a particular dbHook
     default_conn_name = 'default_conn_id'
     # Override if this db supports autocommit.
@@ -163,10 +164,11 @@ def run(self, sql, autocommit=False, parameters=None):
                 for s in sql:
                     if sys.version_info[0] < 3:
                         s = s.encode('utf-8')
-                    self.log.info(s)
                     if parameters is not None:
+                        self.log.info("{} with parameters {}".format(s, parameters))
                         cur.execute(s, parameters)
                     else:
+                        self.log.info(s)
                         cur.execute(s)
 
             # If autocommit was set to False for db that supports autocommit,
@@ -191,10 +193,11 @@ def get_autocommit(self, conn):
         Return True if conn.autocommit is set to True.
         Return False if conn.autocommit is not set or set to False or conn
         does not support autocommit.
+
         :param conn: Connection to get autocommit setting from.
         :type conn: connection object.
         :return: connection autocommit setting.
-        :rtype bool.
+        :rtype: bool
         """
 
         return getattr(conn, 'autocommit', False) and self.supports_autocommit
@@ -254,12 +257,11 @@ def insert_rows(self, table, rows, target_fields=None, commit_every=1000,
                     if commit_every and i % commit_every == 0:
                         conn.commit()
                         self.log.info(
-                            "Loaded {i} into {table} rows so far".format(**locals())
+                            "Loaded %s into %s rows so far", i, table
                         )
 
             conn.commit()
-        self.log.info(
-            "Done loading. Loaded a total of {i} rows".format(**locals()))
+        self.log.info("Done loading. Loaded a total of %s rows", i)
 
     @staticmethod
     def _serialize_cell(cell, conn=None):
diff --git a/airflow/hooks/druid_hook.py b/airflow/hooks/druid_hook.py
index ef4f2338d29dc..5e2f94f1258dd 100644
--- a/airflow/hooks/druid_hook.py
+++ b/airflow/hooks/druid_hook.py
@@ -35,9 +35,10 @@ class DruidHook(BaseHook):
 
     :param druid_ingest_conn_id: The connection id to the Druid overlord machine
                                  which accepts index jobs
-    :type druid_ingest_conn_id: string
+    :type druid_ingest_conn_id: str
     :param timeout: The interval between polling
-                    the Druid job for the status of the ingestion job
+                    the Druid job for the status of the ingestion job.
+                    Must be greater than or equal to 1
     :type timeout: int
     :param max_ingestion_time: The maximum ingestion time before assuming the job failed
     :type max_ingestion_time: int
@@ -53,6 +54,9 @@ def __init__(
         self.max_ingestion_time = max_ingestion_time
         self.header = {'content-type': 'application/json'}
 
+        if self.timeout < 1:
+            raise ValueError("Druid timeout should be equal or greater than 1")
+
     def get_conn_url(self):
         conn = self.get_connection(self.druid_ingest_conn_id)
         host = conn.host
@@ -64,14 +68,16 @@ def get_conn_url(self):
     def submit_indexing_job(self, json_index_spec):
         url = self.get_conn_url()
 
-        req_index = requests.post(url, data=json_index_spec, headers=self.header)
-        if (req_index.status_code != 200):
+        self.log.info("Druid ingestion spec: %s", json_index_spec)
+        req_index = requests.post(url, json=json_index_spec, headers=self.header)
+        if req_index.status_code != 200:
             raise AirflowException('Did not get 200 when '
                                    'submitting the Druid job to {}'.format(url))
 
         req_json = req_index.json()
         # Wait until the job is completed
         druid_task_id = req_json['task']
+        self.log.info("Druid indexing task-id: %s", druid_task_id)
 
         running = True
 
@@ -81,8 +87,6 @@ def submit_indexing_job(self, json_index_spec):
 
             self.log.info("Job still running for %s seconds...", sec)
 
-            sec = sec + 1
-
             if self.max_ingestion_time and sec > self.max_ingestion_time:
                 # ensure that the job gets killed if the max ingestion time is exceeded
                 requests.post("{0}/{1}/shutdown".format(url, druid_task_id))
@@ -91,6 +95,8 @@ def submit_indexing_job(self, json_index_spec):
 
             time.sleep(self.timeout)
 
+            sec = sec + self.timeout
+
             status = req_status.json()['status']['status']
             if status == 'RUNNING':
                 running = True
@@ -130,8 +136,7 @@ def get_conn(self):
             path=conn.extra_dejson.get('endpoint', '/druid/v2/sql'),
             scheme=conn.extra_dejson.get('schema', 'http')
         )
-        self.log.info('Get the connection to druid '
-                      'broker on {host}'.format(host=conn.host))
+        self.log.info('Get the connection to druid broker on %s', conn.host)
         return druid_broker_conn
 
     def get_uri(self):
diff --git a/airflow/hooks/hdfs_hook.py b/airflow/hooks/hdfs_hook.py
index 3c9136b1fd165..31b2f11501a01 100644
--- a/airflow/hooks/hdfs_hook.py
+++ b/airflow/hooks/hdfs_hook.py
@@ -16,18 +16,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from six import PY2
-
 from airflow import configuration
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
 
-
-snakebite_imported = False
-if PY2:
+try:
     from snakebite.client import Client, HAClient, Namenode, AutoConfigClient
-    snakebite_imported = True
+    snakebite_loaded = True
+except ImportError:
+    snakebite_loaded = False
 
 
 class HDFSHookException(AirflowException):
@@ -39,15 +36,15 @@ class HDFSHook(BaseHook):
     Interact with HDFS. This class is a wrapper around the snakebite library.
 
     :param hdfs_conn_id: Connection id to fetch connection info
-    :type conn_id: string
+    :type hdfs_conn_id: str
     :param proxy_user: effective user for HDFS operations
-    :type proxy_user: string
+    :type proxy_user: str
     :param autoconfig: use snakebite's automatically configured client
     :type autoconfig: bool
     """
     def __init__(self, hdfs_conn_id='hdfs_default', proxy_user=None,
                  autoconfig=False):
-        if not snakebite_imported:
+        if not snakebite_loaded:
             raise ImportError(
                 'This HDFSHook implementation requires snakebite, but '
                 'snakebite is not compatible with Python 3 '
diff --git a/airflow/hooks/hive_hooks.py b/airflow/hooks/hive_hooks.py
index 93e1f45fe6402..2cf22a36498aa 100644
--- a/airflow/hooks/hive_hooks.py
+++ b/airflow/hooks/hive_hooks.py
@@ -21,30 +21,39 @@
 
 import contextlib
 import os
-
-from six.moves import zip
-from past.builtins import basestring, unicode
-
-import unicodecsv as csv
 import re
-import six
 import subprocess
 import time
 from collections import OrderedDict
 from tempfile import NamedTemporaryFile
-import hmsclient
 
-from airflow import configuration as conf
+import six
+import unicodecsv as csv
+from past.builtins import basestring
+from past.builtins import unicode
+from six.moves import zip
+
+import airflow.security.utils as utils
+from airflow import configuration
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
-from airflow.utils.helpers import as_flattened_list
 from airflow.utils.file import TemporaryDirectory
-from airflow import configuration
-import airflow.security.utils as utils
+from airflow.utils.helpers import as_flattened_list
+from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING
 
 HIVE_QUEUE_PRIORITIES = ['VERY_HIGH', 'HIGH', 'NORMAL', 'LOW', 'VERY_LOW']
 
 
+def get_context_from_env_var():
+    """
+    Extract context from env variable, e.g. dag_id, task_id and execution_date,
+    so that they can be used inside BashOperator and PythonOperator.
+    :return: The context of interest.
+    """
+    return {format_map['default']: os.environ.get(format_map['env_var_format'], '')
+            for format_map in AIRFLOW_VAR_NAME_FORMAT_MAPPING.values()}
+
+
 class HiveCliHook(BaseHook):
     """Simple wrapper around the hive CLI.
 
@@ -62,13 +71,13 @@ class HiveCliHook(BaseHook):
     connection string as is.
 
     :param mapred_queue: queue used by the Hadoop Scheduler (Capacity or Fair)
-    :type  mapred_queue: string
+    :type  mapred_queue: str
     :param mapred_queue_priority: priority within the job queue.
         Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW
-    :type  mapred_queue_priority: string
+    :type  mapred_queue_priority: str
     :param mapred_job_name: This name will appear in the jobtracker.
         This can make monitoring easier.
-    :type  mapred_job_name: string
+    :type  mapred_job_name: str
     """
 
     def __init__(
@@ -92,8 +101,8 @@ def __init__(
                     "Invalid Mapred Queue Priority.  Valid values are: "
                     "{}".format(', '.join(HIVE_QUEUE_PRIORITIES)))
 
-        self.mapred_queue = mapred_queue or conf.get('hive',
-                                                     'default_hive_mapred_queue')
+        self.mapred_queue = mapred_queue or configuration.get('hive',
+                                                              'default_hive_mapred_queue')
         self.mapred_queue_priority = mapred_queue_priority
         self.mapred_job_name = mapred_job_name
 
@@ -126,6 +135,7 @@ def _prepare_cli_cmd(self):
                 jdbc_url += ";auth=" + self.auth
 
             jdbc_url = jdbc_url.format(**locals())
+            jdbc_url = '"{}"'.format(jdbc_url)
 
             cmd_extra += ['-u', jdbc_url]
             if conn.login:
@@ -137,7 +147,8 @@ def _prepare_cli_cmd(self):
 
         return [hive_bin] + cmd_extra + hive_params_list
 
-    def _prepare_hiveconf(self, d):
+    @staticmethod
+    def _prepare_hiveconf(d):
         """
         This function prepares a list of hiveconf params
         from a dictionary of key value pairs.
@@ -184,10 +195,15 @@ def run_cli(self, hql, schema=None, verbose=True, hive_conf=None):
 
         with TemporaryDirectory(prefix='airflow_hiveop_') as tmp_dir:
             with NamedTemporaryFile(dir=tmp_dir) as f:
+                hql = hql + '\n'
                 f.write(hql.encode('UTF-8'))
                 f.flush()
                 hive_cmd = self._prepare_cli_cmd()
-                hive_conf_params = self._prepare_hiveconf(hive_conf)
+                env_context = get_context_from_env_var()
+                # Only extend the hive_conf if it is defined.
+                if hive_conf:
+                    env_context.update(hive_conf)
+                hive_conf_params = self._prepare_hiveconf(env_context)
                 if self.mapred_queue:
                     hive_conf_params.extend(
                         ['-hiveconf',
@@ -217,7 +233,7 @@ def run_cli(self, hql, schema=None, verbose=True, hive_conf=None):
                 hive_cmd.extend(['-f', f.name])
 
                 if verbose:
-                    self.log.info(" ".join(hive_cmd))
+                    self.log.info("%s", " ".join(hive_cmd))
                 sp = subprocess.Popen(
                     hive_cmd,
                     stdout=subprocess.PIPE,
@@ -273,7 +289,7 @@ def test_hql(self, hql):
                 except AirflowException as e:
                     message = e.args[0].split('\n')[-2]
                     self.log.info(message)
-                    error_loc = re.search('(\d+):(\d+)', message)
+                    error_loc = re.search(r'(\d+):(\d+)', message)
                     if error_loc and error_loc.group(1).isdigit():
                         lst = int(error_loc.group(1))
                         begin = max(lst - 2, 0)
@@ -298,16 +314,16 @@ def load_df(
         not be sanitized.
 
         :param df: DataFrame to load into a Hive table
-        :type df: DataFrame
+        :type df: pandas.DataFrame
         :param table: target Hive table, use dot notation to target a
             specific database
         :type table: str
         :param field_dict: mapping from column name to hive data type.
             Note that it must be OrderedDict so as to keep columns' order.
-        :type field_dict: OrderedDict
+        :type field_dict: collections.OrderedDict
         :param delimiter: field delimiter in the file
         :type delimiter: str
-        :param encoding: string encoding to use when writing DataFrame to file
+        :param encoding: str encoding to use when writing DataFrame to file
         :type encoding: str
         :param pandas_kwargs: passed to DataFrame.to_csv
         :type pandas_kwargs: dict
@@ -390,7 +406,7 @@ def load_file(
         :param field_dict: A dictionary of the fields name in the file
             as keys and their Hive types as values.
             Note that it must be OrderedDict so as to keep columns' order.
-        :type field_dict: OrderedDict
+        :type field_dict: collections.OrderedDict
         :param create: whether to create the table if it doesn't exist
         :type create: bool
         :param overwrite: whether to overwrite the data in table or partition
@@ -435,11 +451,11 @@ def load_file(
         if partition:
             pvals = ", ".join(
                 ["{0}='{1}'".format(k, v) for k, v in partition.items()])
-            hql += "PARTITION ({pvals});"
+            hql += "PARTITION ({pvals})"
 
         # As a workaround for HIVE-10541, add a newline character
         # at the end of hql (AIRFLOW-2412).
-        hql += '\n'
+        hql += ';\n'
 
         hql = hql.format(**locals())
         self.log.info(hql)
@@ -479,6 +495,7 @@ def get_metastore_client(self):
         """
         Returns a Hive thrift client.
         """
+        import hmsclient
         from thrift.transport import TSocket, TTransport
         from thrift.protocol import TBinaryProtocol
         ms = self.metastore_conn
@@ -519,13 +536,13 @@ def check_for_partition(self, schema, table, partition):
         Checks whether a partition exists
 
         :param schema: Name of hive schema (database) @table belongs to
-        :type schema: string
+        :type schema: str
         :param table: Name of hive table @partition belongs to
-        :type schema: string
+        :type schema: str
         :partition: Expression that matches the partitions to check for
             (eg `a = 'b' AND c = 'd'`)
-        :type schema: string
-        :rtype: boolean
+        :type schema: str
+        :rtype: bool
 
         >>> hh = HiveMetastoreHook()
         >>> t = 'static_babynames_partitioned'
@@ -546,12 +563,12 @@ def check_for_named_partition(self, schema, table, partition_name):
         Checks whether a partition with a given name exists
 
         :param schema: Name of hive schema (database) @table belongs to
-        :type schema: string
+        :type schema: str
         :param table: Name of hive table @partition belongs to
-        :type schema: string
+        :type schema: str
         :partition: Name of the partitions to check for (eg `a=b/c=d`)
-        :type schema: string
-        :rtype: boolean
+        :type schema: str
+        :rtype: bool
 
         >>> hh = HiveMetastoreHook()
         >>> t = 'static_babynames_partitioned'
@@ -635,7 +652,7 @@ def _get_max_partition_from_part_specs(part_specs, partition_key, filter_map):
         :param part_specs: list of partition specs.
         :type part_specs: list
         :param partition_key: partition key name.
-        :type partition_key: string
+        :type partition_key: str
         :param filter_map: partition_key:partition_value map used for partition filtering,
                            e.g. {'key1': 'value1', 'key2': 'value2'}.
                            Only partitions matching all partition_key:partition_value
@@ -675,11 +692,11 @@ def max_partition(self, schema, table_name, field=None, filter_map=None):
         filter out partitions.
 
         :param schema: schema name.
-        :type schema: string
+        :type schema: str
         :param table_name: table name.
-        :type table_name: string
+        :type table_name: str
         :param field: partition key to get max partition from.
-        :type field: string
+        :type field: str
         :param filter_map: partition_key:partition_value map used for partition filtering.
         :type filter_map: map
 
@@ -744,6 +761,9 @@ def __init__(self, hiveserver2_conn_id='hiveserver2_default'):
         self.hiveserver2_conn_id = hiveserver2_conn_id
 
     def get_conn(self, schema=None):
+        """
+        Returns a Hive connection object.
+        """
         db = self.get_connection(self.hiveserver2_conn_id)
         auth_mechanism = db.extra_dejson.get('authMechanism', 'NONE')
         if auth_mechanism == 'NONE' and db.login is None:
@@ -772,7 +792,7 @@ def get_conn(self, schema=None):
             username=db.login or username,
             database=schema or db.schema or 'default')
 
-    def _get_results(self, hql, schema='default', fetch_size=None):
+    def _get_results(self, hql, schema='default', fetch_size=None, hive_conf=None):
         from pyhive.exc import ProgrammingError
         if isinstance(hql, basestring):
             hql = [hql]
@@ -780,12 +800,21 @@ def _get_results(self, hql, schema='default', fetch_size=None):
         with contextlib.closing(self.get_conn(schema)) as conn, \
                 contextlib.closing(conn.cursor()) as cur:
             cur.arraysize = fetch_size or 1000
+
+            env_context = get_context_from_env_var()
+            if hive_conf:
+                env_context.update(hive_conf)
+            for k, v in env_context.items():
+                cur.execute("set {}={}".format(k, v))
+
             for statement in hql:
                 cur.execute(statement)
                 # we only get results of statements that returns
                 lowered_statement = statement.lower().strip()
                 if (lowered_statement.startswith('select') or
-                   lowered_statement.startswith('with')):
+                    lowered_statement.startswith('with') or
+                    (lowered_statement.startswith('set') and
+                     '=' not in lowered_statement)):
                     description = [c for c in cur.description]
                     if previous_description and previous_description != description:
                         message = '''The statements are producing different descriptions:
@@ -805,8 +834,23 @@ def _get_results(self, hql, schema='default', fetch_size=None):
                     except ProgrammingError:
                         self.log.debug("get_results returned no records")
 
-    def get_results(self, hql, schema='default', fetch_size=None):
-        results_iter = self._get_results(hql, schema, fetch_size=fetch_size)
+    def get_results(self, hql, schema='default', fetch_size=None, hive_conf=None):
+        """
+        Get results of the provided hql in target schema.
+
+        :param hql: hql to be executed.
+        :type hql: str or list
+        :param schema: target schema, default to 'default'.
+        :type schema: str
+        :param fetch_size: max size of result to fetch.
+        :type fetch_size: int
+        :param hive_conf: hive_conf to execute alone with the hql.
+        :type hive_conf: dict
+        :return: results of hql execution, dict with data (list of results) and header
+        :rtype: dict
+        """
+        results_iter = self._get_results(hql, schema,
+                                         fetch_size=fetch_size, hive_conf=hive_conf)
         header = next(results_iter)
         results = {
             'data': list(results_iter),
@@ -822,12 +866,36 @@ def to_csv(
             delimiter=',',
             lineterminator='\r\n',
             output_header=True,
-            fetch_size=1000):
+            fetch_size=1000,
+            hive_conf=None):
+        """
+        Execute hql in target schema and write results to a csv file.
+
+        :param hql: hql to be executed.
+        :type hql: str or list
+        :param csv_filepath: filepath of csv to write results into.
+        :type csv_filepath: str
+        :param schema: target schema, default to 'default'.
+        :type schema: str
+        :param delimiter: delimiter of the csv file, default to ','.
+        :type delimiter: str
+        :param lineterminator: lineterminator of the csv file.
+        :type lineterminator: str
+        :param output_header: header of the csv file, default to True.
+        :type output_header: bool
+        :param fetch_size: number of result rows to write into the csv file, default to 1000.
+        :type fetch_size: int
+        :param hive_conf: hive_conf to execute alone with the hql.
+        :type hive_conf: dict
+
+        """
 
-        results_iter = self._get_results(hql, schema, fetch_size=fetch_size)
+        results_iter = self._get_results(hql, schema,
+                                         fetch_size=fetch_size, hive_conf=hive_conf)
         header = next(results_iter)
         message = None
 
+        i = 0
         with open(csv_filepath, 'wb') as f:
             writer = csv.writer(f,
                                 delimiter=delimiter,
@@ -838,7 +906,7 @@ def to_csv(
                     self.log.debug('Cursor description is %s', header)
                     writer.writerow([c[0] for c in header])
 
-                for i, row in enumerate(results_iter):
+                for i, row in enumerate(results_iter, 1):
                     writer.writerow(row)
                     if i % fetch_size == 0:
                         self.log.info("Written %s rows so far.", i)
@@ -856,6 +924,15 @@ def get_records(self, hql, schema='default'):
         """
         Get a set of records from a Hive query.
 
+        :param hql: hql to be executed.
+        :type hql: str or list
+        :param schema: target schema, default to 'default'.
+        :type schema: str
+        :param hive_conf: hive_conf to execute alone with the hql.
+        :type hive_conf: dict
+        :return: result of hive execution
+        :rtype: list
+
         >>> hh = HiveServer2Hook()
         >>> sql = "SELECT * FROM airflow.static_babynames LIMIT 100"
         >>> len(hh.get_records(sql))
@@ -867,11 +944,20 @@ def get_pandas_df(self, hql, schema='default'):
         """
         Get a pandas dataframe from a Hive query
 
+        :param hql: hql to be executed.
+        :type hql: str or list
+        :param schema: target schema, default to 'default'.
+        :type schema: str
+        :return: result of hql execution
+        :rtype: DataFrame
+
         >>> hh = HiveServer2Hook()
         >>> sql = "SELECT * FROM airflow.static_babynames LIMIT 100"
         >>> df = hh.get_pandas_df(sql)
         >>> len(df.index)
         100
+
+        :return: pandas.DateFrame
         """
         import pandas as pd
         res = self.get_results(hql, schema=schema)
diff --git a/airflow/hooks/http_hook.py b/airflow/hooks/http_hook.py
index c449fe0c15ffb..9c8c3854f2200 100644
--- a/airflow/hooks/http_hook.py
+++ b/airflow/hooks/http_hook.py
@@ -29,6 +29,7 @@
 class HttpHook(BaseHook):
     """
     Interact with HTTP servers.
+
     :param http_conn_id: connection that has the base API url i.e https://www.google.com/
         and optional authentication credentials. Default headers can also be specified in
         the Extra field in json format.
@@ -52,6 +53,7 @@ def __init__(
     def get_conn(self, headers=None):
         """
         Returns http session for use with requests
+
         :param headers: additional headers to be passed through as a dictionary
         :type headers: dict
         """
@@ -70,7 +72,10 @@ def get_conn(self, headers=None):
         if conn.login:
             session.auth = (conn.login, conn.password)
         if conn.extra:
-            session.headers.update(conn.extra_dejson)
+            try:
+                session.headers.update(conn.extra_dejson)
+            except TypeError:
+                self.log.warn('Connection to %s has invalid extra field.', conn.host)
         if headers:
             session.headers.update(headers)
 
@@ -79,6 +84,7 @@ def get_conn(self, headers=None):
     def run(self, endpoint, data=None, headers=None, extra_options=None):
         """
         Performs the request
+
         :param endpoint: the endpoint to be called i.e. resource/v1/query?
         :type endpoint: str
         :param data: payload to be uploaded or request parameters
@@ -94,7 +100,11 @@ def run(self, endpoint, data=None, headers=None, extra_options=None):
 
         session = self.get_conn(headers)
 
-        url = self.base_url + endpoint
+        if not self.base_url.endswith('/') and not endpoint.startswith('/'):
+            url = self.base_url + '/' + endpoint
+        else:
+            url = self.base_url + endpoint
+
         req = None
         if self.method == 'GET':
             # GET uses params
@@ -122,6 +132,7 @@ def check_response(self, response):
         """
         Checks the status code and raise an AirflowException exception on non 2XX or 3XX
         status codes
+
         :param response: A requests response object
         :type response: requests.response
         """
@@ -137,6 +148,7 @@ def run_and_check(self, session, prepped_request, extra_options):
         """
         Grabs extra options like timeout and actually runs the request,
         checking for the result
+
         :param session: the session to be used to execute the request
         :type session: requests.Session
         :param prepped_request: the prepared request generated in run()
@@ -171,12 +183,14 @@ def run_with_advanced_retry(self, _retry_args, *args, **kwargs):
         Runs Hook.run() with a Tenacity decorator attached to it. This is useful for
         connectors which might be disturbed by intermittent issues and should not
         instantly fail.
+
         :param _retry_args: Arguments which define the retry behaviour.
             See Tenacity documentation at https://github.com/jd/tenacity
         :type _retry_args: dict
 
 
-        Example: ::
+        :Example::
+
             hook = HttpHook(http_conn_id='my_conn',method='GET')
             retry_args = dict(
                  wait=tenacity.wait_exponential(),
diff --git a/airflow/hooks/mssql_hook.py b/airflow/hooks/mssql_hook.py
index 7ba34892a7d56..6dc3980fd20e2 100644
--- a/airflow/hooks/mssql_hook.py
+++ b/airflow/hooks/mssql_hook.py
@@ -50,3 +50,6 @@ def get_conn(self):
 
     def set_autocommit(self, conn, autocommit):
         conn.autocommit(autocommit)
+
+    def get_autocommit(self, conn):
+        return conn.autocommit_state
diff --git a/airflow/hooks/mysql_hook.py b/airflow/hooks/mysql_hook.py
index c02c0f43b5e37..3e839527ee9a5 100644
--- a/airflow/hooks/mysql_hook.py
+++ b/airflow/hooks/mysql_hook.py
@@ -19,6 +19,8 @@
 
 import MySQLdb
 import MySQLdb.cursors
+import json
+import six
 
 from airflow.hooks.dbapi_hook import DbApiHook
 
@@ -49,10 +51,11 @@ def set_autocommit(self, conn, autocommit):
     def get_autocommit(self, conn):
         """
         MySql connection gets autocommit in a different way.
+
         :param conn: connection to get autocommit setting from.
         :type conn: connection object.
         :return: connection autocommit setting
-        :rtype bool
+        :rtype: bool
         """
         return conn.get_autocommit()
 
@@ -87,7 +90,15 @@ def get_conn(self):
                 conn_config["cursorclass"] = MySQLdb.cursors.SSDictCursor
         local_infile = conn.extra_dejson.get('local_infile', False)
         if conn.extra_dejson.get('ssl', False):
-            conn_config['ssl'] = conn.extra_dejson['ssl']
+            # SSL parameter for MySQL has to be a dictionary and in case
+            # of extra/dejson we can get string if extra is passed via
+            # URL parameters
+            dejson_ssl = conn.extra_dejson['ssl']
+            if isinstance(dejson_ssl, six.string_types):
+                dejson_ssl = json.loads(dejson_ssl)
+            conn_config['ssl'] = dejson_ssl
+        if conn.extra_dejson.get('unix_socket'):
+            conn_config['unix_socket'] = conn.extra_dejson['unix_socket']
         if local_infile:
             conn_config["local_infile"] = 1
         conn = MySQLdb.connect(**conn_config)
@@ -121,7 +132,7 @@ def bulk_dump(self, table, tmp_file):
     def _serialize_cell(cell, conn):
         """
         MySQLdb converts an argument to a literal
-        when passing those seperately to execute. Hence, this method does nothing.
+        when passing those separately to execute. Hence, this method does nothing.
 
         :param cell: The cell to insert into the table
         :type cell: object
diff --git a/airflow/hooks/oracle_hook.py b/airflow/hooks/oracle_hook.py
index 39e447d010793..5d617c9a87073 100644
--- a/airflow/hooks/oracle_hook.py
+++ b/airflow/hooks/oracle_hook.py
@@ -46,24 +46,69 @@ def get_conn(self):
         :param dsn: the host address for the Oracle server
         :param service_name: the db_unique_name of the database
               that you are connecting to (CONNECT_DATA part of TNS)
+
         You can set these parameters in the extra fields of your connection
         as in ``{ "dsn":"some.host.address" , "service_name":"some.service.name" }``
+        see more param detail in
+        `cx_Oracle.connect <https://cx-oracle.readthedocs.io/en/latest/module.html#cx_Oracle.connect>`_
         """
         conn = self.get_connection(self.oracle_conn_id)
+        conn_config = {
+            'user': conn.login,
+            'password': conn.password
+        }
         dsn = conn.extra_dejson.get('dsn', None)
         sid = conn.extra_dejson.get('sid', None)
         mod = conn.extra_dejson.get('module', None)
 
         service_name = conn.extra_dejson.get('service_name', None)
+        port = conn.port if conn.port else 1521
         if dsn and sid and not service_name:
-            dsn = cx_Oracle.makedsn(dsn, conn.port, sid)
-            conn = cx_Oracle.connect(conn.login, conn.password, dsn=dsn)
+            conn_config['dsn'] = cx_Oracle.makedsn(dsn, port, sid)
         elif dsn and service_name and not sid:
-            dsn = cx_Oracle.makedsn(dsn, conn.port, service_name=service_name)
-            conn = cx_Oracle.connect(conn.login, conn.password, dsn=dsn)
+            conn_config['dsn'] = cx_Oracle.makedsn(dsn, port, service_name=service_name)
         else:
-            conn = cx_Oracle.connect(conn.login, conn.password, conn.host)
+            conn_config['dsn'] = conn.host
+
+        if 'encoding' in conn.extra_dejson:
+            conn_config['encoding'] = conn.extra_dejson.get('encoding')
+            # if `encoding` is specific but `nencoding` is not
+            # `nencoding` should use same values as `encoding` to set encoding, inspired by
+            # https://github.com/oracle/python-cx_Oracle/issues/157#issuecomment-371877993
+            if 'nencoding' not in conn.extra_dejson:
+                conn_config['nencoding'] = conn.extra_dejson.get('encoding')
+        if 'nencoding' in conn.extra_dejson:
+            conn_config['nencoding'] = conn.extra_dejson.get('nencoding')
+        if 'threaded' in conn.extra_dejson:
+            conn_config['threaded'] = conn.extra_dejson.get('threaded')
+        if 'events' in conn.extra_dejson:
+            conn_config['events'] = conn.extra_dejson.get('events')
+
+        mode = conn.extra_dejson.get('mode', '').lower()
+        if mode == 'sysdba':
+            conn_config['mode'] = cx_Oracle.SYSDBA
+        elif mode == 'sysasm':
+            conn_config['mode'] = cx_Oracle.SYSASM
+        elif mode == 'sysoper':
+            conn_config['mode'] = cx_Oracle.SYSOPER
+        elif mode == 'sysbkp':
+            conn_config['mode'] = cx_Oracle.SYSBKP
+        elif mode == 'sysdgd':
+            conn_config['mode'] = cx_Oracle.SYSDGD
+        elif mode == 'syskmt':
+            conn_config['mode'] = cx_Oracle.SYSKMT
+        elif mode == 'sysrac':
+            conn_config['mode'] = cx_Oracle.SYSRAC
+
+        purity = conn.extra_dejson.get('purity', '').lower()
+        if purity == 'new':
+            conn_config['purity'] = cx_Oracle.ATTR_PURITY_NEW
+        elif purity == 'self':
+            conn_config['purity'] = cx_Oracle.ATTR_PURITY_SELF
+        elif purity == 'default':
+            conn_config['purity'] = cx_Oracle.ATTR_PURITY_DEFAULT
 
+        conn = cx_Oracle.connect(**conn_config)
         if mod is not None:
             conn.module = mod
 
@@ -74,10 +119,23 @@ def insert_rows(self, table, rows, target_fields=None, commit_every=1000):
         A generic way to insert a set of tuples into a table,
         the whole set of inserts is treated as one transaction
         Changes from standard DbApiHook implementation:
-        - Oracle SQL queries in cx_Oracle can not be terminated with a semicolon (';')
-        - Replace NaN values with NULL using numpy.nan_to_num (not using is_nan()
-          because of input types error for strings)
+
+        - Oracle SQL queries in cx_Oracle can not be terminated with a semicolon (`;`)
+        - Replace NaN values with NULL using `numpy.nan_to_num` (not using
+          `is_nan()` because of input types error for strings)
         - Coerce datetime cells to Oracle DATETIME format during insert
+
+        :param table: target Oracle table, use dot notation to target a
+            specific database
+        :type table: str
+        :param rows: the rows to insert into the table
+        :type rows: iterable of tuples
+        :param target_fields: the names of the columns to fill in the table
+        :type target_fields: iterable of str
+        :param commit_every: the maximum number of rows to insert in one transaction
+            Default 1000, Set greater than 0.
+            Set 1 to insert each row in each single transaction
+        :type commit_every: int
         """
         if target_fields:
             target_fields = ', '.join(target_fields)
@@ -117,25 +175,39 @@ def insert_rows(self, table, rows, target_fields=None, commit_every=1000):
             cur.execute(sql)
             if i % commit_every == 0:
                 conn.commit()
-                self.log.info('Loaded {i} into {table} rows so far'.format(**locals()))
+                self.log.info('Loaded %s into %s rows so far', i, table)
         conn.commit()
         cur.close()
         conn.close()
-        self.log.info('Done loading. Loaded a total of {i} rows'.format(**locals()))
+        self.log.info('Done loading. Loaded a total of %s rows', i)
 
     def bulk_insert_rows(self, table, rows, target_fields=None, commit_every=5000):
         """
         A performant bulk insert for cx_Oracle
         that uses prepared statements via `executemany()`.
         For best performance, pass in `rows` as an iterator.
+
+        :param table: target Oracle table, use dot notation to target a
+            specific database
+        :type table: str
+        :param rows: the rows to insert into the table
+        :type rows: iterable of tuples
+        :param target_fields: the names of the columns to fill in the table, default None.
+            If None, each rows should have some order as table columns name
+        :type target_fields: iterable of str Or None
+        :param commit_every: the maximum number of rows to insert in one transaction
+            Default 5000. Set greater than 0. Set 1 to insert each row in each transaction
+        :type commit_every: int
         """
+        if not rows:
+            raise ValueError("parameter rows could not be None or empty iterable")
         conn = self.get_conn()
         cursor = conn.cursor()
-        values = ', '.join(':%s' % i for i in range(1, len(target_fields) + 1))
-        prepared_stm = 'insert into {tablename} ({columns}) values ({values})'.format(
+        values_base = target_fields if target_fields else rows[0]
+        prepared_stm = 'insert into {tablename} {columns} values ({values})'.format(
             tablename=table,
-            columns=', '.join(target_fields),
-            values=values,
+            columns='({})'.format(', '.join(target_fields)) if target_fields else '',
+            values=', '.join(':%s' % i for i in range(1, len(values_base) + 1)),
         )
         row_count = 0
         # Chunk the rows
diff --git a/airflow/hooks/pig_hook.py b/airflow/hooks/pig_hook.py
index a3836b1ba321f..7cb7d70ca413e 100644
--- a/airflow/hooks/pig_hook.py
+++ b/airflow/hooks/pig_hook.py
@@ -55,7 +55,7 @@ def run_cli(self, pig, verbose=True):
 
         with TemporaryDirectory(prefix='airflow_pigop_') as tmp_dir:
             with NamedTemporaryFile(dir=tmp_dir) as f:
-                f.write(pig)
+                f.write(pig.encode('utf-8'))
                 f.flush()
                 fname = f.name
                 pig_bin = 'pig'
@@ -67,7 +67,7 @@ def run_cli(self, pig, verbose=True):
                     pig_properties_list = self.pig_properties.split()
                     pig_cmd.extend(pig_properties_list)
                 if verbose:
-                    self.log.info(" ".join(pig_cmd))
+                    self.log.info("%s", " ".join(pig_cmd))
                 sp = subprocess.Popen(
                     pig_cmd,
                     stdout=subprocess.PIPE,
@@ -76,8 +76,8 @@ def run_cli(self, pig, verbose=True):
                     close_fds=True)
                 self.sp = sp
                 stdout = ''
-                for line in iter(sp.stdout.readline, ''):
-                    stdout += line
+                for line in iter(sp.stdout.readline, b''):
+                    stdout += line.decode('utf-8')
                     if verbose:
                         self.log.info(line.strip())
                 sp.wait()
diff --git a/airflow/hooks/presto_hook.py b/airflow/hooks/presto_hook.py
index d6b5293fc0f97..20c918c1a4f49 100644
--- a/airflow/hooks/presto_hook.py
+++ b/airflow/hooks/presto_hook.py
@@ -21,6 +21,7 @@
 
 from pyhive import presto
 from pyhive.exc import DatabaseError
+from requests.auth import HTTPBasicAuth
 
 from airflow.hooks.dbapi_hook import DbApiHook
 
@@ -45,18 +46,25 @@ class PrestoHook(DbApiHook):
     def get_conn(self):
         """Returns a connection object"""
         db = self.get_connection(self.presto_conn_id)
+        reqkwargs = None
+        if db.password is not None:
+            reqkwargs = {'auth': HTTPBasicAuth(db.login, db.password)}
         return presto.connect(
             host=db.host,
             port=db.port,
             username=db.login,
+            source=db.extra_dejson.get('source', 'airflow'),
+            protocol=db.extra_dejson.get('protocol', 'http'),
             catalog=db.extra_dejson.get('catalog', 'hive'),
+            requests_kwargs=reqkwargs,
             schema=db.schema)
 
     @staticmethod
     def _strip_sql(sql):
         return sql.strip().rstrip(';')
 
-    def _get_pretty_exception_message(self, e):
+    @staticmethod
+    def _get_pretty_exception_message(e):
         """
         Parses some DatabaseError to provide a better error message
         """
diff --git a/airflow/hooks/slack_hook.py b/airflow/hooks/slack_hook.py
index 17006fc9a5cc9..953ce319b1eee 100644
--- a/airflow/hooks/slack_hook.py
+++ b/airflow/hooks/slack_hook.py
@@ -34,9 +34,9 @@ def __init__(self, token=None, slack_conn_id=None):
         If both supplied, Slack API token will be used.
 
         :param token: Slack API token
-        :type token: string
+        :type token: str
         :param slack_conn_id: connection that has Slack API token in the password field
-        :type slack_conn_id: string
+        :type slack_conn_id: str
         """
         self.token = self.__get_token(token, slack_conn_id)
 
diff --git a/airflow/hooks/webhdfs_hook.py b/airflow/hooks/webhdfs_hook.py
index c4dbe8be6505a..58289ba40653a 100644
--- a/airflow/hooks/webhdfs_hook.py
+++ b/airflow/hooks/webhdfs_hook.py
@@ -66,8 +66,8 @@ def get_conn(self):
                 return client
             except HdfsError as e:
                 self.log.debug(
-                    "Read operation on namenode {nn.host} "
-                    "failed with error: {e}".format(**locals())
+                    "Read operation on namenode %s "
+                    "failed with error: %s", nn.host, e
                 )
         nn_hosts = [c.host for c in nn_connections]
         no_nn_error = "Read operations failed " \
@@ -83,7 +83,7 @@ def check_for_path(self, hdfs_path):
 
     def load_file(self, source, destination, overwrite=True, parallelism=1,
                   **kwargs):
-        """
+        r"""
         Uploads a file to HDFS
 
         :param source: Local path to file or folder. If a folder, all the files
diff --git a/airflow/hooks/zendesk_hook.py b/airflow/hooks/zendesk_hook.py
index 3cf8353344f11..a6f758345018b 100644
--- a/airflow/hooks/zendesk_hook.py
+++ b/airflow/hooks/zendesk_hook.py
@@ -33,7 +33,8 @@ def __init__(self, zendesk_conn_id):
     def get_conn(self):
         conn = self.get_connection(self.__zendesk_conn_id)
         self.__url = "https://" + conn.host
-        return Zendesk(self.__url, conn.login, conn.password, True)
+        return Zendesk(zdesk_url=self.__url, zdesk_email=conn.login, zdesk_password=conn.password,
+                       zdesk_token=True)
 
     def __handle_rate_limit_exception(self, rate_limit_exception):
         """
diff --git a/airflow/jobs.py b/airflow/jobs.py
index ad114abda3d6f..0a403b0818662 100644
--- a/airflow/jobs.py
+++ b/airflow/jobs.py
@@ -26,46 +26,47 @@
 import logging
 import multiprocessing
 import os
-import psutil
 import signal
-import six
 import sys
 import threading
 import time
-import datetime
+from collections import defaultdict, OrderedDict
+from datetime import timedelta
+from time import sleep
+from typing import Any
 
-from collections import defaultdict
+import six
 from past.builtins import basestring
-from sqlalchemy import (
-    Column, Integer, String, func, Index, or_, and_, not_)
+from sqlalchemy import (Column, Index, Integer, String, and_, func, not_, or_)
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm.session import make_transient
-from sqlalchemy_utc import UtcDateTime
-from tabulate import tabulate
-from time import sleep
 
 from airflow import configuration as conf
 from airflow import executors, models, settings
 from airflow.exceptions import AirflowException
-from airflow.models import DAG, DagRun
+from airflow.models import DAG, DagRun, errors
+from airflow.models.dagpickle import DagPickle
+from airflow.models.slamiss import SlaMiss
 from airflow.settings import Stats
 from airflow.task.task_runner import get_task_runner
 from airflow.ti_deps.dep_context import DepContext, QUEUE_DEPS, RUN_DEPS
 from airflow.utils import asciiart, helpers, timezone
+from airflow.utils.configuration import tmp_configuration_copy
 from airflow.utils.dag_processing import (AbstractDagFileProcessor,
-                                          DagFileProcessorManager,
+                                          DagFileProcessorAgent,
                                           SimpleDag,
                                           SimpleDagBag,
+                                          SimpleTaskInstance,
                                           list_py_file_paths)
 from airflow.utils.db import create_session, provide_session
-from airflow.utils.email import send_email
-from airflow.utils.log.logging_mixin import LoggingMixin, set_context, StreamLogWriter
-from airflow.utils.state import State
-from airflow.utils.configuration import tmp_configuration_copy
+from airflow.utils.email import get_email_address_list, send_email
+from airflow.utils.log.logging_mixin import LoggingMixin, StreamLogWriter, set_context
 from airflow.utils.net import get_hostname
+from airflow.utils.sqlalchemy import UtcDateTime
+from airflow.utils.state import State
 
-Base = models.Base
-ID_LEN = models.ID_LEN
+Base = models.base.Base    # type: Any
+ID_LEN = models.base.ID_LEN
 
 
 class BaseJob(Base, LoggingMixin):
@@ -96,6 +97,7 @@ class BaseJob(Base, LoggingMixin):
 
     __table_args__ = (
         Index('job_type_heart', job_type, latest_heartbeat),
+        Index('idx_job_state_heartbeat', state, latest_heartbeat),
     )
 
     def __init__(
@@ -126,7 +128,7 @@ def kill(self, session=None):
         try:
             self.on_kill()
         except Exception as e:
-            self.log.error('on_kill() method failed: {}'.format(e))
+            self.log.error('on_kill() method failed: %s', str(e))
         session.merge(job)
         session.commit()
         raise AirflowException("Job shut down externally.")
@@ -159,32 +161,38 @@ def heartbeat(self):
         heart rate. If you go over 60 seconds before calling it, it won't
         sleep at all.
         """
-        with create_session() as session:
-            job = session.query(BaseJob).filter_by(id=self.id).one()
-            make_transient(job)
-            session.commit()
-
-        if job.state == State.SHUTDOWN:
-            self.kill()
-
-        # Figure out how long to sleep for
-        sleep_for = 0
-        if job.latest_heartbeat:
-            sleep_for = max(
-                0,
-                self.heartrate - (timezone.utcnow() - job.latest_heartbeat).total_seconds())
-
-        sleep(sleep_for)
-
-        # Update last heartbeat time
-        with create_session() as session:
-            job = session.query(BaseJob).filter(BaseJob.id == self.id).first()
-            job.latest_heartbeat = timezone.utcnow()
-            session.merge(job)
-            session.commit()
-
-            self.heartbeat_callback(session=session)
-            self.log.debug('[heartbeat]')
+        try:
+            with create_session() as session:
+                job = session.query(BaseJob).filter_by(id=self.id).one()
+                make_transient(job)
+                session.commit()
+
+            if job.state == State.SHUTDOWN:
+                self.kill()
+
+            is_unit_test = conf.getboolean('core', 'unit_test_mode')
+            if not is_unit_test:
+                # Figure out how long to sleep for
+                sleep_for = 0
+                if job.latest_heartbeat:
+                    seconds_remaining = self.heartrate - \
+                        (timezone.utcnow() - job.latest_heartbeat)\
+                        .total_seconds()
+                    sleep_for = max(0, seconds_remaining)
+
+                sleep(sleep_for)
+
+            # Update last heartbeat time
+            with create_session() as session:
+                job = session.query(BaseJob).filter(BaseJob.id == self.id).first()
+                job.latest_heartbeat = timezone.utcnow()
+                session.merge(job)
+                session.commit()
+
+                self.heartbeat_callback(session=session)
+                self.log.debug('[heartbeat]')
+        except OperationalError as e:
+            self.log.error("Scheduler heartbeat got an exception: %s", str(e))
 
     def run(self):
         Stats.incr(self.__class__.__name__.lower() + '_start', 1, 1)
@@ -197,14 +205,20 @@ def run(self):
             make_transient(self)
             self.id = id_
 
-            # Run
-            self._execute()
-
-            # Marking the success in the DB
-            self.end_date = timezone.utcnow()
-            self.state = State.SUCCESS
-            session.merge(self)
-            session.commit()
+            try:
+                self._execute()
+                # In case of max runs or max duration
+                self.state = State.SUCCESS
+            except SystemExit:
+                # In case of ^C or SIGTERM
+                self.state = State.SUCCESS
+            except Exception:
+                self.state = State.FAILED
+                raise
+            finally:
+                self.end_date = timezone.utcnow()
+                session.merge(self)
+                session.commit()
 
         Stats.incr(self.__class__.__name__.lower() + '_end', 1, 1)
 
@@ -222,9 +236,9 @@ def reset_state_for_orphaned_tasks(self, filter_by_dag_run=None, session=None):
         sequence.
 
         :param filter_by_dag_run: the dag_run we want to process, None if all
-        :type filter_by_dag_run: models.DagRun
+        :type filter_by_dag_run: airflow.models.DagRun
         :return: the TIs reset (in expired SQLAlchemy state)
-        :rtype: List(TaskInstance)
+        :rtype: list[airflow.models.TaskInstance]
         """
         queued_tis = self.executor.queued_tasks
         # also consider running as the state might not have changed in the db yet
@@ -280,7 +294,7 @@ def query(result, items):
                                              self.max_tis_per_query)
 
         task_instance_str = '\n\t'.join(
-            ["{}".format(x) for x in reset_tis])
+            [repr(x) for x in reset_tis])
         session.commit()
 
         self.log.info(
@@ -296,7 +310,7 @@ class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin):
     # Counter that increments everytime an instance of this class is created
     class_creation_counter = 0
 
-    def __init__(self, file_path, pickle_dags, dag_id_white_list):
+    def __init__(self, file_path, pickle_dags, dag_id_white_list, zombies):
         """
         :param file_path: a Python file containing Airflow DAG definitions
         :type file_path: unicode
@@ -304,6 +318,8 @@ def __init__(self, file_path, pickle_dags, dag_id_white_list):
         :type pickle_dags: bool
         :param dag_id_whitelist: If specified, only look at these DAG ID's
         :type dag_id_whitelist: list[unicode]
+        :param zombies: zombie task instances to kill
+        :type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]
         """
         self._file_path = file_path
         # Queue that's used to pass results from the child process.
@@ -312,6 +328,7 @@ def __init__(self, file_path, pickle_dags, dag_id_white_list):
         self._process = None
         self._dag_id_white_list = dag_id_white_list
         self._pickle_dags = pickle_dags
+        self._zombies = zombies
         # The result of Scheduler.process_file(file_path).
         self._result = None
         # Whether the process is done running.
@@ -332,7 +349,8 @@ def _launch_process(result_queue,
                         file_path,
                         pickle_dags,
                         dag_id_white_list,
-                        thread_name):
+                        thread_name,
+                        zombies):
         """
         Launch a process to process the given file.
 
@@ -341,15 +359,17 @@ def _launch_process(result_queue,
         :param file_path: the file to process
         :type file_path: unicode
         :param pickle_dags: whether to pickle the DAGs found in the file and
-        save them to the DB
+            save them to the DB
         :type pickle_dags: bool
         :param dag_id_white_list: if specified, only examine DAG ID's that are
-        in this list
+            in this list
         :type dag_id_white_list: list[unicode]
         :param thread_name: the name to use for the process that is launched
         :type thread_name: unicode
         :return: the process that was launched
         :rtype: multiprocessing.Process
+        :param zombies: zombie task instances to kill
+        :type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]
         """
         def helper():
             # This helper runs in the newly created process
@@ -378,13 +398,14 @@ def helper():
                          os.getpid(), file_path)
                 scheduler_job = SchedulerJob(dag_ids=dag_id_white_list, log=log)
                 result = scheduler_job.process_file(file_path,
+                                                    zombies,
                                                     pickle_dags)
                 result_queue.put(result)
                 end_time = time.time()
                 log.info(
                     "Processing %s took %.3f seconds", file_path, end_time - start_time
                 )
-            except:
+            except Exception:
                 # Log exceptions through the logging framework.
                 log.exception("Got an exception! Propagating...")
                 raise
@@ -410,12 +431,14 @@ def start(self):
             self.file_path,
             self._pickle_dags,
             self._dag_id_white_list,
-            "DagFileProcessor{}".format(self._instance_id))
+            "DagFileProcessor{}".format(self._instance_id),
+            self._zombies)
         self._start_time = timezone.utcnow()
 
     def terminate(self, sigkill=False):
         """
         Terminate (and then kill) the process launched to process the file.
+
         :param sigkill: whether to issue a SIGKILL if SIGTERM doesn't work.
         :type sigkill: bool
         """
@@ -444,6 +467,7 @@ def pid(self):
     def exit_code(self):
         """
         After the process is finished, this can be called to get the return code
+
         :return: the exit code of the process
         :rtype: int
         """
@@ -455,6 +479,7 @@ def exit_code(self):
     def done(self):
         """
         Check if the process launched to process this file is done.
+
         :return: whether the process is finished running
         :rtype: bool
         """
@@ -464,7 +489,8 @@ def done(self):
         if self._done:
             return True
 
-        if not self._result_queue.empty():
+        # In case result queue is corrupted.
+        if self._result_queue and not self._result_queue.empty():
             self._result = self._result_queue.get_nowait()
             self._done = True
             self.log.debug("Waiting for %s", self._process)
@@ -472,7 +498,7 @@ def done(self):
             return True
 
         # Potential error case when process dies
-        if not self._process.is_alive():
+        if self._result_queue and not self._process.is_alive():
             self._done = True
             # Get the object from the queue or else join() can hang.
             if not self._result_queue.empty():
@@ -487,7 +513,7 @@ def done(self):
     def result(self):
         """
         :return: result of running SchedulerJob.process_file()
-        :rtype: SimpleDag
+        :rtype: airflow.utils.dag_processing.SimpleDag
         """
         if not self.done:
             raise AirflowException("Tried to get the result before it's done!")
@@ -523,10 +549,7 @@ def __init__(
             dag_ids=None,
             subdir=settings.DAGS_FOLDER,
             num_runs=-1,
-            file_process_interval=conf.getint('scheduler',
-                                              'min_file_process_interval'),
-            min_file_parsing_loop_time=conf.getint('scheduler',
-                                                   'min_file_parsing_loop_time'),
+            processor_poll_interval=1.0,
             run_duration=None,
             do_pickle=False,
             log=None,
@@ -537,14 +560,18 @@ def __init__(
         :param dag_ids: if specified, only schedule tasks with these DAG IDs
         :type dag_ids: list[unicode]
         :param subdir: directory containing Python files with Airflow DAG
-        definitions, or a specific path to a file
+            definitions, or a specific path to a file
         :type subdir: unicode
         :param num_runs: The number of times to try to schedule each DAG file.
-        -1 for unlimited within the run_duration.
+            -1 for unlimited within the run_duration.
+        :type num_runs: int
+        :param processor_poll_interval: The number of seconds to wait between
+            polls of running processors
+        :type processor_poll_interval: int
         :param run_duration: how long to run (in seconds) before exiting
         :type run_duration: int
         :param do_pickle: once a DAG object is obtained by executing the Python
-        file, whether to serialize the DAG object to the DB
+            file, whether to serialize the DAG object to the DB
         :type do_pickle: bool
         """
         # for BaseJob compatibility
@@ -557,6 +584,7 @@ def __init__(
 
         self.num_runs = num_runs
         self.run_duration = run_duration
+        self._processor_poll_interval = processor_poll_interval
 
         self.do_pickle = do_pickle
         super(SchedulerJob, self).__init__(*args, **kwargs)
@@ -569,30 +597,28 @@ def __init__(
 
         self.using_sqlite = False
         if 'sqlite' in conf.get('core', 'sql_alchemy_conn'):
-            if self.max_threads > 1:
-                self.log.error("Cannot use more than 1 thread when using sqlite. Setting max_threads to 1")
-            self.max_threads = 1
             self.using_sqlite = True
 
-        # How often to scan the DAGs directory for new files. Default to 5 minutes.
-        self.dag_dir_list_interval = conf.getint('scheduler',
-                                                 'dag_dir_list_interval')
-        # How often to print out DAG file processing stats to the log. Default to
-        # 30 seconds.
-        self.print_stats_interval = conf.getint('scheduler',
-                                                'print_stats_interval')
-        # Parse and schedule each file no faster than this interval. Default
-        # to 3 minutes.
-        self.file_process_interval = file_process_interval
-
-        # Wait until at least this many seconds have passed before parsing files once all
-        # files have finished parsing.
-        self.min_file_parsing_loop_time = min_file_parsing_loop_time
-
+        self.max_tis_per_query = conf.getint('scheduler', 'max_tis_per_query')
         if run_duration is None:
             self.run_duration = conf.getint('scheduler',
                                             'run_duration')
 
+        self.processor_agent = None
+        self._last_loop = False
+
+        signal.signal(signal.SIGINT, self._exit_gracefully)
+        signal.signal(signal.SIGTERM, self._exit_gracefully)
+
+    def _exit_gracefully(self, signum, frame):
+        """
+        Helper method to clean up processor_agent to avoid leaving orphan processes.
+        """
+        self.log.info("Exiting gracefully upon receiving signal %s", signum)
+        if self.processor_agent:
+            self.processor_agent.end()
+        sys.exit(os.EX_OK)
+
     @provide_session
     def manage_slas(self, dag, session=None):
         """
@@ -602,11 +628,8 @@ def manage_slas(self, dag, session=None):
         Where assuming that the scheduler runs often, so we only check for
         tasks that should have succeeded in the past hour.
         """
-        if not any([ti.sla for ti in dag.tasks]):
-            self.log.info(
-                "Skipping SLA check for %s because no tasks in DAG have SLAs",
-                dag
-            )
+        if not any([isinstance(ti.sla, timedelta) for ti in dag.tasks]):
+            self.log.info("Skipping SLA check for %s because no tasks in DAG have SLAs", dag)
             return
 
         TI = models.TaskInstance
@@ -631,16 +654,15 @@ def manage_slas(self, dag, session=None):
         ).all()
 
         ts = timezone.utcnow()
-        SlaMiss = models.SlaMiss
         for ti in max_tis:
             task = dag.get_task(ti.task_id)
             dttm = ti.execution_date
-            if task.sla:
+            if isinstance(task.sla, timedelta):
                 dttm = dag.following_schedule(dttm)
                 while dttm < timezone.utcnow():
                     following_schedule = dag.following_schedule(dttm)
                     if following_schedule + task.sla < timezone.utcnow():
-                        session.merge(models.SlaMiss(
+                        session.merge(SlaMiss(
                             task_id=ti.task_id,
                             dag_id=ti.dag_id,
                             execution_date=dttm,
@@ -651,7 +673,7 @@ def manage_slas(self, dag, session=None):
         slas = (
             session
             .query(SlaMiss)
-            .filter(SlaMiss.notification_sent == False)
+            .filter(SlaMiss.notification_sent == False)  # noqa: E712
             .filter(SlaMiss.dag_id == dag.dag_id)
             .all()
         )
@@ -701,16 +723,13 @@ def manage_slas(self, dag, session=None):
             Blocking tasks:
             <pre><code>{blocking_task_list}\n{bug}<code></pre>
             """.format(bug=asciiart.bug, **locals())
-            emails = []
-            for t in dag.tasks:
-                if t.email:
-                    if isinstance(t.email, basestring):
-                        l = [t.email]
-                    elif isinstance(t.email, (list, tuple)):
-                        l = t.email
-                    for email in l:
-                        if email not in emails:
-                            emails.append(email)
+            emails = set()
+            for task in dag.tasks:
+                if task.email:
+                    if isinstance(task.email, basestring):
+                        emails |= set(get_email_address_list(task.email))
+                    elif isinstance(task.email, (list, tuple)):
+                        emails |= set(task.email)
             if emails and len(slas):
                 try:
                     send_email(
@@ -731,25 +750,6 @@ def manage_slas(self, dag, session=None):
                     session.merge(sla)
             session.commit()
 
-    @staticmethod
-    @provide_session
-    def clear_nonexistent_import_errors(session, known_file_paths):
-        """
-        Clears import errors for files that no longer exist.
-
-        :param session: session for ORM operations
-        :type session: sqlalchemy.orm.session.Session
-        :param known_file_paths: The list of existing files that are parsed for DAGs
-        :type known_file_paths: list[unicode]
-        """
-        query = session.query(models.ImportError)
-        if known_file_paths:
-            query = query.filter(
-                ~models.ImportError.filename.in_(known_file_paths)
-            )
-        query.delete(synchronize_session='fetch')
-        session.commit()
-
     @staticmethod
     def update_import_errors(session, dagbag):
         """
@@ -760,17 +760,17 @@ def update_import_errors(session, dagbag):
         :param session: session for ORM operations
         :type session: sqlalchemy.orm.session.Session
         :param dagbag: DagBag containing DAGs with import errors
-        :type dagbag: models.Dagbag
+        :type dagbag: airflow.models.DagBag
         """
         # Clear the errors of the processed files
         for dagbag_file in dagbag.file_last_changed:
-            session.query(models.ImportError).filter(
-                models.ImportError.filename == dagbag_file
+            session.query(errors.ImportError).filter(
+                errors.ImportError.filename == dagbag_file
             ).delete()
 
         # Add the errors of the processed files
         for filename, stacktrace in six.iteritems(dagbag.import_errors):
-            session.add(models.ImportError(
+            session.add(errors.ImportError(
                 filename=filename,
                 stacktrace=stacktrace))
         session.commit()
@@ -779,10 +779,10 @@ def update_import_errors(session, dagbag):
     def create_dag_run(self, dag, session=None):
         """
         This method checks whether a new DagRun needs to be created
-        for a DAG based on scheduling interval
+        for a DAG based on scheduling interval.
         Returns DagRun if one is scheduled. Otherwise returns None.
         """
-        if dag.schedule_interval:
+        if dag.schedule_interval and conf.getboolean('scheduler', 'USE_JOB_SCHEDULE'):
             active_runs = DagRun.find(
                 dag_id=dag.dag_id,
                 state=State.RUNNING,
@@ -811,7 +811,7 @@ def create_dag_run(self, dag, session=None):
                 session.query(func.max(DagRun.execution_date))
                 .filter_by(dag_id=dag.dag_id)
                 .filter(or_(
-                    DagRun.external_trigger == False,
+                    DagRun.external_trigger == False,  # noqa: E712
                     # add % as a wildcard for the like query
                     DagRun.run_id.like(DagRun.ID_PREFIX + '%')
                 ))
@@ -872,8 +872,8 @@ def create_dag_run(self, dag, session=None):
                     dag.start_date, next_run_date
                 )
 
-            # don't ever schedule in the future
-            if next_run_date > timezone.utcnow():
+            # don't ever schedule in the future or if next_run_date is None
+            if not next_run_date or next_run_date > timezone.utcnow():
                 return
 
             # this structure is necessary to avoid a TypeError from concatenating
@@ -948,7 +948,8 @@ def _process_task_instances(self, dag, queue, session=None):
             self.log.debug("Examining active DAG run: %s", run)
             # this needs a fresh session sometimes tis get detached
             tis = run.get_task_instances(state=(State.NONE,
-                                                State.UP_FOR_RETRY))
+                                                State.UP_FOR_RETRY,
+                                                State.UP_FOR_RESCHEDULE))
 
             # this loop is quite slow as it uses are_dependencies_met for
             # every task (in ti.is_runnable). This is also called in
@@ -959,10 +960,6 @@ def _process_task_instances(self, dag, queue, session=None):
                 # fixme: ti.task is transient but needs to be set
                 ti.task = task
 
-                # future: remove adhoc
-                if task.adhoc:
-                    continue
-
                 if ti.are_dependencies_met(
                         dep_context=DepContext(flag_upstream_failed=True),
                         session=session):
@@ -983,12 +980,12 @@ def _change_state_for_tis_without_dagrun(self,
         changed manually.
 
         :param old_states: examine TaskInstances in this state
-        :type old_state: list[State]
+        :type old_state: list[airflow.utils.state.State]
         :param new_state: set TaskInstances to this state
-        :type new_state: State
+        :type new_state: airflow.utils.state.State
         :param simple_dag_bag: TaskInstances associated with DAGs in the
-        simple_dag_bag and with states in the old_state will be examined
-        :type simple_dag_bag: SimpleDagBag
+            simple_dag_bag and with states in the old_state will be examined
+        :type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
         """
         tis_changed = 0
         query = session \
@@ -1033,9 +1030,9 @@ def __get_task_concurrency_map(self, states, session=None):
         Returns a map from tasks to number in the states list given.
 
         :param states: List of states to query for
-        :type states: List[State]
+        :type states: list[airflow.utils.state.State]
         :return: A map from (dag_id, task_id) to count of tasks in states
-        :rtype: Dict[[String, String], Int]
+        :rtype: dict[tuple[str, str], int]
 
         """
         TI = models.TaskInstance
@@ -1058,17 +1055,14 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
         dag concurrency, executor state, and priority.
 
         :param simple_dag_bag: TaskInstances associated with DAGs in the
-        simple_dag_bag will be fetched from the DB and executed
-        :type simple_dag_bag: SimpleDagBag
+            simple_dag_bag will be fetched from the DB and executed
+        :type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
         :param executor: the executor that runs task instances
         :type executor: BaseExecutor
         :param states: Execute TaskInstances in these states
-        :type states: Tuple[State]
-        :return: List[TaskInstance]
+        :type states: tuple[airflow.utils.state.State]
+        :return: list[airflow.models.TaskInstance]
         """
-        # TODO(saguziel): Change this to include QUEUED, for concurrency
-        # purposes we may want to count queued tasks
-        states_to_count_as_running = [State.RUNNING]
         executable_tis = []
 
         # Get all the queued task instances from associated with scheduled
@@ -1081,30 +1075,36 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
             session
             .query(TI)
             .filter(TI.dag_id.in_(simple_dag_bag.dag_ids))
-            .outerjoin(DR,
-                and_(DR.dag_id == TI.dag_id,
-                     DR.execution_date == TI.execution_date))
-            .filter(or_(DR.run_id == None,
+            .outerjoin(
+                DR,
+                and_(DR.dag_id == TI.dag_id, DR.execution_date == TI.execution_date)
+            )
+            .filter(or_(DR.run_id == None,  # noqa: E711
                     not_(DR.run_id.like(BackfillJob.ID_PREFIX + '%'))))
-            .outerjoin(DM, DM.dag_id==TI.dag_id)
-            .filter(or_(DM.dag_id == None,
+            .outerjoin(DM, DM.dag_id == TI.dag_id)
+            .filter(or_(DM.dag_id == None,  # noqa: E711
                     not_(DM.is_paused)))
         )
         if None in states:
-            ti_query = ti_query.filter(or_(TI.state == None, TI.state.in_(states)))
+            ti_query = ti_query.filter(
+                or_(TI.state == None, TI.state.in_(states))  # noqa: E711
+            )
         else:
             ti_query = ti_query.filter(TI.state.in_(states))
 
         task_instances_to_examine = ti_query.all()
 
         if len(task_instances_to_examine) == 0:
-            self.log.info("No tasks to consider for execution.")
+            self.log.debug("No tasks to consider for execution.")
             return executable_tis
 
         # Put one task instance on each line
         task_instance_str = "\n\t".join(
-            ["{}".format(x) for x in task_instances_to_examine])
-        self.log.info("Tasks up for execution:\n\t%s", task_instance_str)
+            [repr(x) for x in task_instances_to_examine])
+        self.log.info(
+            "%s tasks up for execution:\n\t%s", len(task_instances_to_examine),
+            task_instance_str
+        )
 
         # Get the pool settings
         pools = {p.pool: p for p in session.query(models.Pool).all()}
@@ -1113,16 +1113,20 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
         for task_instance in task_instances_to_examine:
             pool_to_task_instances[task_instance.pool].append(task_instance)
 
-        task_concurrency_map = self.__get_task_concurrency_map(states=states_to_count_as_running, session=session)
+        states_to_count_as_running = [State.RUNNING, State.QUEUED]
+        task_concurrency_map = self.__get_task_concurrency_map(
+            states=states_to_count_as_running, session=session)
 
         # Go through each pool, and queue up a task for execution if there are
         # any open slots in the pool.
         for pool, task_instances in pool_to_task_instances.items():
+            pool_name = pool
             if not pool:
                 # Arbitrary:
                 # If queued outside of a pool, trigger no more than
                 # non_pooled_task_slot_count per run
                 open_slots = conf.getint('core', 'non_pooled_task_slot_count')
+                pool_name = 'not_pooled'
             else:
                 if pool not in pools:
                     self.log.warning(
@@ -1135,10 +1139,9 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
 
             num_queued = len(task_instances)
             self.log.info(
-                "Figuring out tasks to run in Pool(name={pool}) with {open_slots} "
-                "open slots and {num_queued} task instances in queue".format(
-                    **locals()
-                )
+                "Figuring out tasks to run in Pool(name=%s) with %s open slots "
+                "and %s task instances in queue",
+                pool, open_slots, num_queued
             )
 
             priority_sorted_task_instances = sorted(
@@ -1147,13 +1150,16 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
             # DAG IDs with running tasks that equal the concurrency limit of the dag
             dag_id_to_possibly_running_task_count = {}
 
-            for task_instance in priority_sorted_task_instances:
+            # Number of tasks that cannot be scheduled because of no open slot in pool
+            num_starving_tasks = 0
+            for current_index, task_instance in enumerate(priority_sorted_task_instances):
                 if open_slots <= 0:
                     self.log.info(
                         "Not scheduling since there are %s open slots in pool %s",
                         open_slots, pool
                     )
                     # Can't schedule any more since there are no more open slots.
+                    num_starving_tasks = len(priority_sorted_task_instances) - current_index
                     break
 
                 # Check to make sure that the task concurrency of the DAG hasn't been
@@ -1162,7 +1168,6 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
                 simple_dag = simple_dag_bag.get_dag(dag_id)
 
                 if dag_id not in dag_id_to_possibly_running_task_count:
-                    # TODO(saguziel): also check against QUEUED state, see AIRFLOW-1104
                     dag_id_to_possibly_running_task_count[dag_id] = \
                         DAG.get_num_task_instances(
                             dag_id,
@@ -1178,15 +1183,20 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
                 )
                 if current_task_concurrency >= task_concurrency_limit:
                     self.log.info(
-                        "Not executing %s since the number of tasks running or queued from DAG %s"
-                        " is >= to the DAG's task concurrency limit of %s",
+                        "Not executing %s since the number of tasks running or queued "
+                        "from DAG %s is >= to the DAG's task concurrency limit of %s",
                         task_instance, dag_id, task_concurrency_limit
                     )
                     continue
 
-                task_concurrency = simple_dag.get_task_special_arg(task_instance.task_id, 'task_concurrency')
+                task_concurrency = simple_dag.get_task_special_arg(
+                    task_instance.task_id,
+                    'task_concurrency')
                 if task_concurrency is not None:
-                    num_running = task_concurrency_map[((task_instance.dag_id, task_instance.task_id))]
+                    num_running = task_concurrency_map[
+                        (task_instance.dag_id, task_instance.task_id)
+                    ]
+
                     if num_running >= task_concurrency:
                         self.log.info("Not executing %s since the task concurrency for"
                                       " this task has been reached.", task_instance)
@@ -1204,9 +1214,13 @@ def _find_executable_task_instances(self, simple_dag_bag, states, session=None):
                 open_slots -= 1
                 dag_id_to_possibly_running_task_count[dag_id] += 1
 
+            Stats.gauge('pool.starving_tasks.{pool_name}'.format(pool_name=pool_name),
+                        num_starving_tasks)
+
         task_instance_str = "\n\t".join(
-            ["{}".format(x) for x in executable_tis])
-        self.log.info("Setting the follow tasks to queued state:\n\t%s", task_instance_str)
+            [repr(x) for x in executable_tis])
+        self.log.info(
+            "Setting the follow tasks to queued state:\n\t%s", task_instance_str)
         # so these dont expire on commit
         for ti in executable_tis:
             copy_dag_id = ti.dag_id
@@ -1223,13 +1237,13 @@ def _change_state_for_executable_task_instances(self, task_instances,
                                                     acceptable_states, session=None):
         """
         Changes the state of task instances in the list with one of the given states
-        to QUEUED atomically, and returns the TIs changed.
+        to QUEUED atomically, and returns the TIs changed in SimpleTaskInstance format.
 
         :param task_instances: TaskInstances to change the state of
-        :type task_instances: List[TaskInstance]
+        :type task_instances: list[airflow.models.TaskInstance]
         :param acceptable_states: Filters the TaskInstances updated to be in these states
         :type acceptable_states: Iterable[State]
-        :return: List[TaskInstance]
+        :rtype: list[airflow.utils.dag_processing.SimpleTaskInstance]
         """
         if len(task_instances) == 0:
             session.commit()
@@ -1248,7 +1262,9 @@ def _change_state_for_executable_task_instances(self, task_instances,
             .filter(or_(*filter_for_ti_state_change)))
 
         if None in acceptable_states:
-            ti_query = ti_query.filter(or_(TI.state == None, TI.state.in_(acceptable_states)))
+            ti_query = ti_query.filter(
+                or_(TI.state == None, TI.state.in_(acceptable_states))  # noqa: E711
+            )
         else:
             ti_query = ti_query.filter(TI.state.in_(acceptable_states))
 
@@ -1269,80 +1285,57 @@ def _change_state_for_executable_task_instances(self, task_instances,
                                          else task_instance.queued_dttm)
             session.merge(task_instance)
 
-        # save which TIs we set before session expires them
-        filter_for_ti_enqueue = ([and_(TI.dag_id == ti.dag_id,
-                                  TI.task_id == ti.task_id,
-                                  TI.execution_date == ti.execution_date)
-                                  for ti in tis_to_set_to_queued])
-        session.commit()
-
-        # requery in batches since above was expired by commit
+        # Generate a list of SimpleTaskInstance for the use of queuing
+        # them in the executor.
+        simple_task_instances = [SimpleTaskInstance(ti) for ti in
+                                 tis_to_set_to_queued]
 
-        def query(result, items):
-            tis_to_be_queued = (
-                session
-                .query(TI)
-                .filter(or_(*items))
-                .all())
-            task_instance_str = "\n\t".join(
-                ["{}".format(x) for x in tis_to_be_queued])
-            self.log.info("Setting the follow tasks to queued state:\n\t%s",
-                          task_instance_str)
-            return result + tis_to_be_queued
-
-        tis_to_be_queued = helpers.reduce_in_chunks(query,
-                                                    filter_for_ti_enqueue,
-                                                    [],
-                                                    self.max_tis_per_query)
+        task_instance_str = "\n\t".join(
+            [repr(x) for x in tis_to_set_to_queued])
 
-        return tis_to_be_queued
+        session.commit()
+        self.log.info("Setting the following %s tasks to queued state:\n\t%s",
+                      len(tis_to_set_to_queued), task_instance_str)
+        return simple_task_instances
 
-    def _enqueue_task_instances_with_queued_state(self, simple_dag_bag, task_instances):
+    def _enqueue_task_instances_with_queued_state(self, simple_dag_bag,
+                                                  simple_task_instances):
         """
         Takes task_instances, which should have been set to queued, and enqueues them
         with the executor.
 
-        :param task_instances: TaskInstances to enqueue
-        :type task_instances: List[TaskInstance]
+        :param simple_task_instances: TaskInstances to enqueue
+        :type simple_task_instances: list[SimpleTaskInstance]
         :param simple_dag_bag: Should contains all of the task_instances' dags
-        :type simple_dag_bag: SimpleDagBag
+        :type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
         """
         TI = models.TaskInstance
         # actually enqueue them
-        for task_instance in task_instances:
-            simple_dag = simple_dag_bag.get_dag(task_instance.dag_id)
-            command = " ".join(TI.generate_command(
-                task_instance.dag_id,
-                task_instance.task_id,
-                task_instance.execution_date,
+        for simple_task_instance in simple_task_instances:
+            simple_dag = simple_dag_bag.get_dag(simple_task_instance.dag_id)
+            command = TI.generate_command(
+                simple_task_instance.dag_id,
+                simple_task_instance.task_id,
+                simple_task_instance.execution_date,
                 local=True,
                 mark_success=False,
                 ignore_all_deps=False,
                 ignore_depends_on_past=False,
                 ignore_task_deps=False,
                 ignore_ti_state=False,
-                pool=task_instance.pool,
+                pool=simple_task_instance.pool,
                 file_path=simple_dag.full_filepath,
-                pickle_id=simple_dag.pickle_id))
+                pickle_id=simple_dag.pickle_id)
 
-            priority = task_instance.priority_weight
-            queue = task_instance.queue
+            priority = simple_task_instance.priority_weight
+            queue = simple_task_instance.queue
             self.log.info(
                 "Sending %s to executor with priority %s and queue %s",
-                task_instance.key, priority, queue
+                simple_task_instance.key, priority, queue
             )
 
-            # save attributes so sqlalchemy doesnt expire them
-            copy_dag_id = task_instance.dag_id
-            copy_task_id = task_instance.task_id
-            copy_execution_date = task_instance.execution_date
-            make_transient(task_instance)
-            task_instance.dag_id = copy_dag_id
-            task_instance.task_id = copy_task_id
-            task_instance.execution_date = copy_execution_date
-
             self.executor.queue_command(
-                task_instance,
+                simple_task_instance,
                 command,
                 priority=priority,
                 queue=queue)
@@ -1362,28 +1355,68 @@ def _execute_task_instances(self,
         3. Enqueue the TIs in the executor.
 
         :param simple_dag_bag: TaskInstances associated with DAGs in the
-        simple_dag_bag will be fetched from the DB and executed
-        :type simple_dag_bag: SimpleDagBag
+            simple_dag_bag will be fetched from the DB and executed
+        :type simple_dag_bag: airflow.utils.dag_processing.SimpleDagBag
         :param states: Execute TaskInstances in these states
-        :type states: Tuple[State]
-        :return: None
+        :type states: tuple[airflow.utils.state.State]
+        :return: Number of task instance with state changed.
         """
         executable_tis = self._find_executable_task_instances(simple_dag_bag, states,
                                                               session=session)
 
         def query(result, items):
-            tis_with_state_changed = self._change_state_for_executable_task_instances(
-                items,
-                states,
-                session=session)
+            simple_tis_with_state_changed = \
+                self._change_state_for_executable_task_instances(items,
+                                                                 states,
+                                                                 session=session)
             self._enqueue_task_instances_with_queued_state(
                 simple_dag_bag,
-                tis_with_state_changed)
+                simple_tis_with_state_changed)
             session.commit()
-            return result + len(tis_with_state_changed)
+            return result + len(simple_tis_with_state_changed)
 
         return helpers.reduce_in_chunks(query, executable_tis, 0, self.max_tis_per_query)
 
+    @provide_session
+    def _change_state_for_tasks_failed_to_execute(self, session):
+        """
+        If there are tasks left over in the executor,
+        we set them back to SCHEDULED to avoid creating hanging tasks.
+
+        :param session: session for ORM operations
+        """
+        if self.executor.queued_tasks:
+            TI = models.TaskInstance
+            filter_for_ti_state_change = (
+                [and_(
+                    TI.dag_id == dag_id,
+                    TI.task_id == task_id,
+                    TI.execution_date == execution_date,
+                    # The TI.try_number will return raw try_number+1 since the
+                    # ti is not running. And we need to -1 to match the DB record.
+                    TI._try_number == try_number - 1,
+                    TI.state == State.QUEUED)
+                    for dag_id, task_id, execution_date, try_number
+                    in self.executor.queued_tasks.keys()])
+            ti_query = (session.query(TI)
+                        .filter(or_(*filter_for_ti_state_change)))
+            tis_to_set_to_scheduled = (ti_query
+                                       .with_for_update()
+                                       .all())
+            if len(tis_to_set_to_scheduled) == 0:
+                session.commit()
+                return
+
+            # set TIs to queued state
+            for task_instance in tis_to_set_to_scheduled:
+                task_instance.state = State.SCHEDULED
+
+            task_instance_str = "\n\t".join(
+                [repr(x) for x in tis_to_set_to_scheduled])
+
+            session.commit()
+            self.log.info("Set the following tasks to scheduled state:\n\t%s", task_instance_str)
+
     def _process_dags(self, dagbag, dags, tis_out):
         """
         Iterates over the dags and processes them. Processing includes:
@@ -1393,12 +1426,12 @@ def _process_dags(self, dagbag, dags, tis_out):
         3. Send emails for tasks that have missed SLAs.
 
         :param dagbag: a collection of DAGs to process
-        :type dagbag: models.DagBag
+        :type dagbag: airflow.models.DagBag
         :param dags: the DAGs from the DagBag to process
-        :type dags: DAG
+        :type dags: airflow.models.DAG
         :param tis_out: A queue to add generated TaskInstance objects
         :type tis_out: multiprocessing.Queue[TaskInstance]
-        :return: None
+        :rtype: None
         """
         for dag in dags:
             dag = dagbag.get_dag(dag.dag_id)
@@ -1418,8 +1451,6 @@ def _process_dags(self, dagbag, dags, tis_out):
             self._process_task_instances(dag, tis_out)
             self.manage_slas(dag)
 
-        models.DagStat.update([d.dag_id for d in dags])
-
     @provide_session
     def _process_executor_events(self, simple_dag_bag, session=None):
         """
@@ -1430,10 +1461,11 @@ def _process_executor_events(self, simple_dag_bag, session=None):
         TI = models.TaskInstance
         for key, state in list(self.executor.get_event_buffer(simple_dag_bag.dag_ids)
                                    .items()):
-            dag_id, task_id, execution_date = key
+            dag_id, task_id, execution_date, try_number = key
             self.log.info(
-                "Executor reports %s.%s execution_date=%s as %s",
-                dag_id, task_id, execution_date, state
+                "Executor reports execution of %s.%s execution_date=%s "
+                "exited with status %s for try_number %s",
+                dag_id, task_id, execution_date, state, try_number
             )
             if state == State.FAILED or state == State.SUCCESS:
                 qry = session.query(TI).filter(TI.dag_id == dag_id,
@@ -1445,7 +1477,7 @@ def _process_executor_events(self, simple_dag_bag, session=None):
                     continue
 
                 # TODO: should we fail RUNNING as well, as we do in Backfills?
-                if ti.state == State.QUEUED:
+                if ti.try_number == try_number and ti.state == State.QUEUED:
                     msg = ("Executor reports task instance {} finished ({}) "
                            "although the task says its {}. Was the task "
                            "killed externally?".format(ti, state, ti.state))
@@ -1464,72 +1496,6 @@ def _process_executor_events(self, simple_dag_bag, session=None):
                         session.merge(ti)
                         session.commit()
 
-    def _log_file_processing_stats(self,
-                                   known_file_paths,
-                                   processor_manager):
-        """
-        Print out stats about how files are getting processed.
-
-        :param known_file_paths: a list of file paths that may contain Airflow
-        DAG definitions
-        :type known_file_paths: list[unicode]
-        :param processor_manager: manager for the file processors
-        :type stats: DagFileProcessorManager
-        :return: None
-        """
-
-        # File Path: Path to the file containing the DAG definition
-        # PID: PID associated with the process that's processing the file. May
-        # be empty.
-        # Runtime: If the process is currently running, how long it's been
-        # running for in seconds.
-        # Last Runtime: If the process ran before, how long did it take to
-        # finish in seconds
-        # Last Run: When the file finished processing in the previous run.
-        headers = ["File Path",
-                   "PID",
-                   "Runtime",
-                   "Last Runtime",
-                   "Last Run"]
-
-        rows = []
-        for file_path in known_file_paths:
-            last_runtime = processor_manager.get_last_runtime(file_path)
-            processor_pid = processor_manager.get_pid(file_path)
-            processor_start_time = processor_manager.get_start_time(file_path)
-            runtime = ((timezone.utcnow() - processor_start_time).total_seconds()
-                       if processor_start_time else None)
-            last_run = processor_manager.get_last_finish_time(file_path)
-
-            rows.append((file_path,
-                         processor_pid,
-                         runtime,
-                         last_runtime,
-                         last_run))
-
-        # Sort by longest last runtime. (Can't sort None values in python3)
-        rows = sorted(rows, key=lambda x: x[3] or 0.0)
-
-        formatted_rows = []
-        for file_path, pid, runtime, last_runtime, last_run in rows:
-            formatted_rows.append((file_path,
-                                   pid,
-                                   "{:.2f}s".format(runtime)
-                                   if runtime else None,
-                                   "{:.2f}s".format(last_runtime)
-                                   if last_runtime else None,
-                                   last_run.strftime("%Y-%m-%dT%H:%M:%S")
-                                   if last_run else None))
-        log_str = ("\n" +
-                   "=" * 80 +
-                   "\n" +
-                   "DAG File Processing Stats\n\n" +
-                   tabulate(formatted_rows, headers=headers) +
-                   "\n" +
-                   "=" * 80)
-
-        self.log.info(log_str)
-
     def _execute(self):
         self.log.info("Starting the scheduler")
 
@@ -1539,100 +1505,67 @@ def _execute(self):
                 (executors.LocalExecutor, executors.SequentialExecutor):
             pickle_dags = True
 
-        # Use multiple processes to parse and generate tasks for the
-        # DAGs in parallel. By processing them in separate processes,
-        # we can get parallelism and isolation from potentially harmful
-        # user code.
-        self.log.info("Processing files using up to %s processes at a time",
-                      self.max_threads)
         self.log.info("Running execute loop for %s seconds", self.run_duration)
         self.log.info("Processing each file at most %s times", self.num_runs)
-        self.log.info("Process each file at most once every %s seconds",
-                      self.file_process_interval)
-        self.log.info("Wait until at least %s seconds have passed between file parsing "
-                      "loops", self.min_file_parsing_loop_time)
-        self.log.info("Checking for new files in %s every %s seconds",
-                      self.subdir, self.dag_dir_list_interval)
 
         # Build up a list of Python files that could contain DAGs
         self.log.info("Searching for files in %s", self.subdir)
         known_file_paths = list_py_file_paths(self.subdir)
         self.log.info("There are %s files in %s", len(known_file_paths), self.subdir)
 
-        def processor_factory(file_path):
+        def processor_factory(file_path, zombies):
             return DagFileProcessor(file_path,
                                     pickle_dags,
-                                    self.dag_ids)
+                                    self.dag_ids,
+                                    zombies)
+
+        # When using sqlite, we do not use async_mode
+        # so the scheduler job and DAG parser don't access the DB at the same time.
+        async_mode = not self.using_sqlite
 
-        processor_manager = DagFileProcessorManager(self.subdir,
-                                                    known_file_paths,
-                                                    self.max_threads,
-                                                    self.file_process_interval,
-                                                    self.min_file_parsing_loop_time,
-                                                    self.num_runs,
-                                                    processor_factory)
+        self.processor_agent = DagFileProcessorAgent(self.subdir,
+                                                     known_file_paths,
+                                                     self.num_runs,
+                                                     processor_factory,
+                                                     async_mode)
 
         try:
-            self._execute_helper(processor_manager)
+            self._execute_helper()
+        except Exception:
+            self.log.exception("Exception when executing execute_helper")
         finally:
+            self.processor_agent.end()
             self.log.info("Exited execute loop")
 
-            # Kill all child processes on exit since we don't want to leave
-            # them as orphaned.
-            pids_to_kill = processor_manager.get_all_pids()
-            if len(pids_to_kill) > 0:
-                # First try SIGTERM
-                this_process = psutil.Process(os.getpid())
-                # Only check child processes to ensure that we don't have a case
-                # where we kill the wrong process because a child process died
-                # but the PID got reused.
-                child_processes = [x for x in this_process.children(recursive=True)
-                                   if x.is_running() and x.pid in pids_to_kill]
-                for child in child_processes:
-                    self.log.info("Terminating child PID: %s", child.pid)
-                    child.terminate()
-                # TODO: Remove magic number
-                timeout = 5
-                self.log.info("Waiting up to %s seconds for processes to exit...", timeout)
-                try:
-                    psutil.wait_procs(
-                        child_processes, timeout=timeout,
-                        callback=lambda x: self.log.info('Terminated PID %s', x.pid))
-                except psutil.TimeoutExpired:
-                    self.log.debug("Ran out of time while waiting for processes to exit")
-
-                # Then SIGKILL
-                child_processes = [x for x in this_process.children(recursive=True)
-                                   if x.is_running() and x.pid in pids_to_kill]
-                if len(child_processes) > 0:
-                    self.log.info("SIGKILL processes that did not terminate gracefully")
-                    for child in child_processes:
-                        self.log.info("Killing child PID: %s", child.pid)
-                        child.kill()
-                        child.wait()
-
-    def _execute_helper(self, processor_manager):
-        """
-        :param processor_manager: manager to use
-        :type processor_manager: DagFileProcessorManager
-        :return: None
+    def _execute_helper(self):
+        """
+        The actual scheduler loop. The main steps in the loop are:
+            #. Harvest DAG parsing results through DagFileProcessorAgent
+            #. Find and queue executable tasks
+                #. Change task instance state in DB
+                #. Queue tasks in executor
+            #. Heartbeat executor
+                #. Execute queued tasks in executor asynchronously
+                #. Sync on the states of running tasks
+
+        Following is a graphic representation of these steps.
+
+        .. image:: ../docs/img/scheduler_loop.jpg
+
+        :rtype: None
         """
         self.executor.start()
 
         self.log.info("Resetting orphaned tasks for active dag runs")
         self.reset_state_for_orphaned_tasks()
 
+        # Start after resetting orphaned tasks to avoid stressing out DB.
+        self.processor_agent.start()
+
         execute_start_time = timezone.utcnow()
 
-        # Last time stats were printed
-        last_stat_print_time = datetime.datetime(2000, 1, 1, tzinfo=timezone.utc)
         # Last time that self.heartbeat() was called.
         last_self_heartbeat_time = timezone.utcnow()
-        # Last time that the DAG dir was traversed to look for files
-        last_dag_dir_refresh_time = timezone.utcnow()
-
-        # Use this value initially
-        known_file_paths = processor_manager.file_paths
 
         # For the execute duration, parse and schedule DAGs
         while (timezone.utcnow() - execute_start_time).total_seconds() < \
@@ -1640,61 +1573,56 @@ def _execute_helper(self, processor_manager):
             self.log.debug("Starting Loop...")
             loop_start_time = time.time()
 
-            # Traverse the DAG directory for Python files containing DAGs
-            # periodically
-            elapsed_time_since_refresh = (timezone.utcnow() -
-                                          last_dag_dir_refresh_time).total_seconds()
-
-            if elapsed_time_since_refresh > self.dag_dir_list_interval:
-                # Build up a list of Python files that could contain DAGs
-                self.log.info("Searching for files in %s", self.subdir)
-                known_file_paths = list_py_file_paths(self.subdir)
-                last_dag_dir_refresh_time = timezone.utcnow()
-                self.log.info("There are %s files in %s", len(known_file_paths), self.subdir)
-                processor_manager.set_file_paths(known_file_paths)
-
-                self.log.debug("Removing old import errors")
-                self.clear_nonexistent_import_errors(known_file_paths=known_file_paths)
-
-            # Kick of new processes and collect results from finished ones
-            self.log.debug("Heartbeating the process manager")
-            simple_dags = processor_manager.heartbeat()
-
             if self.using_sqlite:
+                self.processor_agent.heartbeat()
                 # For the sqlite case w/ 1 thread, wait until the processor
                 # is finished to avoid concurrent access to the DB.
-                self.log.debug("Waiting for processors to finish since we're using sqlite")
-                processor_manager.wait_until_finished()
+                self.log.debug(
+                    "Waiting for processors to finish since we're using sqlite")
+                self.processor_agent.wait_until_finished()
+
+            self.log.debug("Harvesting DAG parsing results")
+            simple_dags = self.processor_agent.harvest_simple_dags()
+            self.log.debug("Harvested {} SimpleDAGs".format(len(simple_dags)))
 
             # Send tasks for execution if available
             simple_dag_bag = SimpleDagBag(simple_dags)
             if len(simple_dags) > 0:
-
-                # Handle cases where a DAG run state is set (perhaps manually) to
-                # a non-running state. Handle task instances that belong to
-                # DAG runs in those states
-
-                # If a task instance is up for retry but the corresponding DAG run
-                # isn't running, mark the task instance as FAILED so we don't try
-                # to re-run it.
-                self._change_state_for_tis_without_dagrun(simple_dag_bag,
-                                                          [State.UP_FOR_RETRY],
-                                                          State.FAILED)
-                # If a task instance is scheduled or queued, but the corresponding
-                # DAG run isn't running, set the state to NONE so we don't try to
-                # re-run it.
-                self._change_state_for_tis_without_dagrun(simple_dag_bag,
-                                                          [State.QUEUED,
-                                                           State.SCHEDULED],
-                                                          State.NONE)
-
-                self._execute_task_instances(simple_dag_bag,
-                                             (State.SCHEDULED,))
+                try:
+                    simple_dag_bag = SimpleDagBag(simple_dags)
+
+                    # Handle cases where a DAG run state is set (perhaps manually) to
+                    # a non-running state. Handle task instances that belong to
+                    # DAG runs in those states
+
+                    # If a task instance is up for retry but the corresponding DAG run
+                    # isn't running, mark the task instance as FAILED so we don't try
+                    # to re-run it.
+                    self._change_state_for_tis_without_dagrun(simple_dag_bag,
+                                                              [State.UP_FOR_RETRY],
+                                                              State.FAILED)
+                    # If a task instance is scheduled or queued or up for reschedule,
+                    # but the corresponding DAG run isn't running, set the state to
+                    # NONE so we don't try to re-run it.
+                    self._change_state_for_tis_without_dagrun(simple_dag_bag,
+                                                              [State.QUEUED,
+                                                               State.SCHEDULED,
+                                                               State.UP_FOR_RESCHEDULE],
+                                                              State.NONE)
+
+                    self._execute_task_instances(simple_dag_bag,
+                                                 (State.SCHEDULED,))
+                except Exception as e:
+                    self.log.error("Error queuing tasks")
+                    self.log.exception(e)
+                    continue
 
             # Call heartbeats
             self.log.debug("Heartbeating the executor")
             self.executor.heartbeat()
 
+            self._change_state_for_tasks_failed_to_execute()
+
             # Process events from the executor
             self._process_executor_events(simple_dag_bag)
 
@@ -1706,36 +1634,42 @@ def _execute_helper(self, processor_manager):
                 self.heartbeat()
                 last_self_heartbeat_time = timezone.utcnow()
 
-            # Occasionally print out stats about how fast the files are getting processed
-            if ((timezone.utcnow() - last_stat_print_time).total_seconds() >
-                    self.print_stats_interval):
-                if len(known_file_paths) > 0:
-                    self._log_file_processing_stats(known_file_paths,
-                                                    processor_manager)
-                last_stat_print_time = timezone.utcnow()
-
+            is_unit_test = conf.getboolean('core', 'unit_test_mode')
             loop_end_time = time.time()
-            self.log.debug("Ran scheduling loop in %.2f seconds",
-                           loop_end_time - loop_start_time)
-
-            # Exit early for a test mode
-            if processor_manager.max_runs_reached():
-                self.log.info("Exiting loop as all files have been processed %s times",
-                              self.num_runs)
+            loop_duration = loop_end_time - loop_start_time
+            self.log.debug(
+                "Ran scheduling loop in %.2f seconds",
+                loop_duration)
+
+            if not is_unit_test:
+                self.log.debug("Sleeping for %.2f seconds", self._processor_poll_interval)
+                time.sleep(self._processor_poll_interval)
+
+            # Exit early for a test mode, run one additional scheduler loop
+            # to reduce the possibility that parsed DAG was put into the queue
+            # by the DAG manager but not yet received by DAG agent.
+            if self.processor_agent.done:
+                self._last_loop = True
+
+            if self._last_loop:
+                self.log.info("Exiting scheduler loop as all files"
+                              " have been processed {} times".format(self.num_runs))
                 break
 
+            if loop_duration < 1 and not is_unit_test:
+                sleep_length = 1 - loop_duration
+                self.log.debug(
+                    "Sleeping for {0:.2f} seconds to prevent excessive logging"
+                    .format(sleep_length))
+                sleep(sleep_length)
+
         # Stop any processors
-        processor_manager.terminate()
+        self.processor_agent.terminate()
 
         # Verify that all files were processed, and if so, deactivate DAGs that
         # haven't been touched by the scheduler as they likely have been
         # deleted.
-        all_files_processed = True
-        for file_path in known_file_paths:
-            if processor_manager.get_last_finish_time(file_path) is None:
-                all_files_processed = False
-                break
-        if all_files_processed:
+        if self.processor_agent.all_files_processed:
             self.log.info(
                 "Deactivating DAGs that haven't been touched since %s",
                 execute_start_time.isoformat()
@@ -1747,7 +1681,7 @@ def _execute_helper(self, processor_manager):
         settings.Session.remove()
 
     @provide_session
-    def process_file(self, file_path, pickle_dags=False, session=None):
+    def process_file(self, file_path, zombies, pickle_dags=False, session=None):
         """
         Process a Python file containing Airflow DAGs.
 
@@ -1766,18 +1700,20 @@ def process_file(self, file_path, pickle_dags=False, session=None):
 
         :param file_path: the path to the Python file that should be executed
         :type file_path: unicode
+        :param zombies: zombie task instances to kill.
+        :type zombies: list[airflow.utils.dag_processing.SimpleTaskInstance]
         :param pickle_dags: whether serialize the DAGs found in the file and
-        save them to the db
+            save them to the db
         :type pickle_dags: bool
         :return: a list of SimpleDags made from the Dags found in the file
-        :rtype: list[SimpleDag]
+        :rtype: list[airflow.utils.dag_processing.SimpleDagBag]
         """
         self.log.info("Processing file %s for tasks to queue", file_path)
         # As DAGs are parsed from this file, they will be converted into SimpleDags
         simple_dags = []
 
         try:
-            dagbag = models.DagBag(file_path)
+            dagbag = models.DagBag(file_path, include_examples=False)
         except Exception:
             self.log.exception("Failed at reloading the DAG file %s", file_path)
             Stats.incr('dag_file_refresh_error', 1, 1)
@@ -1799,13 +1735,12 @@ def process_file(self, file_path, pickle_dags=False, session=None):
 
         # Pickle the DAGs (if necessary) and put them into a SimpleDag
         for dag_id in dagbag.dags:
-            dag = dagbag.get_dag(dag_id)
-            pickle_id = None
-            if pickle_dags:
-                pickle_id = dag.pickle(session).id
-
             # Only return DAGs that are not paused
             if dag_id not in paused_dag_ids:
+                dag = dagbag.get_dag(dag_id)
+                pickle_id = None
+                if pickle_dags:
+                    pickle_id = dag.pickle(session).id
                 simple_dags.append(SimpleDag(dag, pickle_id=pickle_id))
 
         if len(self.dag_ids) > 0:
@@ -1860,7 +1795,7 @@ def process_file(self, file_path, pickle_dags=False, session=None):
         except Exception:
             self.log.exception("Error logging import errors!")
         try:
-            dagbag.kill_zombies()
+            dagbag.kill_zombies(zombies)
         except Exception:
             self.log.exception("Error killing zombies!")
 
@@ -1868,7 +1803,7 @@ def process_file(self, file_path, pickle_dags=False, session=None):
 
     @provide_session
     def heartbeat_callback(self, session=None):
-        Stats.gauge('scheduler_heartbeat', 1, 1)
+        Stats.incr('scheduler_heartbeat', 1, 1)
 
 
 class BackfillJob(BaseJob):
@@ -1910,29 +1845,29 @@ def __init__(self,
                      ):
             """
             :param to_run: Tasks to run in the backfill
-            :type to_run: dict[Tuple[String, String, DateTime], TaskInstance]
+            :type to_run: dict[tuple[string, string, datetime.datetime], airflow.models.TaskInstance]
             :param running: Maps running task instance key to task instance object
-            :type running: dict[Tuple[String, String, DateTime], TaskInstance]
+            :type running: dict[tuple[string, string, datetime.datetime], airflow.models.TaskInstance]
             :param skipped: Tasks that have been skipped
-            :type skipped: set[Tuple[String, String, DateTime]]
+            :type skipped: set[tuple[string, string, datetime.datetime]]
             :param succeeded: Tasks that have succeeded so far
-            :type succeeded: set[Tuple[String, String, DateTime]]
+            :type succeeded: set[tuple[string, string, datetime.datetime]]
             :param failed: Tasks that have failed
-            :type failed: set[Tuple[String, String, DateTime]]
+            :type failed: set[tuple[string, string, datetime.datetime]]
             :param not_ready: Tasks not ready for execution
-            :type not_ready: set[Tuple[String, String, DateTime]]
+            :type not_ready: set[tuple[string, string, datetime.datetime]]
             :param deadlocked: Deadlocked tasks
-            :type deadlocked: set[Tuple[String, String, DateTime]]
+            :type deadlocked: set[tuple[string, string, datetime.datetime]]
             :param active_runs: Active dag runs at a certain point in time
             :type active_runs: list[DagRun]
             :param executed_dag_run_dates: Datetime objects for the executed dag runs
-            :type executed_dag_run_dates: set[Datetime]
+            :type executed_dag_run_dates: set[datetime.datetime]
             :param finished_runs: Number of finished runs so far
             :type finished_runs: int
             :param total_runs: Number of total dag runs able to run
             :type total_runs: int
             """
-            self.to_run = to_run or dict()
+            self.to_run = to_run or OrderedDict()
             self.running = running or dict()
             self.skipped = skipped or set()
             self.succeeded = succeeded or set()
@@ -1958,14 +1893,15 @@ def __init__(
             verbose=False,
             conf=None,
             rerun_failed_tasks=False,
+            run_backwards=False,
             *args, **kwargs):
         """
         :param dag: DAG object.
-        :type dag: `class DAG`.
+        :type dag: airflow.models.DAG
         :param start_date: start date for the backfill date range.
-        :type start_date: datetime.
+        :type start_date: datetime.datetime
         :param end_date: end date for the backfill date range.
-        :type end_date: datetime
+        :type end_date: datetime.datetime
         :param mark_success: flag whether to mark the task auto success.
         :type mark_success: bool
         :param donot_pickle: whether pickle
@@ -1984,6 +1920,8 @@ def __init__(
         :param rerun_failed_tasks: flag to whether to
                                    auto rerun the failed task in backfill
         :type rerun_failed_tasks: bool
+        :param run_backwards: Whether to process the dates from most to least recent
+        :type run_backwards bool
         :param args:
         :param kwargs:
         """
@@ -2000,12 +1938,14 @@ def __init__(
         self.verbose = verbose
         self.conf = conf
         self.rerun_failed_tasks = rerun_failed_tasks
+        self.run_backwards = run_backwards
         super(BackfillJob, self).__init__(*args, **kwargs)
 
     def _update_counters(self, ti_status):
         """
         Updates the counters per state of the tasks that were running. Can re-add
         to tasks to run in case required.
+
         :param ti_status: the internal status of the backfill job tasks
         :type ti_status: BackfillJob._DagRunTaskStatus
         """
@@ -2031,6 +1971,11 @@ def _update_counters(self, ti_status):
                 self.log.warning("Task instance %s is up for retry", ti)
                 ti_status.running.pop(key)
                 ti_status.to_run[key] = ti
+            # special case: if the task needs to be rescheduled put it back
+            elif ti.state == State.UP_FOR_RESCHEDULE:
+                self.log.warning("Task instance %s is up for reschedule", ti)
+                ti_status.running.pop(key)
+                ti_status.to_run[key] = ti
             # special case: The state of the task can be set to NONE by the task itself
             # when it reaches concurrency limits. It could also happen when the state
             # is changed externally, e.g. by clearing tasks from the ui. We need to cover
@@ -2050,6 +1995,7 @@ def _manage_executor_state(self, running):
         """
         Checks if the executor agrees with the state of task instances
         that are running
+
         :param running: dict of key, task to verify
         """
         executor = self.executor
@@ -2081,10 +2027,11 @@ def _get_dag_run(self, run_date, session=None):
         Returns a dag run for the given run date, which will be matched to an existing
         dag run if available or create a new dag run otherwise. If the max_active_runs
         limit is reached, this function will return None.
+
         :param run_date: the execution date for the dag run
-        :type run_date: datetime
+        :type run_date: datetime.datetime
         :param session: the database session object
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :return: a DagRun in state RUNNING or None
         """
         run_id = BackfillJob.ID_FORMAT_PREFIX.format(run_date.isoformat())
@@ -2140,10 +2087,11 @@ def _task_instances_for_dag_run(self, dag_run, session=None):
         """
         Returns a map of task instance key to task instance object for the tasks to
         run in the given dag run.
+
         :param dag_run: the dag run to get the tasks from
-        :type dag_run: models.DagRun
+        :type dag_run: airflow.models.DagRun
         :param session: the database session object
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         """
         tasks_to_run = {}
 
@@ -2168,27 +2116,13 @@ def _task_instances_for_dag_run(self, dag_run, session=None):
         return tasks_to_run
 
     def _log_progress(self, ti_status):
-        msg = ' | '.join([
-            "[backfill progress]",
-            "finished run {0} of {1}",
-            "tasks waiting: {2}",
-            "succeeded: {3}",
-            "running: {4}",
-            "failed: {5}",
-            "skipped: {6}",
-            "deadlocked: {7}",
-            "not ready: {8}"
-        ]).format(
-            ti_status.finished_runs,
-            ti_status.total_runs,
-            len(ti_status.to_run),
-            len(ti_status.succeeded),
-            len(ti_status.running),
-            len(ti_status.failed),
-            len(ti_status.skipped),
-            len(ti_status.deadlocked),
-            len(ti_status.not_ready))
-        self.log.info(msg)
+        self.log.info(
+            '[backfill progress] | finished run %s of %s | tasks waiting: %s | succeeded: %s | '
+            'running: %s | failed: %s | skipped: %s | deadlocked: %s | not ready: %s',
+            ti_status.finished_runs, ti_status.total_runs, len(ti_status.to_run), len(ti_status.succeeded),
+            len(ti_status.running), len(ti_status.failed), len(ti_status.skipped), len(ti_status.deadlocked),
+            len(ti_status.not_ready)
+        )
 
         self.log.debug(
             "Finished dag run loop iteration. Remaining tasks %s",
@@ -2205,6 +2139,7 @@ def _process_backfill_task_instances(self,
         Process a set of task instances from a set of dag runs. Special handling is done
         to account for different task instance states that could be present when running
         them in a backfill process.
+
         :param ti_status: the internal status of the job
         :type ti_status: BackfillJob._DagRunTaskStatus
         :param executor: the executor to run the task instances
@@ -2212,9 +2147,9 @@ def _process_backfill_task_instances(self,
         :param pickle_id: the pickle_id if dag is pickled, None otherwise
         :type pickle_id: int
         :param start_date: the start date of the backfill job
-        :type start_date: datetime
+        :type start_date: datetime.datetime
         :param session: the current session object
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :return: the list of execution_dates for the finished dag runs
         :rtype: list
         """
@@ -2230,134 +2165,149 @@ def _process_backfill_task_instances(self,
             # or leaf to root, as otherwise tasks might be
             # determined deadlocked while they are actually
             # waiting for their upstream to finish
-            for task in self.dag.topological_sort():
-                for key, ti in list(ti_status.to_run.items()):
-                    if task.task_id != ti.task_id:
-                        continue
+            @provide_session
+            def _per_task_process(task, key, ti, session=None):
+                if task.task_id != ti.task_id:
+                    return
 
-                    ti.refresh_from_db()
+                ti.refresh_from_db()
 
-                    task = self.dag.get_task(ti.task_id)
-                    ti.task = task
+                task = self.dag.get_task(ti.task_id)
+                ti.task = task
 
-                    ignore_depends_on_past = (
-                        self.ignore_first_depends_on_past and
-                        ti.execution_date == (start_date or ti.start_date))
-                    self.log.debug(
-                        "Task instance to run %s state %s", ti, ti.state)
+                ignore_depends_on_past = (
+                    self.ignore_first_depends_on_past and
+                    ti.execution_date == (start_date or ti.start_date))
+                self.log.debug(
+                    "Task instance to run %s state %s", ti, ti.state)
+
+                # The task was already marked successful or skipped by a
+                # different Job. Don't rerun it.
+                if ti.state == State.SUCCESS:
+                    ti_status.succeeded.add(key)
+                    self.log.debug("Task instance %s succeeded. Don't rerun.", ti)
+                    ti_status.to_run.pop(key)
+                    if key in ti_status.running:
+                        ti_status.running.pop(key)
+                    return
+                elif ti.state == State.SKIPPED:
+                    ti_status.skipped.add(key)
+                    self.log.debug("Task instance %s skipped. Don't rerun.", ti)
+                    ti_status.to_run.pop(key)
+                    if key in ti_status.running:
+                        ti_status.running.pop(key)
+                    return
 
-                    # The task was already marked successful or skipped by a
-                    # different Job. Don't rerun it.
-                    if ti.state == State.SUCCESS:
-                        ti_status.succeeded.add(key)
-                        self.log.debug("Task instance %s succeeded. Don't rerun.", ti)
-                        ti_status.to_run.pop(key)
+                # guard against externally modified tasks instances or
+                # in case max concurrency has been reached at task runtime
+                elif ti.state == State.NONE:
+                    self.log.warning(
+                        "FIXME: task instance {} state was set to None "
+                        "externally. This should not happen"
+                    )
+                    ti.set_state(State.SCHEDULED, session=session)
+                if self.rerun_failed_tasks:
+                    # Rerun failed tasks or upstreamed failed tasks
+                    if ti.state in (State.FAILED, State.UPSTREAM_FAILED):
+                        self.log.error("Task instance {ti} "
+                                       "with state {state}".format(ti=ti,
+                                                                   state=ti.state))
                         if key in ti_status.running:
                             ti_status.running.pop(key)
-                        continue
-                    elif ti.state == State.SKIPPED:
-                        ti_status.skipped.add(key)
-                        self.log.debug("Task instance %s skipped. Don't rerun.", ti)
+                        # Reset the failed task in backfill to scheduled state
+                        ti.set_state(State.SCHEDULED, session=session)
+                else:
+                    # Default behaviour which works for subdag.
+                    if ti.state in (State.FAILED, State.UPSTREAM_FAILED):
+                        self.log.error("Task instance {ti} "
+                                       "with {state} state".format(ti=ti,
+                                                                   state=ti.state))
+                        ti_status.failed.add(key)
                         ti_status.to_run.pop(key)
                         if key in ti_status.running:
                             ti_status.running.pop(key)
-                        continue
+                        return
 
-                    # guard against externally modified tasks instances or
-                    # in case max concurrency has been reached at task runtime
-                    elif ti.state == State.NONE:
-                        self.log.warning(
-                            "FIXME: task instance {} state was set to None "
-                            "externally. This should not happen"
-                        )
-                        ti.set_state(State.SCHEDULED, session=session)
-                    if self.rerun_failed_tasks:
-                        # Rerun failed tasks or upstreamed failed tasks
-                        if ti.state in (State.FAILED, State.UPSTREAM_FAILED):
-                            self.log.error("Task instance {ti} "
-                                           "with state {state}".format(ti=ti,
-                                                                       state=ti.state))
-                            if key in ti_status.running:
-                                ti_status.running.pop(key)
-                            # Reset the failed task in backfill to scheduled state
-                            ti.set_state(State.SCHEDULED, session=session)
-                    else:
-                        # Default behaviour which works for subdag.
-                        if ti.state in (State.FAILED, State.UPSTREAM_FAILED):
-                            self.log.error("Task instance {ti} "
-                                           "with {state} state".format(ti=ti,
-                                                                       state=ti.state))
-                            ti_status.failed.add(key)
+                backfill_context = DepContext(
+                    deps=RUN_DEPS,
+                    ignore_depends_on_past=ignore_depends_on_past,
+                    ignore_task_deps=self.ignore_task_deps,
+                    flag_upstream_failed=True)
+
+                # Is the task runnable? -- then run it
+                # the dependency checker can change states of tis
+                if ti.are_dependencies_met(
+                        dep_context=backfill_context,
+                        session=session,
+                        verbose=self.verbose):
+                    ti.refresh_from_db(lock_for_update=True, session=session)
+                    if ti.state in (State.SCHEDULED, State.UP_FOR_RETRY, State.UP_FOR_RESCHEDULE):
+                        if executor.has_task(ti):
+                            self.log.debug(
+                                "Task Instance %s already in executor "
+                                "waiting for queue to clear",
+                                ti
+                            )
+                        else:
+                            self.log.debug('Sending %s to executor', ti)
+                            # Skip scheduled state, we are executing immediately
+                            ti.state = State.QUEUED
+                            ti.queued_dttm = timezone.utcnow() if not ti.queued_dttm else ti.queued_dttm
+                            session.merge(ti)
+
+                            cfg_path = None
+                            if executor.__class__ in (executors.LocalExecutor,
+                                                      executors.SequentialExecutor):
+                                cfg_path = tmp_configuration_copy()
+
+                            executor.queue_task_instance(
+                                ti,
+                                mark_success=self.mark_success,
+                                pickle_id=pickle_id,
+                                ignore_task_deps=self.ignore_task_deps,
+                                ignore_depends_on_past=ignore_depends_on_past,
+                                pool=self.pool,
+                                cfg_path=cfg_path)
+                            ti_status.running[key] = ti
                             ti_status.to_run.pop(key)
-                            if key in ti_status.running:
-                                ti_status.running.pop(key)
-                            continue
-
-                    backfill_context = DepContext(
-                        deps=RUN_DEPS,
-                        ignore_depends_on_past=ignore_depends_on_past,
-                        ignore_task_deps=self.ignore_task_deps,
-                        flag_upstream_failed=True)
-
-                    # Is the task runnable? -- then run it
-                    # the dependency checker can change states of tis
-                    if ti.are_dependencies_met(
-                            dep_context=backfill_context,
-                            session=session,
-                            verbose=self.verbose):
-                        ti.refresh_from_db(lock_for_update=True, session=session)
-                        if ti.state == State.SCHEDULED or ti.state == State.UP_FOR_RETRY:
-                            if executor.has_task(ti):
-                                self.log.debug(
-                                    "Task Instance %s already in executor "
-                                    "waiting for queue to clear",
-                                    ti
-                                )
-                            else:
-                                self.log.debug('Sending %s to executor', ti)
-                                # Skip scheduled state, we are executing immediately
-                                ti.state = State.QUEUED
-                                session.merge(ti)
-
-                                cfg_path = None
-                                if executor.__class__ in (executors.LocalExecutor,
-                                                          executors.SequentialExecutor):
-                                    cfg_path = tmp_configuration_copy()
-
-                                executor.queue_task_instance(
-                                    ti,
-                                    mark_success=self.mark_success,
-                                    pickle_id=pickle_id,
-                                    ignore_task_deps=self.ignore_task_deps,
-                                    ignore_depends_on_past=ignore_depends_on_past,
-                                    pool=self.pool,
-                                    cfg_path=cfg_path)
-                                ti_status.running[key] = ti
-                                ti_status.to_run.pop(key)
-                        session.commit()
-                        continue
+                    session.commit()
+                    return
 
-                    if ti.state == State.UPSTREAM_FAILED:
-                        self.log.error("Task instance %s upstream failed", ti)
-                        ti_status.failed.add(key)
-                        ti_status.to_run.pop(key)
-                        if key in ti_status.running:
-                            ti_status.running.pop(key)
-                        continue
+                if ti.state == State.UPSTREAM_FAILED:
+                    self.log.error("Task instance %s upstream failed", ti)
+                    ti_status.failed.add(key)
+                    ti_status.to_run.pop(key)
+                    if key in ti_status.running:
+                        ti_status.running.pop(key)
+                    return
 
-                    # special case
-                    if ti.state == State.UP_FOR_RETRY:
-                        self.log.debug(
-                            "Task instance %s retry period not "
-                            "expired yet", ti)
-                        if key in ti_status.running:
-                            ti_status.running.pop(key)
-                        ti_status.to_run[key] = ti
-                        continue
+                # special case
+                if ti.state == State.UP_FOR_RETRY:
+                    self.log.debug(
+                        "Task instance %s retry period not "
+                        "expired yet", ti)
+                    if key in ti_status.running:
+                        ti_status.running.pop(key)
+                    ti_status.to_run[key] = ti
+                    return
 
-                    # all remaining tasks
-                    self.log.debug('Adding %s to not_ready', ti)
-                    ti_status.not_ready.add(key)
+                # special case
+                if ti.state == State.UP_FOR_RESCHEDULE:
+                    self.log.debug(
+                        "Task instance %s reschedule period not "
+                        "expired yet", ti)
+                    if key in ti_status.running:
+                        ti_status.running.pop(key)
+                    ti_status.to_run[key] = ti
+                    return
+
+                # all remaining tasks
+                self.log.debug('Adding %s to not_ready', ti)
+                ti_status.not_ready.add(key)
+
+            for task in self.dag.topological_sort():
+                for key, ti in list(ti_status.to_run.items()):
+                    _per_task_process(task, key, ti)
 
             # execute the tasks in the queue
             self.heartbeat()
@@ -2391,9 +2341,6 @@ def _process_backfill_task_instances(self,
                     ti_status.active_runs.remove(run)
                     executed_run_dates.append(run.execution_date)
 
-                if run.dag.is_paused:
-                    models.DagStat.update([run.dag_id], session=session)
-
             self._log_progress(ti_status)
 
         # return updated status
@@ -2442,6 +2389,7 @@ def _execute_for_run_dates(self, run_dates, ti_status, executor, pickle_id,
         Computes the dag runs and their respective task instances for
         the given run dates and executes the task instances.
         Returns a list of execution dates of the dag runs that were executed.
+
         :param run_dates: Execution dates for dag runs
         :type run_dates: list
         :param ti_status: internal BackfillJob status structure to tis track progress
@@ -2451,9 +2399,9 @@ def _execute_for_run_dates(self, run_dates, ti_status, executor, pickle_id,
         :param pickle_id: numeric id of the pickled dag, None if not pickled
         :type pickle_id: int
         :param start_date: backfill start date
-        :type start_date: datetime
+        :type start_date: datetime.datetime
         :param session: the current session object
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         """
         for next_run_date in run_dates:
             dag_run = self._get_dag_run(next_run_date, session=session)
@@ -2487,6 +2435,14 @@ def _execute(self, session=None):
         # Get intervals between the start/end dates, which will turn into dag runs
         run_dates = self.dag.get_run_dates(start_date=start_date,
                                            end_date=self.bf_end_date)
+        if self.run_backwards:
+            tasks_that_depend_on_past = [t.task_id for t in self.dag.task_dict.values() if t.depends_on_past]
+            if tasks_that_depend_on_past:
+                raise AirflowException(
+                    'You cannot backfill backwards because one or more tasks depend_on_past: {}'.format(
+                        ",".join(tasks_that_depend_on_past)))
+            run_dates = run_dates[::-1]
+
         if len(run_dates) == 0:
             self.log.info("No run dates were found for the given dates and dag interval.")
             return
@@ -2495,7 +2451,7 @@ def _execute(self, session=None):
         pickle_id = None
         if not self.donot_pickle and self.executor.__class__ not in (
                 executors.LocalExecutor, executors.SequentialExecutor):
-            pickle = models.DagPickle(self.dag)
+            pickle = DagPickle(self.dag)
             session.add(pickle)
             session.commit()
             pickle_id = pickle.id
@@ -2557,6 +2513,7 @@ def __init__(
             pool=None,
             *args, **kwargs):
         self.task_instance = task_instance
+        self.dag_id = task_instance.dag_id
         self.ignore_all_deps = ignore_all_deps
         self.ignore_depends_on_past = ignore_depends_on_past
         self.ignore_task_deps = ignore_task_deps
diff --git a/airflow/lineage/__init__.py b/airflow/lineage/__init__.py
index 3e2af478bb5e0..124cbabbd9a3d 100644
--- a/airflow/lineage/__init__.py
+++ b/airflow/lineage/__init__.py
@@ -21,7 +21,7 @@
 from airflow import configuration as conf
 from airflow.lineage.datasets import DataSet
 from airflow.utils.log.logging_mixin import LoggingMixin
-from airflow.utils.module_loading import import_string, prepare_classpath
+from airflow.utils.module_loading import import_string
 
 from itertools import chain
 
@@ -36,7 +36,6 @@ def _get_backend():
 
     try:
         _backend_str = conf.get("lineage", "backend")
-        prepare_classpath()
         backend = import_string(_backend_str)
     except ImportError as ie:
         log.debug("Cannot import %s due to %s", _backend_str, ie)
@@ -85,13 +84,13 @@ def wrapper(self, context, *args, **kwargs):
 
 def prepare_lineage(func):
     """
-    Prepares the lineage inlets and outlets
-    inlets can be:
-        "auto" -> picks up any outlets from direct upstream tasks that have outlets
-        defined, as such that if A -> B -> C and B does not have outlets but A does,
-        these are provided as inlets.
-        "list of task_ids" -> picks up outlets from the upstream task_ids
-        "list of datasets" -> manually defined list of DataSet
+    Prepares the lineage inlets and outlets. Inlets can be:
+
+    * "auto" -> picks up any outlets from direct upstream tasks that have outlets defined, as such that
+      if A -> B -> C and B does not have outlets but A does, these are provided as inlets.
+    * "list of task_ids" -> picks up outlets from the upstream task_ids
+    * "list of datasets" -> manually defined list of DataSet
+
     """
     @wraps(func)
     def wrapper(self, context, *args, **kwargs):
diff --git a/airflow/lineage/datasets.py b/airflow/lineage/datasets.py
index 40c8edc9a8f9d..260277065b6f0 100644
--- a/airflow/lineage/datasets.py
+++ b/airflow/lineage/datasets.py
@@ -18,6 +18,7 @@
 # under the License.
 import six
 
+from typing import List
 from jinja2 import Environment
 
 
@@ -28,7 +29,7 @@ def _inherited(cls):
 
 
 class DataSet(object):
-    attributes = []
+    attributes = []  # type: List[str]
     type_name = "dataSet"
 
     def __init__(self, qualified_name=None, data=None, **kwargs):
@@ -96,7 +97,7 @@ def map_type(name):
             if cls.type_name == name:
                 return cls
 
-        raise NotImplemented("No known mapping for {}".format(name))
+        raise NotImplementedError("No known mapping for {}".format(name))
 
 
 class DataBase(DataSet):
diff --git a/airflow/logging_config.py b/airflow/logging_config.py
index 33c2dc82e1be5..0cf8f4db0d59b 100644
--- a/airflow/logging_config.py
+++ b/airflow/logging_config.py
@@ -18,11 +18,12 @@
 # under the License.
 #
 import logging
+import warnings
 from logging.config import dictConfig
 
 from airflow import configuration as conf
 from airflow.exceptions import AirflowConfigException
-from airflow.utils.module_loading import import_string, prepare_classpath
+from airflow.utils.module_loading import import_string
 
 log = logging.getLogger(__name__)
 
@@ -30,10 +31,6 @@
 def configure_logging():
     logging_class_path = ''
     try:
-        # Prepare the classpath so we are sure that the config folder
-        # is on the python classpath and it is reachable
-        prepare_classpath()
-
         logging_class_path = conf.get('core', 'logging_config_class')
     except AirflowConfigException:
         log.debug('Could not find key logging_config_class in config')
@@ -56,9 +53,9 @@ def configure_logging():
                 .format(logging_class_path, err)
             )
     else:
-        from airflow.config_templates.airflow_local_settings import (
-            DEFAULT_LOGGING_CONFIG as logging_config
-        )
+        logging_class_path = 'airflow.config_templates.' \
+                             'airflow_local_settings.DEFAULT_LOGGING_CONFIG'
+        logging_config = import_string(logging_class_path)
         log.debug('Unable to load custom logging, using default config instead')
 
     try:
@@ -70,4 +67,36 @@ def configure_logging():
         # otherwise Airflow would silently fall back on the default config
         raise e
 
-    return logging_config
+    validate_logging_config(logging_config)
+
+    return logging_class_path
+
+
+def validate_logging_config(logging_config):
+    # Now lets validate the other logging-related settings
+    task_log_reader = conf.get('core', 'task_log_reader')
+
+    logger = logging.getLogger('airflow.task')
+
+    def _get_handler(name):
+        return next((h for h in logger.handlers if h.name == name), None)
+
+    if _get_handler(task_log_reader) is None:
+        # Check for pre 1.10 setting that might be in deployed airflow.cfg files
+        if task_log_reader == "file.task" and _get_handler("task"):
+            warnings.warn(
+                "task_log_reader setting in [core] has a deprecated value of "
+                "{!r}, but no handler with this name was found. Please update "
+                "your config to use {!r}. Running config has been adjusted to "
+                "match".format(
+                    task_log_reader,
+                    "task",
+                ),
+                DeprecationWarning,
+            )
+            conf.set('core', 'task_log_reader', 'task')
+        else:
+            raise AirflowConfigException(
+                "Configured task_log_reader {!r} was not a handler of the 'airflow.task' "
+                "logger.".format(task_log_reader)
+            )
diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py
index bb60203e95347..914b9af2c4b1e 100644
--- a/airflow/macros/hive.py
+++ b/airflow/macros/hive.py
@@ -27,14 +27,14 @@ def max_partition(
     Gets the max partition for a table.
 
     :param schema: The hive schema the table lives in
-    :type schema: string
+    :type schema: str
     :param table: The hive table you are interested in, supports the dot
         notation as in "my_database.my_table", if a dot is found,
         the schema param is disregarded
-    :type table: string
+    :type table: str
     :param metastore_conn_id: The hive connection you are interested in.
         If your default is set you don't need to use this parameter.
-    :type metastore_conn_id: string
+    :type metastore_conn_id: str
     :param filter_map: partition_key:partition_value map used for partition filtering,
                        e.g. {'key1': 'value1', 'key2': 'value2'}.
                        Only partitions matching all partition_key:partition_value
@@ -63,7 +63,7 @@ def _closest_date(target_dt, date_list, before_target=None):
     :param target_dt: The target date
     :type target_dt: datetime.date
     :param date_list: The list of dates to search
-    :type date_list: datetime.date list
+    :type date_list: list[datetime.date]
     :param before_target: closest before or after the target
     :type before_target: bool or None
     :returns: The closest date
@@ -90,7 +90,7 @@ def closest_ds_partition(
     :param table: A hive table name
     :type table: str
     :param ds: A datestamp ``%Y-%m-%d`` e.g. ``yyyy-mm-dd``
-    :type ds: datetime.date list
+    :type ds: list[datetime.date]
     :param before: closest before (True), after (False) or either side of ds
     :type before: bool or None
     :returns: The closest date
diff --git a/airflow/migrations/__init__.py b/airflow/migrations/__init__.py
index f0f8b68337da6..114d189da14ab 100644
--- a/airflow/migrations/__init__.py
+++ b/airflow/migrations/__init__.py
@@ -7,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
diff --git a/airflow/migrations/env.py b/airflow/migrations/env.py
index 828cacf0e0fd6..76c2eb329b0df 100644
--- a/airflow/migrations/env.py
+++ b/airflow/migrations/env.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -36,7 +36,7 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-target_metadata = models.Base.metadata
+target_metadata = models.base.Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -85,6 +85,7 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:
diff --git a/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py b/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py
new file mode 100644
index 0000000000000..fc8468155c235
--- /dev/null
+++ b/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""merge_heads_2
+
+Revision ID: 03bc53e68815
+Revises: 0a2a5b66e19d, bf00311e1990
+Create Date: 2018-11-24 20:21:46.605414
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '03bc53e68815'
+down_revision = ('0a2a5b66e19d', 'bf00311e1990')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index('sm_dag', 'sla_miss', ['dag_id'], unique=False)
+
+
+def downgrade():
+    op.drop_index('sm_dag', table_name='sla_miss')
diff --git a/airflow/migrations/versions/05f30312d566_merge_heads.py b/airflow/migrations/versions/05f30312d566_merge_heads.py
new file mode 100644
index 0000000000000..f869cb8c08e5a
--- /dev/null
+++ b/airflow/migrations/versions/05f30312d566_merge_heads.py
@@ -0,0 +1,40 @@
+# flake8: noqa
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""merge heads
+
+Revision ID: 05f30312d566
+Revises: 86770d1215c0, 0e2a74e0fc9f
+Create Date: 2018-06-17 10:47:23.339972
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '05f30312d566'
+down_revision = ('86770d1215c0', '0e2a74e0fc9f')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass
diff --git a/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py b/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py
new file mode 100644
index 0000000000000..643a1ca81b678
--- /dev/null
+++ b/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py
@@ -0,0 +1,91 @@
+# flake8: noqa
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""add task_reschedule table
+
+Revision ID: 0a2a5b66e19d
+Revises: 9635ae0956e7
+Create Date: 2018-06-17 22:50:00.053620
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0a2a5b66e19d'
+down_revision = '9635ae0956e7'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+
+TABLE_NAME = 'task_reschedule'
+INDEX_NAME = 'idx_' + TABLE_NAME + '_dag_task_date'
+
+# For Microsoft SQL Server, TIMESTAMP is a row-id type,
+# having nothing to do with date-time.  DateTime() will
+# be sufficient.
+def mssql_timestamp():
+    return sa.DateTime()
+
+def mysql_timestamp():
+    return mysql.TIMESTAMP(fsp=6)
+
+def sa_timestamp():
+    return sa.TIMESTAMP(timezone=True)
+
+def upgrade():
+    # See 0e2a74e0fc9f_add_time_zone_awareness
+    conn = op.get_bind()
+    if conn.dialect.name == 'mysql':
+        timestamp = mysql_timestamp
+    elif conn.dialect.name == 'mssql':
+        timestamp = mssql_timestamp
+    else:
+        timestamp = sa_timestamp
+
+    op.create_table(
+        TABLE_NAME,
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('task_id', sa.String(length=250), nullable=False),
+        sa.Column('dag_id', sa.String(length=250), nullable=False),
+        # use explicit server_default=None otherwise mysql implies defaults for first timestamp column
+        sa.Column('execution_date', timestamp(), nullable=False, server_default=None),
+        sa.Column('try_number', sa.Integer(), nullable=False),
+        sa.Column('start_date', timestamp(), nullable=False),
+        sa.Column('end_date', timestamp(), nullable=False),
+        sa.Column('duration', sa.Integer(), nullable=False),
+        sa.Column('reschedule_date', timestamp(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['task_id', 'dag_id', 'execution_date'],
+                                ['task_instance.task_id', 'task_instance.dag_id','task_instance.execution_date'],
+                                name='task_reschedule_dag_task_date_fkey')
+    )
+    op.create_index(
+        INDEX_NAME,
+        TABLE_NAME,
+        ['dag_id', 'task_id', 'execution_date'],
+        unique=False
+    )
+
+
+def downgrade():
+    op.drop_index(INDEX_NAME, table_name=TABLE_NAME)
+    op.drop_table(TABLE_NAME)
diff --git a/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py b/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py
index 64ee41c44d9e2..ac7824fd9e2f5 100644
--- a/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py
+++ b/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py
@@ -25,16 +25,16 @@
 
 """
 
+from alembic import op
+from sqlalchemy.dialects import mysql
+import sqlalchemy as sa
+
 # revision identifiers, used by Alembic.
 revision = '0e2a74e0fc9f'
 down_revision = 'd2ae31099d61'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-from sqlalchemy.dialects import mysql
-import sqlalchemy as sa
-
 
 def upgrade():
     conn = op.get_bind()
@@ -69,14 +69,16 @@ def upgrade():
         op.alter_column(table_name='log', column_name='dttm', type_=mysql.TIMESTAMP(fsp=6))
         op.alter_column(table_name='log', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6))
 
-        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6), nullable=False)
+        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6),
+                        nullable=False)
         op.alter_column(table_name='sla_miss', column_name='timestamp', type_=mysql.TIMESTAMP(fsp=6))
 
         op.alter_column(table_name='task_fail', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6))
         op.alter_column(table_name='task_fail', column_name='start_date', type_=mysql.TIMESTAMP(fsp=6))
         op.alter_column(table_name='task_fail', column_name='end_date', type_=mysql.TIMESTAMP(fsp=6))
 
-        op.alter_column(table_name='task_instance', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6), nullable=False)
+        op.alter_column(table_name='task_instance', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6),
+                        nullable=False)
         op.alter_column(table_name='task_instance', column_name='start_date', type_=mysql.TIMESTAMP(fsp=6))
         op.alter_column(table_name='task_instance', column_name='end_date', type_=mysql.TIMESTAMP(fsp=6))
         op.alter_column(table_name='task_instance', column_name='queued_dttm', type_=mysql.TIMESTAMP(fsp=6))
@@ -84,8 +86,8 @@ def upgrade():
         op.alter_column(table_name='xcom', column_name='timestamp', type_=mysql.TIMESTAMP(fsp=6))
         op.alter_column(table_name='xcom', column_name='execution_date', type_=mysql.TIMESTAMP(fsp=6))
     else:
-        # sqlite datetime is fine as is not converting
-        if conn.dialect.name == 'sqlite':
+        # sqlite and mssql datetime are fine as is.  Therefore, not converting
+        if conn.dialect.name in ('sqlite', 'mssql'):
             return
 
         # we try to be database agnostic, but not every db (e.g. sqlserver)
@@ -117,14 +119,16 @@ def upgrade():
         op.alter_column(table_name='log', column_name='dttm', type_=sa.TIMESTAMP(timezone=True))
         op.alter_column(table_name='log', column_name='execution_date', type_=sa.TIMESTAMP(timezone=True))
 
-        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=sa.TIMESTAMP(timezone=True), nullable=False)
+        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=sa.TIMESTAMP(timezone=True),
+                        nullable=False)
         op.alter_column(table_name='sla_miss', column_name='timestamp', type_=sa.TIMESTAMP(timezone=True))
 
         op.alter_column(table_name='task_fail', column_name='execution_date', type_=sa.TIMESTAMP(timezone=True))
         op.alter_column(table_name='task_fail', column_name='start_date', type_=sa.TIMESTAMP(timezone=True))
         op.alter_column(table_name='task_fail', column_name='end_date', type_=sa.TIMESTAMP(timezone=True))
 
-        op.alter_column(table_name='task_instance', column_name='execution_date', type_=sa.TIMESTAMP(timezone=True), nullable=False)
+        op.alter_column(table_name='task_instance', column_name='execution_date', type_=sa.TIMESTAMP(timezone=True),
+                        nullable=False)
         op.alter_column(table_name='task_instance', column_name='start_date', type_=sa.TIMESTAMP(timezone=True))
         op.alter_column(table_name='task_instance', column_name='end_date', type_=sa.TIMESTAMP(timezone=True))
         op.alter_column(table_name='task_instance', column_name='queued_dttm', type_=sa.TIMESTAMP(timezone=True))
@@ -161,14 +165,16 @@ def downgrade():
         op.alter_column(table_name='log', column_name='dttm', type_=mysql.DATETIME(fsp=6))
         op.alter_column(table_name='log', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
 
-        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=mysql.DATETIME(fsp=6), nullable=False)
+        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=mysql.DATETIME(fsp=6),
+                        nullable=False)
         op.alter_column(table_name='sla_miss', column_name='DATETIME', type_=mysql.DATETIME(fsp=6))
 
         op.alter_column(table_name='task_fail', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
         op.alter_column(table_name='task_fail', column_name='start_date', type_=mysql.DATETIME(fsp=6))
         op.alter_column(table_name='task_fail', column_name='end_date', type_=mysql.DATETIME(fsp=6))
 
-        op.alter_column(table_name='task_instance', column_name='execution_date', type_=mysql.DATETIME(fsp=6), nullable=False)
+        op.alter_column(table_name='task_instance', column_name='execution_date', type_=mysql.DATETIME(fsp=6),
+                        nullable=False)
         op.alter_column(table_name='task_instance', column_name='start_date', type_=mysql.DATETIME(fsp=6))
         op.alter_column(table_name='task_instance', column_name='end_date', type_=mysql.DATETIME(fsp=6))
         op.alter_column(table_name='task_instance', column_name='queued_dttm', type_=mysql.DATETIME(fsp=6))
@@ -176,7 +182,7 @@ def downgrade():
         op.alter_column(table_name='xcom', column_name='DATETIME', type_=mysql.DATETIME(fsp=6))
         op.alter_column(table_name='xcom', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
     else:
-        if conn.dialect.name == 'sqlite':
+        if conn.dialect.name in ('sqlite', 'mssql'):
             return
 
         # we try to be database agnostic, but not every db (e.g. sqlserver)
diff --git a/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py b/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py
index 58517256949ab..288a0b60aa821 100644
--- a/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py
+++ b/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,7 @@
 Create Date: 2017-01-25 11:43:51.635667
 
 """
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '127d2bf2dfa7'
@@ -30,8 +31,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
 
 def upgrade():
     op.create_index('dag_id_state', 'dag_run', ['dag_id', 'state'], unique=False)
@@ -39,4 +38,3 @@ def upgrade():
 
 def downgrade():
     op.drop_index('dag_id_state', table_name='dag_run')
-
diff --git a/airflow/migrations/versions/13eb55f81627_for_compatibility.py b/airflow/migrations/versions/13eb55f81627_for_compatibility.py
index a14e195bbefe1..b9e88bd01becd 100644
--- a/airflow/migrations/versions/13eb55f81627_for_compatibility.py
+++ b/airflow/migrations/versions/13eb55f81627_for_compatibility.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py
index 47bdf13d672d3..fe84254c38caa 100644
--- a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py
+++ b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,9 @@
 Create Date: 2015-08-18 18:57:51.927315
 
 """
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
 
 # revision identifiers, used by Alembic.
 revision = '1507a7289a2f'
@@ -31,10 +34,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.engine.reflection import Inspector
-
 connectionhelper = sa.Table(
     'connection',
     sa.MetaData(),
diff --git a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py
index 57ee27aeb4bfb..16ab349563428 100644
--- a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py
+++ b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,8 @@
 Create Date: 2016-02-02 17:20:55.692295
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '1968acfc09e3'
@@ -31,12 +33,9 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
-    op.add_column('variable', sa.Column('is_encrypted', sa.Boolean,default=False))
+    op.add_column('variable', sa.Column('is_encrypted', sa.Boolean, default=False))
 
 
 def downgrade():
diff --git a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py
index a0af7fa7ad6c2..50d53652c4734 100644
--- a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py
+++ b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,28 +25,27 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+
 # revision identifiers, used by Alembic.
 revision = '1b38cef5b76e'
 down_revision = '502898887f84'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.create_table('dag_run',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('dag_id', sa.String(length=250), nullable=True),
-        sa.Column('execution_date', sa.DateTime(), nullable=True),
-        sa.Column('state', sa.String(length=50), nullable=True),
-        sa.Column('run_id', sa.String(length=250), nullable=True),
-        sa.Column('external_trigger', sa.Boolean(), nullable=True),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('dag_id', 'execution_date'),
-        sa.UniqueConstraint('dag_id', 'run_id'),
-    )
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('dag_id', sa.String(length=250), nullable=True),
+                    sa.Column('execution_date', sa.DateTime(), nullable=True),
+                    sa.Column('state', sa.String(length=50), nullable=True),
+                    sa.Column('run_id', sa.String(length=250), nullable=True),
+                    sa.Column('external_trigger', sa.Boolean(), nullable=True),
+                    sa.PrimaryKeyConstraint('id'),
+                    sa.UniqueConstraint('dag_id', 'execution_date'),
+                    sa.UniqueConstraint('dag_id', 'run_id'))
 
 
 def downgrade():
diff --git a/airflow/migrations/versions/211e584da130_add_ti_state_index.py b/airflow/migrations/versions/211e584da130_add_ti_state_index.py
index 93347a2649214..b17f390e0b65b 100644
--- a/airflow/migrations/versions/211e584da130_add_ti_state_index.py
+++ b/airflow/migrations/versions/211e584da130_add_ti_state_index.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,7 @@
 Create Date: 2016-06-30 10:54:24.323588
 
 """
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '211e584da130'
@@ -31,9 +32,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.create_index('ti_state', 'task_instance', ['state'], unique=False)
diff --git a/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py b/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py
index b7213a3031ddd..a757d2770971b 100644
--- a/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py
+++ b/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py
@@ -1,16 +1,22 @@
 # flake8: noqa
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 
 """kubernetes_resource_checkpointing
 
@@ -20,18 +26,16 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+import dill
+
 # revision identifiers, used by Alembic.
 revision = '27c6a30d7c24'
 down_revision = '33ae817a1ff4'
 branch_labels = None
 depends_on = None
 
-
-from alembic import op
-import sqlalchemy as sa
-import dill
-
-
 TASK_INSTANCE_TABLE = "task_instance"
 NEW_COLUMN = "executor_config"
 
@@ -42,4 +46,3 @@ def upgrade():
 
 def downgrade():
     op.drop_column(TASK_INSTANCE_TABLE, NEW_COLUMN)
-
diff --git a/airflow/migrations/versions/2e541a1dcfed_task_duration.py b/airflow/migrations/versions/2e541a1dcfed_task_duration.py
index 7b540430a5dd0..595a5774a6b27 100644
--- a/airflow/migrations/versions/2e541a1dcfed_task_duration.py
+++ b/airflow/migrations/versions/2e541a1dcfed_task_duration.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,16 +25,16 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
 # revision identifiers, used by Alembic.
 revision = '2e541a1dcfed'
 down_revision = '1b38cef5b76e'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
-
 
 def upgrade():
     # use batch_alter_table to support SQLite workaround
diff --git a/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py b/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py
index 03266679279f9..fc8a1aab20e69 100644
--- a/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py
+++ b/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,7 @@
 Create Date: 2016-04-02 19:28:15.211915
 
 """
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '2e82aab8ef20'
@@ -31,9 +32,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.rename_table('user', 'users')
@@ -41,4 +39,3 @@ def upgrade():
 
 def downgrade():
     op.rename_table('users', 'user')
-
diff --git a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py
index c101e4a477230..473f76778b89d 100644
--- a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py
+++ b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,13 +17,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""More logging into task_isntance
+"""More logging into task_instance
 
 Revision ID: 338e90f54d61
 Revises: 13eb55f81627
 Create Date: 2015-08-25 06:09:20.460147
 
 """
+# flake8: noqa: E266
+
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '338e90f54d61'
@@ -31,9 +35,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
diff --git a/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py b/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py
index 4347bae92af1e..88a4199d3250e 100644
--- a/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py
+++ b/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py
@@ -1,16 +1,22 @@
 # flake8: noqa
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 
 """kubernetes_resource_checkpointing
 
@@ -19,6 +25,8 @@
 Create Date: 2017-09-11 15:26:47.598494
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '33ae817a1ff4'
@@ -26,20 +34,27 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
-
 RESOURCE_TABLE = "kube_resource_version"
 
 
 def upgrade():
+
+    columns_and_constraints = [
+        sa.Column("one_row_id", sa.Boolean, server_default=sa.true(), primary_key=True),
+        sa.Column("resource_version", sa.String(255))
+    ]
+
+    conn = op.get_bind()
+
+    # alembic creates an invalid SQL for mssql dialect
+    if conn.dialect.name not in ('mssql'):
+        columns_and_constraints.append(sa.CheckConstraint("one_row_id", name="kube_resource_version_one_row_id"))
+
     table = op.create_table(
         RESOURCE_TABLE,
-        sa.Column("one_row_id", sa.Boolean, server_default=sa.true(), primary_key=True),
-        sa.Column("resource_version", sa.String(255)),
-        sa.CheckConstraint("one_row_id", name="kube_resource_version_one_row_id")
+        *columns_and_constraints
     )
+
     op.bulk_insert(table, [
         {"resource_version": ""}
     ])
@@ -47,4 +62,3 @@ def upgrade():
 
 def downgrade():
     op.drop_table(RESOURCE_TABLE)
-
diff --git a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py
index ecfe58cfc3d1b..3da4d5f543038 100644
--- a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py
+++ b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,8 @@
 Create Date: 2015-10-29 08:36:31.726728
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '40e67319e3a9'
@@ -31,9 +33,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.add_column('dag_run', sa.Column('conf', sa.PickleType(), nullable=True))
diff --git a/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py b/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py
new file mode 100644
index 0000000000000..6e02582b7e840
--- /dev/null
+++ b/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""add superuser field
+
+Revision ID: 41f5f12752f8
+Revises: 03bc53e68815
+Create Date: 2018-12-04 15:50:04.456875
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '41f5f12752f8'
+down_revision = '03bc53e68815'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('users', sa.Column('superuser', sa.Boolean(), default=False))
+
+
+def downgrade():
+    op.drop_column('users', 'superuser')
diff --git a/airflow/migrations/versions/4446e08588_dagrun_start_end.py b/airflow/migrations/versions/4446e08588_dagrun_start_end.py
index dc546f8c77750..29932c92060e9 100644
--- a/airflow/migrations/versions/4446e08588_dagrun_start_end.py
+++ b/airflow/migrations/versions/4446e08588_dagrun_start_end.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,15 +25,15 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+
 # revision identifiers, used by Alembic.
 revision = '4446e08588'
 down_revision = '561833c1c74b'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.add_column('dag_run', sa.Column('end_date', sa.DateTime(), nullable=True))
diff --git a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py
index 80af57836771a..655ff61042d1e 100644
--- a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py
+++ b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,93 +24,147 @@
 
 """
 
+from alembic import op
+from sqlalchemy.dialects import mysql
+from alembic import context
+
 # revision identifiers, used by Alembic.
 revision = '4addfa1236f1'
 down_revision = 'f2ca10b85618'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
-from alembic import context
-
 
 def upgrade():
     if context.config.get_main_option('sqlalchemy.url').startswith('mysql'):
-        op.alter_column(table_name='dag', column_name='last_scheduler_run', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='dag', column_name='last_pickled', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='dag', column_name='last_expired', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='dag_pickle', column_name='created_dttm', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='dag_run', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='dag_run', column_name='start_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='dag_run', column_name='end_date', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='import_error', column_name='timestamp', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='job', column_name='start_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='job', column_name='end_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='job', column_name='latest_heartbeat', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='known_event', column_name='start_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='known_event', column_name='end_date', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='log', column_name='dttm', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='log', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=mysql.DATETIME(fsp=6), nullable=False)
-        op.alter_column(table_name='sla_miss', column_name='timestamp', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='task_fail', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='task_fail', column_name='start_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='task_fail', column_name='end_date', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='task_instance', column_name='execution_date', type_=mysql.DATETIME(fsp=6), nullable=False)
-        op.alter_column(table_name='task_instance', column_name='start_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='task_instance', column_name='end_date', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='task_instance', column_name='queued_dttm', type_=mysql.DATETIME(fsp=6))
-
-        op.alter_column(table_name='xcom', column_name='timestamp', type_=mysql.DATETIME(fsp=6))
-        op.alter_column(table_name='xcom', column_name='execution_date', type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='dag', column_name='last_scheduler_run',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='dag', column_name='last_pickled',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='dag', column_name='last_expired',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='dag_pickle', column_name='created_dttm',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='dag_run', column_name='execution_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='dag_run', column_name='start_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='dag_run', column_name='end_date',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='import_error', column_name='timestamp',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='job', column_name='start_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='job', column_name='end_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='job', column_name='latest_heartbeat',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='known_event', column_name='start_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='known_event', column_name='end_date',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='log', column_name='dttm',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='log', column_name='execution_date',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='sla_miss', column_name='execution_date',
+                        type_=mysql.DATETIME(fsp=6),
+                        nullable=False)
+        op.alter_column(table_name='sla_miss', column_name='timestamp',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='task_fail', column_name='execution_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='task_fail', column_name='start_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='task_fail', column_name='end_date',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='task_instance', column_name='execution_date',
+                        type_=mysql.DATETIME(fsp=6),
+                        nullable=False)
+        op.alter_column(table_name='task_instance', column_name='start_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='task_instance', column_name='end_date',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='task_instance', column_name='queued_dttm',
+                        type_=mysql.DATETIME(fsp=6))
+
+        op.alter_column(table_name='xcom', column_name='timestamp',
+                        type_=mysql.DATETIME(fsp=6))
+        op.alter_column(table_name='xcom', column_name='execution_date',
+                        type_=mysql.DATETIME(fsp=6))
 
 
 def downgrade():
     if context.config.get_main_option('sqlalchemy.url').startswith('mysql'):
-        op.alter_column(table_name='dag', column_name='last_scheduler_run', type_=mysql.DATETIME())
-        op.alter_column(table_name='dag', column_name='last_pickled', type_=mysql.DATETIME())
-        op.alter_column(table_name='dag', column_name='last_expired', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='dag_pickle', column_name='created_dttm', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='dag_run', column_name='execution_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='dag_run', column_name='start_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='dag_run', column_name='end_date', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='import_error', column_name='timestamp', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='job', column_name='start_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='job', column_name='end_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='job', column_name='latest_heartbeat', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='known_event', column_name='start_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='known_event', column_name='end_date', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='log', column_name='dttm', type_=mysql.DATETIME())
-        op.alter_column(table_name='log', column_name='execution_date', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='sla_miss', column_name='execution_date', type_=mysql.DATETIME(), nullable=False)
-        op.alter_column(table_name='sla_miss', column_name='timestamp', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='task_fail', column_name='execution_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='task_fail', column_name='start_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='task_fail', column_name='end_date', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='task_instance', column_name='execution_date', type_=mysql.DATETIME(), nullable=False)
-        op.alter_column(table_name='task_instance', column_name='start_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='task_instance', column_name='end_date', type_=mysql.DATETIME())
-        op.alter_column(table_name='task_instance', column_name='queued_dttm', type_=mysql.DATETIME())
-
-        op.alter_column(table_name='xcom', column_name='timestamp', type_=mysql.DATETIME())
-        op.alter_column(table_name='xcom', column_name='execution_date', type_=mysql.DATETIME())
+        op.alter_column(table_name='dag', column_name='last_scheduler_run',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='dag', column_name='last_pickled',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='dag', column_name='last_expired',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='dag_pickle', column_name='created_dttm',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='dag_run', column_name='execution_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='dag_run', column_name='start_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='dag_run', column_name='end_date',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='import_error', column_name='timestamp',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='job', column_name='start_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='job', column_name='end_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='job', column_name='latest_heartbeat',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='known_event', column_name='start_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='known_event', column_name='end_date',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='log', column_name='dttm',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='log', column_name='execution_date',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='sla_miss', column_name='execution_date',
+                        type_=mysql.DATETIME(), nullable=False)
+        op.alter_column(table_name='sla_miss', column_name='timestamp',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='task_fail', column_name='execution_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='task_fail', column_name='start_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='task_fail', column_name='end_date',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='task_instance', column_name='execution_date',
+                        type_=mysql.DATETIME(),
+                        nullable=False)
+        op.alter_column(table_name='task_instance', column_name='start_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='task_instance', column_name='end_date',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='task_instance', column_name='queued_dttm',
+                        type_=mysql.DATETIME())
+
+        op.alter_column(table_name='xcom', column_name='timestamp',
+                        type_=mysql.DATETIME())
+        op.alter_column(table_name='xcom', column_name='execution_date',
+                        type_=mysql.DATETIME())
diff --git a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py
index f5cded7ea5776..632720a4e27b0 100644
--- a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py
+++ b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,8 @@
 Create Date: 2015-11-03 22:50:49.794097
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '502898887f84'
@@ -31,9 +33,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.add_column('log', sa.Column('extra', sa.Text(), nullable=True))
diff --git a/airflow/migrations/versions/52d714495f0_job_id_indices.py b/airflow/migrations/versions/52d714495f0_job_id_indices.py
index 43893dba8a081..94374cb68aa30 100644
--- a/airflow/migrations/versions/52d714495f0_job_id_indices.py
+++ b/airflow/migrations/versions/52d714495f0_job_id_indices.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,7 @@
 Create Date: 2015-10-20 03:17:01.962542
 
 """
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '52d714495f0'
@@ -31,12 +32,10 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
-    op.create_index('idx_job_state_heartbeat', 'job', ['state', 'latest_heartbeat'], unique=False)
+    op.create_index('idx_job_state_heartbeat', 'job',
+                    ['state', 'latest_heartbeat'], unique=False)
 
 
 def downgrade():
diff --git a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py
index ad82c9378b2f1..a26a105ac8299 100644
--- a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py
+++ b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,8 @@
 Create Date: 2015-11-30 06:51:25.872557
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '561833c1c74b'
@@ -31,9 +33,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.add_column('user', sa.Column('password', sa.String(255)))
diff --git a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py
index 5e2963f060665..77a35db48fad5 100644
--- a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py
+++ b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,15 +24,15 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+
 # revision identifiers, used by Alembic.
 revision = '5e7d17757c7a'
 down_revision = '8504051e801b'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.add_column('task_instance', sa.Column('pid', sa.Integer))
diff --git a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py
index be747383289a9..2def57e904870 100644
--- a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py
+++ b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,8 @@
 Create Date: 2016-08-03 14:02:59.203021
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '64de9cddf6c9'
@@ -30,9 +32,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.create_table(
@@ -47,5 +46,6 @@ def upgrade():
         sa.PrimaryKeyConstraint('id'),
     )
 
+
 def downgrade():
     op.drop_table('task_fail')
diff --git a/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py b/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py
index 47473e318d625..fdcbc59df8d00 100644
--- a/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py
+++ b/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,18 +25,18 @@
 
 """
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision = '8504051e801b'
 down_revision = '4addfa1236f1'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
-    op.create_index('idx_xcom_dag_task_date', 'xcom', ['dag_id', 'task_id', 'execution_date'], unique=False)
+    op.create_index('idx_xcom_dag_task_date', 'xcom',
+                    ['dag_id', 'task_id', 'execution_date'], unique=False)
 
 
 def downgrade():
diff --git a/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py b/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py
new file mode 100644
index 0000000000000..52a817081be7c
--- /dev/null
+++ b/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py
@@ -0,0 +1,88 @@
+# flake8: noqa
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""fix sqlite foreign key
+
+Revision ID: 856955da8476
+Revises: f23433877c24
+Create Date: 2018-06-17 15:54:53.844230
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '856955da8476'
+down_revision = 'f23433877c24'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+    if conn.dialect.name == 'sqlite':
+        # Fix broken foreign-key constraint for existing SQLite DBs.
+        #
+        # Re-define tables and use copy_from to avoid reflection
+        # which would fail because referenced user table doesn't exist.
+        #
+        # Use batch_alter_table to support SQLite workaround.
+        chart_table = sa.Table('chart',
+                               sa.MetaData(),
+                               sa.Column('id', sa.Integer(), nullable=False),
+                               sa.Column('label', sa.String(length=200), nullable=True),
+                               sa.Column('conn_id', sa.String(length=250), nullable=False),
+                               sa.Column('user_id', sa.Integer(), nullable=True),
+                               sa.Column('chart_type', sa.String(length=100), nullable=True),
+                               sa.Column('sql_layout', sa.String(length=50), nullable=True),
+                               sa.Column('sql', sa.Text(), nullable=True),
+                               sa.Column('y_log_scale', sa.Boolean(), nullable=True),
+                               sa.Column('show_datatable', sa.Boolean(), nullable=True),
+                               sa.Column('show_sql', sa.Boolean(), nullable=True),
+                               sa.Column('height', sa.Integer(), nullable=True),
+                               sa.Column('default_params', sa.String(length=5000), nullable=True),
+                               sa.Column('x_is_date', sa.Boolean(), nullable=True),
+                               sa.Column('iteration_no', sa.Integer(), nullable=True),
+                               sa.Column('last_modified', sa.DateTime(), nullable=True),
+                               sa.PrimaryKeyConstraint('id'))
+        with op.batch_alter_table('chart', copy_from=chart_table) as batch_op:
+            batch_op.create_foreign_key('chart_user_id_fkey', 'users',
+                                        ['user_id'], ['id'])
+
+        known_event_table = sa.Table('known_event',
+                                     sa.MetaData(),
+                                     sa.Column('id', sa.Integer(), nullable=False),
+                                     sa.Column('label', sa.String(length=200), nullable=True),
+                                     sa.Column('start_date', sa.DateTime(), nullable=True),
+                                     sa.Column('end_date', sa.DateTime(), nullable=True),
+                                     sa.Column('user_id', sa.Integer(), nullable=True),
+                                     sa.Column('known_event_type_id', sa.Integer(), nullable=True),
+                                     sa.Column('description', sa.Text(), nullable=True),
+                                     sa.ForeignKeyConstraint(['known_event_type_id'],
+                                                             ['known_event_type.id'], ),
+                                     sa.PrimaryKeyConstraint('id'))
+        with op.batch_alter_table('chart', copy_from=known_event_table) as batch_op:
+            batch_op.create_foreign_key('known_event_user_id_fkey', 'users',
+                                        ['user_id'], ['id'])
+
+
+def downgrade():
+    # Downgrade would fail because the broken FK constraint can't be re-created.
+    pass
diff --git a/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py b/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py
index 6bc48f1105639..633201b03b6cd 100644
--- a/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py
+++ b/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py
@@ -1,16 +1,22 @@
 # flake8: noqa
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 
 """add kubernetes scheduler uniqueness
 
@@ -19,6 +25,8 @@
 Create Date: 2018-04-03 15:31:20.814328
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '86770d1215c0'
@@ -26,20 +34,27 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
-
 RESOURCE_TABLE = "kube_worker_uuid"
 
 
 def upgrade():
+
+    columns_and_constraints = [
+        sa.Column("one_row_id", sa.Boolean, server_default=sa.true(), primary_key=True),
+        sa.Column("worker_uuid", sa.String(255))
+    ]
+
+    conn = op.get_bind()
+
+    # alembic creates an invalid SQL for mssql dialect
+    if conn.dialect.name not in ('mssql'):
+        columns_and_constraints.append(sa.CheckConstraint("one_row_id", name="kube_worker_one_row_id"))
+
     table = op.create_table(
         RESOURCE_TABLE,
-        sa.Column("one_row_id", sa.Boolean, server_default=sa.true(), primary_key=True),
-        sa.Column("worker_uuid", sa.String(255)),
-        sa.CheckConstraint("one_row_id", name="kube_worker_one_row_id")
+        *columns_and_constraints
     )
+
     op.bulk_insert(table, [
         {"worker_uuid": ""}
     ])
diff --git a/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py b/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py
new file mode 100644
index 0000000000000..b425b4aecec34
--- /dev/null
+++ b/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py
@@ -0,0 +1,62 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""task reschedule fk on cascade delete
+
+Revision ID: 939bb1e647c8
+Revises: 4ebbffe0a39a
+Create Date: 2019-02-04 20:21:50.669751
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '939bb1e647c8'
+down_revision = 'dd4ecb8fbee3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('task_reschedule') as batch_op:
+        batch_op.drop_constraint(
+            'task_reschedule_dag_task_date_fkey',
+            type_='foreignkey'
+        )
+        batch_op.create_foreign_key(
+            'task_reschedule_dag_task_date_fkey',
+            'task_instance',
+            ['task_id', 'dag_id', 'execution_date'],
+            ['task_id', 'dag_id', 'execution_date'],
+            ondelete='CASCADE'
+        )
+
+
+def downgrade():
+    with op.batch_alter_table('task_reschedule') as batch_op:
+        batch_op.drop_constraint(
+            'task_reschedule_dag_task_date_fkey',
+            type_='foreignkey'
+        )
+        batch_op.create_foreign_key(
+            'task_reschedule_dag_task_date_fkey',
+            'task_instance',
+            ['task_id', 'dag_id', 'execution_date'],
+            ['task_id', 'dag_id', 'execution_date']
+        )
diff --git a/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py b/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py
index 4c8bce9b471e7..6ff41baa28c7a 100644
--- a/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py
+++ b/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,7 @@
 Create Date: 2017-08-15 15:12:13.845074
 
 """
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '947454bf1dff'
@@ -31,9 +32,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.create_index('ti_job_id', 'task_instance', ['job_id'], unique=False)
diff --git a/airflow/migrations/versions/9635ae0956e7_index_faskfail.py b/airflow/migrations/versions/9635ae0956e7_index_faskfail.py
new file mode 100644
index 0000000000000..6b21c3474acc4
--- /dev/null
+++ b/airflow/migrations/versions/9635ae0956e7_index_faskfail.py
@@ -0,0 +1,41 @@
+# flake8: noqa
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""index-faskfail
+
+Revision ID: 9635ae0956e7
+Revises: 856955da8476
+Create Date: 2018-06-17 21:40:01.963540
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '9635ae0956e7'
+down_revision = '856955da8476'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index('idx_task_fail_dag_task_date', 'task_fail', ['dag_id', 'task_id', 'execution_date'], unique=False)
+
+
+def downgrade():
+    op.drop_index('idx_task_fail_dag_task_date', table_name='task_fail')
diff --git a/airflow/migrations/versions/__init__.py b/airflow/migrations/versions/__init__.py
index f0f8b68337da6..114d189da14ab 100644
--- a/airflow/migrations/versions/__init__.py
+++ b/airflow/migrations/versions/__init__.py
@@ -7,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
diff --git a/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py b/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py
new file mode 100644
index 0000000000000..89dba33c3e394
--- /dev/null
+++ b/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Remove dag_stat table
+
+Revision ID: a56c9515abdc
+Revises: c8ffec048a3b
+Create Date: 2018-12-27 10:27:59.715872
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'a56c9515abdc'
+down_revision = 'c8ffec048a3b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table("dag_stats")
+
+
+def downgrade():
+    op.create_table('dag_stats',
+                    sa.Column('dag_id', sa.String(length=250), nullable=False),
+                    sa.Column('state', sa.String(length=50), nullable=False),
+                    sa.Column('count', sa.Integer(), nullable=False, default=0),
+                    sa.Column('dirty', sa.Boolean(), nullable=False, default=False),
+                    sa.PrimaryKeyConstraint('dag_id', 'state'))
diff --git a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py
index c780e5830dce6..503cd0b6f0406 100644
--- a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py
+++ b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,18 +25,19 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+
 # revision identifiers, used by Alembic.
 revision = 'bba5a7cfc896'
 down_revision = 'bbc73705a13e'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
-    op.add_column('connection', sa.Column('is_extra_encrypted', sa.Boolean,default=False))
+    op.add_column('connection',
+                  sa.Column('is_extra_encrypted', sa.Boolean, default=False))
 
 
 def downgrade():
diff --git a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py
index 84c54fad7157a..9855a6d4daf3c 100644
--- a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py
+++ b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,6 +24,8 @@
 Create Date: 2016-01-14 18:05:54.871682
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = 'bbc73705a13e'
@@ -31,12 +33,9 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
-    op.add_column('sla_miss', sa.Column('notification_sent', sa.Boolean,default=False))
+    op.add_column('sla_miss', sa.Column('notification_sent', sa.Boolean, default=False))
 
 
 def downgrade():
diff --git a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py
index 5e06766182f6d..a1a5270c7bff4 100644
--- a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py
+++ b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,9 @@
 Create Date: 2017-08-14 16:06:31.568971
 
 """
+from alembic import op
+import dill
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = 'bdaa763e6c56'
@@ -30,15 +33,10 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import dill
-import sqlalchemy as sa
-
 
 def upgrade():
-    # There can be data truncation here as LargeBinary can be smaller than the pickle 
+    # There can be data truncation here as LargeBinary can be smaller than the pickle
     # type.
-
     # use batch_alter_table to support SQLite workaround
     with op.batch_alter_table("xcom") as batch_op:
         batch_op.alter_column('value', type_=sa.LargeBinary())
diff --git a/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py b/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py
new file mode 100644
index 0000000000000..528bd53b366e5
--- /dev/null
+++ b/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""add index to taskinstance
+
+Revision ID: bf00311e1990
+Revises: dd25f486b8ea
+Create Date: 2018-09-12 09:53:52.007433
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'bf00311e1990'
+down_revision = 'dd25f486b8ea'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index(
+        'ti_dag_date',
+        'task_instance',
+        ['dag_id', 'execution_date'],
+        unique=False
+    )
+
+
+def downgrade():
+    op.drop_index('ti_dag_date', table_name='task_instance')
diff --git a/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py b/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py
new file mode 100644
index 0000000000000..70282715edfa6
--- /dev/null
+++ b/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""add fields to dag
+
+Revision ID: c8ffec048a3b
+Revises: 41f5f12752f8
+Create Date: 2018-12-23 21:55:46.463634
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'c8ffec048a3b'
+down_revision = '41f5f12752f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('dag', sa.Column('description', sa.Text(), nullable=True))
+    op.add_column('dag', sa.Column('default_view', sa.String(25), nullable=True))
+
+
+def downgrade():
+    op.drop_column('dag', 'description')
+    op.drop_column('dag', 'default_view')
diff --git a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py
index 0503d93df5bec..03195a7536d3d 100644
--- a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py
+++ b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py
@@ -1,3 +1,4 @@
+# flake8: noqa
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -6,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,24 +25,38 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+from airflow import settings
+from airflow.models import DagBag
+
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.engine.reflection import Inspector
+from sqlalchemy.ext.declarative import declarative_base
+
 # revision identifiers, used by Alembic.
 revision = 'cc1e65623dc7'
 down_revision = '127d2bf2dfa7'
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-from airflow import settings
-from airflow.models import DagBag, TaskInstance
-from sqlalchemy.engine.reflection import Inspector
-
+Base = declarative_base()
 BATCH_SIZE = 5000
+ID_LEN = 250
+
+
+class TaskInstance(Base):
+    __tablename__ = "task_instance"
+
+    task_id = Column(String(ID_LEN), primary_key=True)
+    dag_id = Column(String(ID_LEN), primary_key=True)
+    execution_date = Column(sa.DateTime, primary_key=True)
+    max_tries = Column(Integer)
+    try_number = Column(Integer, default=0)
 
 
 def upgrade():
-    op.add_column('task_instance', sa.Column('max_tries', sa.Integer,
-        server_default="-1"))
+    op.add_column('task_instance', sa.Column('max_tries', sa.Integer, server_default="-1"))
     # Check if table task_instance exist before data migration. This check is
     # needed for database that does not create table until migration finishes.
     # Checking task_instance table exists prevent the error of querying
@@ -111,7 +126,7 @@ def downgrade():
                     # max number of self retry (task.retries) minus number of
                     # times left for task instance to try the task.
                     ti.try_number = max(0, task.retries - (ti.max_tries -
-                        ti.try_number))
+                                                           ti.try_number))
                 ti.max_tries = -1
                 session.merge(ti)
             session.commit()
diff --git a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py
index ff67b4fb16d3a..db5afaf023e54 100644
--- a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py
+++ b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,9 @@
 Create Date: 2017-08-18 17:07:16.686130
 
 """
+from alembic import op
+from sqlalchemy.dialects import mysql
+from alembic import context
 
 # revision identifiers, used by Alembic.
 revision = 'd2ae31099d61'
@@ -30,11 +33,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
-from alembic import context
-
 
 def upgrade():
     if context.config.get_main_option('sqlalchemy.url').startswith('mysql'):
diff --git a/scripts/ci/load_fixtures.sh b/airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py
old mode 100755
new mode 100644
similarity index 64%
rename from scripts/ci/load_fixtures.sh
rename to airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py
index 55beb919dfda2..3249a2e0589cb
--- a/scripts/ci/load_fixtures.sh
+++ b/airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -8,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,17 +16,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -o verbose
+from alembic import op
+
+"""add idx_log_dag
+
+Revision ID: dd25f486b8ea
+Revises: 9635ae0956e7
+Create Date: 2018-08-07 06:41:41.028249
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'dd25f486b8ea'
+down_revision = '9635ae0956e7'
+branch_labels = None
+depends_on = None
+
 
-DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-FIXTURES_DIR="$DIR/ldif"
-LOAD_ORDER=("example.com.ldif" "manager.example.com.ldif" "users.example.com.ldif" "groups.example.com.ldif")
+def upgrade():
+    op.create_index('idx_log_dag', 'log', ['dag_id'], unique=False)
 
-load_fixture () {
-  ldapadd -x -H ldap://127.0.0.1:3890/ -D "cn=Manager,dc=example,dc=com" -w insecure -f $1
-}
 
-for FILE in "${LOAD_ORDER[@]}"
-do
-  load_fixture "${FIXTURES_DIR}/${FILE}"
-done;
+def downgrade():
+    op.drop_index('idx_log_dag', table_name='log')
diff --git a/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py b/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py
new file mode 100644
index 0000000000000..3b2e6d577ac9d
--- /dev/null
+++ b/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Add schedule interval to dag
+
+Revision ID: dd4ecb8fbee3
+Revises: c8ffec048a3b
+Create Date: 2018-12-27 18:39:25.748032
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'dd4ecb8fbee3'
+down_revision = 'c8ffec048a3b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('dag', sa.Column('schedule_interval', sa.Text(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('dag', sa.Column('schedule_interval', sa.Text(), nullable=True))
diff --git a/airflow/migrations/versions/e3a246e0dc1_current_schema.py b/airflow/migrations/versions/e3a246e0dc1_current_schema.py
index 6c63d0a9dd337..cbf98976458d4 100644
--- a/airflow/migrations/versions/e3a246e0dc1_current_schema.py
+++ b/airflow/migrations/versions/e3a246e0dc1_current_schema.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,17 +25,17 @@
 
 """
 
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import func
+from sqlalchemy.engine.reflection import Inspector
+
 # revision identifiers, used by Alembic.
 revision = 'e3a246e0dc1'
 down_revision = None
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import func
-from sqlalchemy.engine.reflection import Inspector
-
 
 def upgrade():
     conn = op.get_bind()
diff --git a/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py b/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py
new file mode 100644
index 0000000000000..3e643f629dfcd
--- /dev/null
+++ b/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py
@@ -0,0 +1,52 @@
+# flake8: noqa
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""fix mysql not null constraint
+
+Revision ID: f23433877c24
+Revises: 05f30312d566
+Create Date: 2018-06-17 10:16:31.412131
+
+"""
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = 'f23433877c24'
+down_revision = '05f30312d566'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+    if conn.dialect.name == 'mysql':
+        conn.execute("SET time_zone = '+00:00'")
+        op.alter_column('task_fail', 'execution_date', existing_type=mysql.TIMESTAMP(fsp=6), nullable=False)
+        op.alter_column('xcom', 'execution_date', existing_type=mysql.TIMESTAMP(fsp=6), nullable=False)
+        op.alter_column('xcom', 'timestamp', existing_type=mysql.TIMESTAMP(fsp=6), nullable=False)
+
+
+def downgrade():
+    conn = op.get_bind()
+    if conn.dialect.name == 'mysql':
+        conn.execute("SET time_zone = '+00:00'")
+        op.alter_column('xcom', 'timestamp', existing_type=mysql.TIMESTAMP(fsp=6), nullable=True)
+        op.alter_column('xcom', 'execution_date', existing_type=mysql.TIMESTAMP(fsp=6), nullable=True)
+        op.alter_column('task_fail', 'execution_date', existing_type=mysql.TIMESTAMP(fsp=6), nullable=True)
diff --git a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py
index 7c23d507127a2..e14b4b8025109 100644
--- a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py
+++ b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,8 @@
 Create Date: 2016-07-20 15:08:28.247537
 
 """
+from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = 'f2ca10b85618'
@@ -30,9 +32,6 @@
 branch_labels = None
 depends_on = None
 
-from alembic import op
-import sqlalchemy as sa
-
 
 def upgrade():
     op.create_table('dag_stats',
diff --git a/airflow/minihivecluster.py b/airflow/minihivecluster.py
deleted file mode 100644
index c5441c67a1b61..0000000000000
--- a/airflow/minihivecluster.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2016 Bolke de Bruin
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-import os
-import subprocess
-import select
-import re
-
-
-class MiniHiveCluster(object):
-    def __init__(self):
-        self._minicluster_home = os.environ['MINICLUSTER_HOME']
-        self._minicluster_class = "com.ing.minicluster.MiniCluster"
-        self._start_mini_cluster()
-        self._is_started()
-
-    def _start_mini_cluster(self):
-        classpath = os.path.join(self._minicluster_home, "*")
-        cmd = ["java", "-cp", classpath, self._minicluster_class]
-
-        self.hive = subprocess.Popen(cmd, bufsize=0, stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE, universal_newlines=True,
-                                     close_fds=True)
-
-    def terminate(self):
-        self.hive.terminate()
-
-    def _is_started(self):
-        while self.hive.poll() is None:
-            rlist, wlist, xlist = select.select([self.hive.stderr, self.hive.stdout], [], [])
-            for f in rlist:
-                line = f.readline()
-                print(line,)
-                m = re.match(".*Starting ThriftBinaryCLIService", line)
-                if m:
-                    return True
diff --git a/airflow/models.py b/airflow/models/__init__.py
similarity index 80%
rename from airflow/models.py
rename to airflow/models/__init__.py
index 4706c2d50b6e1..702ed7f7e0431 100755
--- a/airflow/models.py
+++ b/airflow/models/__init__.py
@@ -22,13 +22,21 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+from builtins import ImportError as BuiltinImportError, bytes, object, str
+from collections import defaultdict, namedtuple, OrderedDict
+import copy
+from typing import Iterable
+
 from future.standard_library import install_aliases
 
-from builtins import str
-from builtins import object, bytes
-import copy
-from collections import namedtuple, defaultdict
-import cryptography
+from airflow.models.base import Base, ID_LEN
+
+try:
+    # Fix Python > 3.7 deprecation
+    from collections.abc import Hashable
+except BuiltinImportError:
+    # Preserve Python < 3.3 compatibility
+    from collections import Hashable
 from datetime import timedelta
 
 import dill
@@ -36,51 +44,61 @@
 import getpass
 import imp
 import importlib
-import itertools
 import zipfile
 import jinja2
 import json
 import logging
-import numbers
 import os
+import pendulum
 import pickle
 import re
 import signal
 import sys
+import time
 import textwrap
 import traceback
 import warnings
 import hashlib
 
-import uuid
 from datetime import datetime
-from urllib.parse import urlparse, quote, parse_qsl
+from urllib.parse import quote
 
 from sqlalchemy import (
-    Column, Integer, String, DateTime, Text, Boolean, ForeignKey, PickleType,
-    Index, Float, LargeBinary)
-from sqlalchemy import func, or_, and_, true as sqltrue
-from sqlalchemy.ext.declarative import declarative_base, declared_attr
+    Boolean, Column, DateTime, Float, ForeignKey, Index,
+    Integer, LargeBinary, PickleType, String, Text, UniqueConstraint, and_,
+    func, or_
+)
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import reconstructor, relationship, synonym
-from sqlalchemy_utc import UtcDateTime
 
-from croniter import croniter
+from croniter import (
+    croniter, CroniterBadCronError, CroniterBadDateError, CroniterNotAlphaError
+)
 import six
 
 from airflow import settings, utils
 from airflow.executors import GetDefaultExecutor, LocalExecutor
 from airflow import configuration
 from airflow.exceptions import (
-    AirflowDagCycleException, AirflowException, AirflowSkipException, AirflowTaskTimeout
+    AirflowDagCycleException, AirflowException, AirflowSkipException, AirflowTaskTimeout,
+    AirflowRescheduleException
 )
 from airflow.dag.base_dag import BaseDag, BaseDagBag
 from airflow.lineage import apply_lineage, prepare_lineage
+from airflow.models.dagpickle import DagPickle
+from airflow.models.errors import ImportError  # noqa: F401
+from airflow.models.slamiss import SlaMiss  # noqa: F401
+from airflow.models.kubernetes import KubeWorkerIdentifier, KubeResourceVersion  # noqa: F401
+from airflow.models.log import Log
+from airflow.models.taskfail import TaskFail
+from airflow.models.taskreschedule import TaskReschedule
 from airflow.ti_deps.deps.not_in_retry_period_dep import NotInRetryPeriodDep
 from airflow.ti_deps.deps.prev_dagrun_dep import PrevDagrunDep
 from airflow.ti_deps.deps.trigger_rule_dep import TriggerRuleDep
 
 from airflow.ti_deps.dep_context import DepContext, QUEUE_DEPS, RUN_DEPS
 from airflow.utils import timezone
+from airflow.utils.dag_processing import list_py_file_paths
 from airflow.utils.dates import cron_presets, date_range as utils_date_range
 from airflow.utils.db import provide_session
 from airflow.utils.decorators import apply_defaults
@@ -89,21 +107,48 @@
     as_tuple, is_container, validate_key, pprinttable)
 from airflow.utils.operator_resources import Resources
 from airflow.utils.state import State
+from airflow.utils.sqlalchemy import UtcDateTime, Interval
 from airflow.utils.timeout import timeout
 from airflow.utils.trigger_rule import TriggerRule
 from airflow.utils.weight_rule import WeightRule
 from airflow.utils.net import get_hostname
 from airflow.utils.log.logging_mixin import LoggingMixin
 
+
 install_aliases()
 
-Base = declarative_base()
-ID_LEN = 250
 XCOM_RETURN_KEY = 'return_value'
 
 Stats = settings.Stats
 
 
+class InvalidFernetToken(Exception):
+    # If Fernet isn't loaded we need a valid exception class to catch. If it is
+    # loaded this will get reset to the actual class once get_fernet() is called
+    pass
+
+
+class NullFernet(object):
+    """
+    A "Null" encryptor class that doesn't encrypt or decrypt but that presents
+    a similar interface to Fernet.
+
+    The purpose of this is to make the rest of the code not have to know the
+    difference, and to only display the message once, not 20 times when
+    `airflow initdb` is ran.
+    """
+    is_encrypted = False
+
+    def decrpyt(self, b):
+        return b
+
+    def encrypt(self, b):
+        return b
+
+
+_fernet = None
+
+
 def get_fernet():
     """
     Deferred load of Fernet key.
@@ -112,17 +157,43 @@ def get_fernet():
     or because the Fernet key is invalid.
 
     :return: Fernet object
-    :raises: AirflowException if there's a problem trying to load Fernet
+    :raises: airflow.exceptions.AirflowException if there's a problem trying to load Fernet
     """
+    global _fernet
+    log = LoggingMixin().log
+
+    if _fernet:
+        return _fernet
     try:
-        from cryptography.fernet import Fernet
-    except ImportError:
-        raise AirflowException('Failed to import Fernet, it may not be installed')
+        from cryptography.fernet import Fernet, MultiFernet, InvalidToken
+        global InvalidFernetToken
+        InvalidFernetToken = InvalidToken
+
+    except BuiltinImportError:
+        log.warning(
+            "cryptography not found - values will not be stored encrypted."
+        )
+        _fernet = NullFernet()
+        return _fernet
+
     try:
-        return Fernet(configuration.conf.get('core', 'FERNET_KEY').encode('utf-8'))
+        fernet_key = configuration.conf.get('core', 'FERNET_KEY')
+        if not fernet_key:
+            log.warning(
+                "empty cryptography key - values will not be stored encrypted."
+            )
+            _fernet = NullFernet()
+        else:
+            _fernet = MultiFernet([
+                Fernet(fernet_part.encode('utf-8'))
+                for fernet_part in fernet_key.split(',')
+            ])
+            _fernet.is_encrypted = True
     except (ValueError, TypeError) as ve:
         raise AirflowException("Could not create Fernet object: {}".format(ve))
 
+    return _fernet
+
 
 # Used by DAG context_managers
 _CONTEXT_MANAGER_DAG = None
@@ -178,6 +249,20 @@ def clear_task_instances(tis,
             dr.start_date = timezone.utcnow()
 
 
+def get_last_dagrun(dag_id, session, include_externally_triggered=False):
+    """
+    Returns the last dag run for a dag, None if there was none.
+    Last dag run can be any type of run eg. scheduled or backfilled.
+    Overridden DagRuns are ignored.
+    """
+    DR = DagRun
+    query = session.query(DR).filter(DR.dag_id == dag_id)
+    if not include_externally_triggered:
+        query = query.filter(DR.external_trigger == False)  # noqa
+    query = query.order_by(DR.execution_date.desc())
+    return query.first()
+
+
 class DagBag(BaseDagBag, LoggingMixin):
     """
     A dagbag is a collection of dags, parsed out of a folder tree and has high
@@ -210,7 +295,8 @@ def __init__(
             self,
             dag_folder=None,
             executor=None,
-            include_examples=configuration.conf.getboolean('core', 'LOAD_EXAMPLES')):
+            include_examples=configuration.conf.getboolean('core', 'LOAD_EXAMPLES'),
+            safe_mode=configuration.conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE')):
 
         # do not use default arg in signature, to fix import cycle on plugin load
         if executor is None:
@@ -225,12 +311,10 @@ def __init__(
         self.import_errors = {}
         self.has_logged = False
 
-        if include_examples:
-            example_dag_folder = os.path.join(
-                os.path.dirname(__file__),
-                'example_dags')
-            self.collect_dags(example_dag_folder)
-        self.collect_dags(dag_folder)
+        self.collect_dags(
+            dag_folder=dag_folder,
+            include_examples=include_examples,
+            safe_mode=safe_mode)
 
     def size(self):
         """
@@ -238,6 +322,10 @@ def size(self):
         """
         return len(self.dags)
 
+    @property
+    def dag_ids(self):
+        return self.dags.keys()
+
     def get_dag(self, dag_id):
         """
         Gets the DAG out of the dictionary, and refreshes it if expired
@@ -296,7 +384,8 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True):
             return found_dags
 
         mods = []
-        if not zipfile.is_zipfile(filepath):
+        is_zipfile = zipfile.is_zipfile(filepath)
+        if not is_zipfile:
             if safe_mode and os.path.isfile(filepath):
                 with open(filepath, 'rb') as f:
                     content = f.read()
@@ -368,13 +457,23 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True):
                 if isinstance(dag, DAG):
                     if not dag.full_filepath:
                         dag.full_filepath = filepath
-                        if dag.fileloc != filepath:
+                        if dag.fileloc != filepath and not is_zipfile:
                             dag.fileloc = filepath
                     try:
                         dag.is_subdag = False
                         self.bag_dag(dag, parent_dag=dag, root_dag=dag)
+                        if isinstance(dag._schedule_interval, six.string_types):
+                            croniter(dag._schedule_interval)
                         found_dags.append(dag)
                         found_dags += dag.subdags
+                    except (CroniterBadCronError,
+                            CroniterBadDateError,
+                            CroniterNotAlphaError) as cron_e:
+                        self.log.exception("Failed to bag_dag: %s", dag.full_filepath)
+                        self.import_errors[dag.full_filepath] = \
+                            "Invalid Cron expression: " + str(cron_e)
+                        self.file_last_changed[dag.full_filepath] = \
+                            file_last_changed_on_disk
                     except AirflowDagCycleException as cycle_exception:
                         self.log.exception("Failed to bag_dag: %s", dag.full_filepath)
                         self.import_errors[dag.full_filepath] = str(cycle_exception)
@@ -385,39 +484,28 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True):
         return found_dags
 
     @provide_session
-    def kill_zombies(self, session=None):
-        """
-        Fails tasks that haven't had a heartbeat in too long
-        """
-        from airflow.jobs import LocalTaskJob as LJ
-        self.log.info("Finding 'running' jobs without a recent heartbeat")
-        TI = TaskInstance
-        secs = configuration.conf.getint('scheduler', 'scheduler_zombie_task_threshold')
-        limit_dttm = timezone.utcnow() - timedelta(seconds=secs)
-        self.log.info("Failing jobs without heartbeat after %s", limit_dttm)
-
-        tis = (
-            session.query(TI)
-            .join(LJ, TI.job_id == LJ.id)
-            .filter(TI.state == State.RUNNING)
-            .filter(
-                or_(
-                    LJ.state != State.RUNNING,
-                    LJ.latest_heartbeat < limit_dttm,
-                ))
-            .all()
-        )
-
-        for ti in tis:
-            if ti and ti.dag_id in self.dags:
-                dag = self.dags[ti.dag_id]
-                if ti.task_id in dag.task_ids:
-                    task = dag.get_task(ti.task_id)
-
-                    # now set non db backed vars on ti
-                    ti.task = task
+    def kill_zombies(self, zombies, session=None):
+        """
+        Fail given zombie tasks, which are tasks that haven't
+        had a heartbeat for too long, in the current DagBag.
+
+        :param zombies: zombie task instances to kill.
+        :type zombies: airflow.utils.dag_processing.SimpleTaskInstance
+        :param session: DB session.
+        :type session: sqlalchemy.orm.session.Session
+        """
+        for zombie in zombies:
+            if zombie.dag_id in self.dags:
+                dag = self.dags[zombie.dag_id]
+                if zombie.task_id in dag.task_ids:
+                    task = dag.get_task(zombie.task_id)
+                    ti = TaskInstance(task, zombie.execution_date)
+                    # Get properties needed for failure handling from SimpleTaskInstance.
+                    ti.start_date = zombie.start_date
+                    ti.end_date = zombie.end_date
+                    ti.try_number = zombie.try_number
+                    ti.state = zombie.state
                     ti.test_mode = configuration.getboolean('core', 'unit_test_mode')
-
                     ti.handle_failure("{} detected as zombie".format(ti),
                                       ti.test_mode, ti.get_template_context())
                     self.log.info(
@@ -449,7 +537,7 @@ def bag_dag(self, dag, parent_dag, root_dag):
                 self.bag_dag(subdag, parent_dag=dag, root_dag=root_dag)
 
             self.dags[dag.dag_id] = dag
-            self.log.debug('Loaded DAG {dag}'.format(**locals()))
+            self.log.debug('Loaded DAG %s', dag)
         except AirflowDagCycleException as cycle_exception:
             # There was an error in bagging the dag. Remove it from the list of dags
             self.log.exception('Exception bagging dag: {dag.dag_id}'.format(**locals()))
@@ -464,15 +552,20 @@ def bag_dag(self, dag, parent_dag, root_dag):
     def collect_dags(
             self,
             dag_folder=None,
-            only_if_updated=True):
+            only_if_updated=True,
+            include_examples=configuration.conf.getboolean('core', 'LOAD_EXAMPLES'),
+            safe_mode=configuration.conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE')):
         """
         Given a file path or a folder, this method looks for python modules,
         imports them and adds them to the dagbag collection.
 
-        Note that if a .airflowignore file is found while processing,
-        the directory, it will behaves much like a .gitignore does,
+        Note that if a ``.airflowignore`` file is found while processing
+        the directory, it will behave much like a ``.gitignore``,
         ignoring files that match any of the regex patterns specified
         in the file.
+
+        **Note**: The patterns in .airflowignore are treated as
+        un-anchored regexes, not shell-like glob patterns.
         """
         start_dttm = timezone.utcnow()
         dag_folder = dag_folder or self.dag_folder
@@ -481,42 +574,27 @@ def collect_dags(
         stats = []
         FileLoadStat = namedtuple(
             'FileLoadStat', "file duration dag_num task_num dags")
-        if os.path.isfile(dag_folder):
-            self.process_file(dag_folder, only_if_updated=only_if_updated)
-        elif os.path.isdir(dag_folder):
-            for root, dirs, files in os.walk(dag_folder, followlinks=True):
-                patterns = []
-                ignore_file = os.path.join(root, '.airflowignore')
-                if os.path.isfile(ignore_file):
-                    with open(ignore_file, 'r') as f:
-                        patterns += [p for p in f.read().split('\n') if p]
-                for f in files:
-                    try:
-                        filepath = os.path.join(root, f)
-                        if not os.path.isfile(filepath):
-                            continue
-                        mod_name, file_ext = os.path.splitext(
-                            os.path.split(filepath)[-1])
-                        if file_ext != '.py' and not zipfile.is_zipfile(filepath):
-                            continue
-                        if not any(
-                                [re.findall(p, filepath) for p in patterns]):
-                            ts = timezone.utcnow()
-                            found_dags = self.process_file(
-                                filepath, only_if_updated=only_if_updated)
-
-                            td = timezone.utcnow() - ts
-                            td = td.total_seconds() + (
-                                float(td.microseconds) / 1000000)
-                            stats.append(FileLoadStat(
-                                filepath.replace(dag_folder, ''),
-                                td,
-                                len(found_dags),
-                                sum([len(dag.tasks) for dag in found_dags]),
-                                str([dag.dag_id for dag in found_dags]),
-                            ))
-                    except Exception as e:
-                        self.log.exception(e)
+
+        for filepath in list_py_file_paths(dag_folder, safe_mode=safe_mode,
+                                           include_examples=include_examples):
+            try:
+                ts = timezone.utcnow()
+                found_dags = self.process_file(
+                    filepath, only_if_updated=only_if_updated,
+                    safe_mode=safe_mode)
+
+                td = timezone.utcnow() - ts
+                td = td.total_seconds() + (
+                    float(td.microseconds) / 1000000)
+                stats.append(FileLoadStat(
+                    filepath.replace(dag_folder, ''),
+                    td,
+                    len(found_dags),
+                    sum([len(dag.tasks) for dag in found_dags]),
+                    str([dag.dag_id for dag in found_dags]),
+                ))
+            except Exception as e:
+                self.log.exception(e)
         Stats.gauge(
             'collect_dags', (timezone.utcnow() - start_dttm).total_seconds(), 1)
         Stats.gauge(
@@ -546,21 +624,6 @@ def dagbag_report(self):
             table=pprinttable(stats),
         )
 
-    @provide_session
-    def deactivate_inactive_dags(self, session=None):
-        active_dag_ids = [dag.dag_id for dag in list(self.dags.values())]
-        for dag in session.query(
-                DagModel).filter(~DagModel.dag_id.in_(active_dag_ids)).all():
-            dag.is_active = False
-            session.merge(dag)
-        session.commit()
-
-    @provide_session
-    def paused_dags(self, session=None):
-        dag_ids = [dp.dag_id for dp in session.query(DagModel).filter(
-            DagModel.is_paused.__eq__(True))]
-        return dag_ids
-
 
 class User(Base):
     __tablename__ = "users"
@@ -568,7 +631,7 @@ class User(Base):
     id = Column(Integer, primary_key=True)
     username = Column(String(ID_LEN), unique=True)
     email = Column(String(500))
-    superuser = False
+    superuser = Column(Boolean(), default=False)
 
     def __repr__(self):
         return self.username
@@ -580,260 +643,6 @@ def is_superuser(self):
         return self.superuser
 
 
-class Connection(Base, LoggingMixin):
-    """
-    Placeholder to store information about different database instances
-    connection information. The idea here is that scripts use references to
-    database instances (conn_id) instead of hard coding hostname, logins and
-    passwords when using operators or hooks.
-    """
-    __tablename__ = "connection"
-
-    id = Column(Integer(), primary_key=True)
-    conn_id = Column(String(ID_LEN))
-    conn_type = Column(String(500))
-    host = Column(String(500))
-    schema = Column(String(500))
-    login = Column(String(500))
-    _password = Column('password', String(5000))
-    port = Column(Integer())
-    is_encrypted = Column(Boolean, unique=False, default=False)
-    is_extra_encrypted = Column(Boolean, unique=False, default=False)
-    _extra = Column('extra', String(5000))
-
-    _types = [
-        ('docker', 'Docker Registry',),
-        ('fs', 'File (path)'),
-        ('ftp', 'FTP',),
-        ('google_cloud_platform', 'Google Cloud Platform'),
-        ('hdfs', 'HDFS',),
-        ('http', 'HTTP',),
-        ('hive_cli', 'Hive Client Wrapper',),
-        ('hive_metastore', 'Hive Metastore Thrift',),
-        ('hiveserver2', 'Hive Server 2 Thrift',),
-        ('jdbc', 'Jdbc Connection',),
-        ('jenkins', 'Jenkins'),
-        ('mysql', 'MySQL',),
-        ('postgres', 'Postgres',),
-        ('oracle', 'Oracle',),
-        ('vertica', 'Vertica',),
-        ('presto', 'Presto',),
-        ('s3', 'S3',),
-        ('samba', 'Samba',),
-        ('sqlite', 'Sqlite',),
-        ('ssh', 'SSH',),
-        ('cloudant', 'IBM Cloudant',),
-        ('mssql', 'Microsoft SQL Server'),
-        ('mesos_framework-id', 'Mesos Framework ID'),
-        ('jira', 'JIRA',),
-        ('redis', 'Redis',),
-        ('wasb', 'Azure Blob Storage'),
-        ('databricks', 'Databricks',),
-        ('aws', 'Amazon Web Services',),
-        ('emr', 'Elastic MapReduce',),
-        ('snowflake', 'Snowflake',),
-        ('segment', 'Segment',),
-        ('azure_data_lake', 'Azure Data Lake'),
-        ('cassandra', 'Cassandra',),
-    ]
-
-    def __init__(
-            self, conn_id=None, conn_type=None,
-            host=None, login=None, password=None,
-            schema=None, port=None, extra=None,
-            uri=None):
-        self.conn_id = conn_id
-        if uri:
-            self.parse_from_uri(uri)
-        else:
-            self.conn_type = conn_type
-            self.host = host
-            self.login = login
-            self.password = password
-            self.schema = schema
-            self.port = port
-            self.extra = extra
-
-    def parse_from_uri(self, uri):
-        temp_uri = urlparse(uri)
-        hostname = temp_uri.hostname or ''
-        if '%2f' in hostname:
-            hostname = hostname.replace('%2f', '/').replace('%2F', '/')
-        conn_type = temp_uri.scheme
-        if conn_type == 'postgresql':
-            conn_type = 'postgres'
-        self.conn_type = conn_type
-        self.host = hostname
-        self.schema = temp_uri.path[1:]
-        self.login = temp_uri.username
-        self.password = temp_uri.password
-        self.port = temp_uri.port
-        if temp_uri.query:
-            self.extra = json.dumps(dict(parse_qsl(temp_uri.query)))
-
-    def get_password(self):
-        if self._password and self.is_encrypted:
-            try:
-                fernet = get_fernet()
-            except AirflowException:
-                raise AirflowException(
-                    "Can't decrypt encrypted password for login={}, \
-                    FERNET_KEY configuration is missing".format(self.login))
-            return fernet.decrypt(bytes(self._password, 'utf-8')).decode()
-        else:
-            return self._password
-
-    def set_password(self, value):
-        if value:
-            try:
-                fernet = get_fernet()
-                self._password = fernet.encrypt(bytes(value, 'utf-8')).decode()
-                self.is_encrypted = True
-            except AirflowException:
-                self.log.exception("Failed to load fernet while encrypting value, "
-                                   "using non-encrypted value.")
-                self._password = value
-                self.is_encrypted = False
-
-    @declared_attr
-    def password(cls):
-        return synonym('_password',
-                       descriptor=property(cls.get_password, cls.set_password))
-
-    def get_extra(self):
-        if self._extra and self.is_extra_encrypted:
-            try:
-                fernet = get_fernet()
-            except AirflowException:
-                raise AirflowException(
-                    "Can't decrypt `extra` params for login={},\
-                    FERNET_KEY configuration is missing".format(self.login))
-            return fernet.decrypt(bytes(self._extra, 'utf-8')).decode()
-        else:
-            return self._extra
-
-    def set_extra(self, value):
-        if value:
-            try:
-                fernet = get_fernet()
-                self._extra = fernet.encrypt(bytes(value, 'utf-8')).decode()
-                self.is_extra_encrypted = True
-            except AirflowException:
-                self.log.exception("Failed to load fernet while encrypting value, "
-                                   "using non-encrypted value.")
-                self._extra = value
-                self.is_extra_encrypted = False
-        else:
-            self._extra = value
-            self.is_extra_encrypted = False
-
-    @declared_attr
-    def extra(cls):
-        return synonym('_extra',
-                       descriptor=property(cls.get_extra, cls.set_extra))
-
-    def get_hook(self):
-        try:
-            if self.conn_type == 'mysql':
-                from airflow.hooks.mysql_hook import MySqlHook
-                return MySqlHook(mysql_conn_id=self.conn_id)
-            elif self.conn_type == 'google_cloud_platform':
-                from airflow.contrib.hooks.bigquery_hook import BigQueryHook
-                return BigQueryHook(bigquery_conn_id=self.conn_id)
-            elif self.conn_type == 'postgres':
-                from airflow.hooks.postgres_hook import PostgresHook
-                return PostgresHook(postgres_conn_id=self.conn_id)
-            elif self.conn_type == 'hive_cli':
-                from airflow.hooks.hive_hooks import HiveCliHook
-                return HiveCliHook(hive_cli_conn_id=self.conn_id)
-            elif self.conn_type == 'presto':
-                from airflow.hooks.presto_hook import PrestoHook
-                return PrestoHook(presto_conn_id=self.conn_id)
-            elif self.conn_type == 'hiveserver2':
-                from airflow.hooks.hive_hooks import HiveServer2Hook
-                return HiveServer2Hook(hiveserver2_conn_id=self.conn_id)
-            elif self.conn_type == 'sqlite':
-                from airflow.hooks.sqlite_hook import SqliteHook
-                return SqliteHook(sqlite_conn_id=self.conn_id)
-            elif self.conn_type == 'jdbc':
-                from airflow.hooks.jdbc_hook import JdbcHook
-                return JdbcHook(jdbc_conn_id=self.conn_id)
-            elif self.conn_type == 'mssql':
-                from airflow.hooks.mssql_hook import MsSqlHook
-                return MsSqlHook(mssql_conn_id=self.conn_id)
-            elif self.conn_type == 'oracle':
-                from airflow.hooks.oracle_hook import OracleHook
-                return OracleHook(oracle_conn_id=self.conn_id)
-            elif self.conn_type == 'vertica':
-                from airflow.contrib.hooks.vertica_hook import VerticaHook
-                return VerticaHook(vertica_conn_id=self.conn_id)
-            elif self.conn_type == 'cloudant':
-                from airflow.contrib.hooks.cloudant_hook import CloudantHook
-                return CloudantHook(cloudant_conn_id=self.conn_id)
-            elif self.conn_type == 'jira':
-                from airflow.contrib.hooks.jira_hook import JiraHook
-                return JiraHook(jira_conn_id=self.conn_id)
-            elif self.conn_type == 'redis':
-                from airflow.contrib.hooks.redis_hook import RedisHook
-                return RedisHook(redis_conn_id=self.conn_id)
-            elif self.conn_type == 'wasb':
-                from airflow.contrib.hooks.wasb_hook import WasbHook
-                return WasbHook(wasb_conn_id=self.conn_id)
-            elif self.conn_type == 'docker':
-                from airflow.hooks.docker_hook import DockerHook
-                return DockerHook(docker_conn_id=self.conn_id)
-            elif self.conn_type == 'azure_data_lake':
-                from airflow.contrib.hooks.azure_data_lake_hook import AzureDataLakeHook
-                return AzureDataLakeHook(azure_data_lake_conn_id=self.conn_id)
-            elif self.conn_type == 'cassandra':
-                from airflow.contrib.hooks.cassandra_hook import CassandraHook
-                return CassandraHook(cassandra_conn_id=self.conn_id)
-        except Exception:
-            pass
-
-    def __repr__(self):
-        return self.conn_id
-
-    @property
-    def extra_dejson(self):
-        """Returns the extra property by deserializing json."""
-        obj = {}
-        if self.extra:
-            try:
-                obj = json.loads(self.extra)
-            except Exception as e:
-                self.log.exception(e)
-                self.log.error("Failed parsing the json for conn_id %s", self.conn_id)
-
-        return obj
-
-
-class DagPickle(Base):
-    """
-    Dags can originate from different places (user repos, master repo, ...)
-    and also get executed in different places (different executors). This
-    object represents a version of a DAG and becomes a source of truth for
-    a BackfillJob execution. A pickle is a native python serialized object,
-    and in this case gets stored in the database for the duration of the job.
-
-    The executors pick up the DagPickle id and read the dag definition from
-    the database.
-    """
-    id = Column(Integer, primary_key=True)
-    pickle = Column(PickleType(pickler=dill))
-    created_dttm = Column(UtcDateTime, default=timezone.utcnow)
-    pickle_hash = Column(Text)
-
-    __tablename__ = "dag_pickle"
-
-    def __init__(self, dag):
-        self.dag_id = dag.dag_id
-        if hasattr(dag, 'template_env'):
-            dag.template_env = None
-        self.pickle_hash = hash(dag)
-        self.pickle = dag
-
-
 class TaskInstance(Base, LoggingMixin):
     """
     Task instances store the state of a task instance. This table is the
@@ -861,7 +670,7 @@ class TaskInstance(Base, LoggingMixin):
     max_tries = Column(Integer)
     hostname = Column(String(1000))
     unixname = Column(String(1000))
-    job_id = Column(Integer, index=True)
+    job_id = Column(Integer)
     pool = Column(String(50))
     queue = Column(String(50))
     priority_weight = Column(Integer)
@@ -872,9 +681,11 @@ class TaskInstance(Base, LoggingMixin):
 
     __table_args__ = (
         Index('ti_dag_state', dag_id, state),
+        Index('ti_dag_date', dag_id, execution_date),
         Index('ti_state', state),
         Index('ti_state_lkp', dag_id, task_id, execution_date, state),
         Index('ti_pool', pool, state, priority_weight),
+        Index('ti_job_id', job_id),
     )
 
     def __init__(self, task, execution_date, state=None):
@@ -921,7 +732,7 @@ def init_on_load(self):
     @property
     def try_number(self):
         """
-        Return the try number that this task number will be when it is acutally
+        Return the try number that this task number will be when it is actually
         run.
 
         If the TI is currently running, this will match the column in the
@@ -1046,15 +857,15 @@ def generate_command(dag_id,
         :type mark_success: bool
         :param ignore_all_deps: Ignore all ignorable dependencies.
             Overrides the other ignore_* parameters.
-        :type ignore_all_deps: boolean
+        :type ignore_all_deps: bool
         :param ignore_depends_on_past: Ignore depends_on_past parameter of DAGs
             (e.g. for Backfills)
-        :type ignore_depends_on_past: boolean
+        :type ignore_depends_on_past: bool
         :param ignore_task_deps: Ignore task-specific dependencies such as depends_on_past
             and trigger rule
-        :type ignore_task_deps: boolean
+        :type ignore_task_deps: bool
         :param ignore_ti_state: Ignore the task instance's previous failure/success
-        :type ignore_ti_state: boolean
+        :type ignore_ti_state: bool
         :param local: Whether to run the task locally
         :type local: bool
         :param pickle_id: If the DAG was serialized to the DB, the ID
@@ -1098,10 +909,10 @@ def log_url(self):
         BASE_URL = configuration.conf.get('webserver', 'BASE_URL')
         if settings.RBAC:
             return BASE_URL + (
-                "/log/list/"
-                "?_flt_3_dag_id={self.dag_id}"
-                "&_flt_3_task_id={self.task_id}"
-                "&_flt_3_execution_date={iso}"
+                "/log?"
+                "execution_date={iso}"
+                "&task_id={self.task_id}"
+                "&dag_id={self.dag_id}"
             ).format(**locals())
         else:
             return BASE_URL + (
@@ -1214,7 +1025,7 @@ def key(self):
         """
         Returns a tuple that identifies the task instance uniquely
         """
-        return self.dag_id, self.task_id, self.execution_date
+        return self.dag_id, self.task_id, self.execution_date, self.try_number
 
     @provide_session
     def set_state(self, state, session=None):
@@ -1257,11 +1068,8 @@ def are_dependents_done(self, session=None):
         count = ti[0][0]
         return count == len(task.downstream_task_ids)
 
-    @property
     @provide_session
-    def previous_ti(self, session=None):
-        """ The task instance for the task that ran before this task instance """
-
+    def _get_previous_ti(self, session=None):
         dag = self.task.dag
         if dag:
             dr = self.get_dagrun(session=session)
@@ -1287,6 +1095,11 @@ def previous_ti(self, session=None):
 
         return None
 
+    @property
+    def previous_ti(self):
+        """The task instance for the task that ran before this task instance."""
+        return self._get_previous_ti()
+
     @provide_session
     def are_dependencies_met(
             self,
@@ -1302,10 +1115,10 @@ def are_dependencies_met(
             should be evaluated.
         :type dep_context: DepContext
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :param verbose: whether log details on failed dependencies on
             info or debug log level
-        :type verbose: boolean
+        :type verbose: bool
         """
         dep_context = dep_context or DepContext()
         failed = False
@@ -1445,19 +1258,19 @@ def _check_and_change_state_before_execution(
         executed, in preparation for _run_raw_task
 
         :param verbose: whether to turn on more verbose logging
-        :type verbose: boolean
+        :type verbose: bool
         :param ignore_all_deps: Ignore all of the non-critical dependencies, just runs
-        :type ignore_all_deps: boolean
+        :type ignore_all_deps: bool
         :param ignore_depends_on_past: Ignore depends_on_past DAG attribute
-        :type ignore_depends_on_past: boolean
+        :type ignore_depends_on_past: bool
         :param ignore_task_deps: Don't check the dependencies of this TI's task
-        :type ignore_task_deps: boolean
+        :type ignore_task_deps: bool
         :param ignore_ti_state: Disregards previous task instance state
-        :type ignore_ti_state: boolean
+        :type ignore_ti_state: bool
         :param mark_success: Don't run the task, mark its state as success
-        :type mark_success: boolean
+        :type mark_success: bool
         :param test_mode: Doesn't record success or failure in the DB
-        :type test_mode: boolean
+        :type test_mode: bool
         :param pool: specifies the pool to use to run the task instance
         :type pool: str
         :return: whether the state was changed to running or not
@@ -1488,15 +1301,17 @@ def _check_and_change_state_before_execution(
             return False
 
         # TODO: Logging needs cleanup, not clear what is being printed
-        hr = "\n" + ("-" * 80) + "\n"  # Line break
+        hr = "\n" + ("-" * 80)  # Line break
 
         # For reporting purposes, we report based on 1-indexed,
         # not 0-indexed lists (i.e. Attempt 1 instead of
         # Attempt 0 for the first attempt).
-        msg = "Starting attempt {attempt} of {total}".format(
-            attempt=self.try_number,
-            total=self.max_tries + 1)
+        # Set the task start date. In case it was re-scheduled use the initial
+        # start date that is recorded in task_reschedule table
         self.start_date = timezone.utcnow()
+        task_reschedules = TaskReschedule.find_for_task_instance(self, session)
+        if task_reschedules:
+            self.start_date = task_reschedules[0].start_date
 
         dep_context = DepContext(
             deps=RUN_DEPS - QUEUE_DEPS,
@@ -1514,11 +1329,12 @@ def _check_and_change_state_before_execution(
             # have been running prematurely. This should be handled in the
             # scheduling mechanism.
             self.state = State.NONE
-            msg = ("FIXME: Rescheduling due to concurrency limits reached at task "
-                   "runtime. Attempt {attempt} of {total}. State set to NONE.").format(
-                attempt=self.try_number,
-                total=self.max_tries + 1)
-            self.log.warning(hr + msg + hr)
+            self.log.warning(hr)
+            self.log.warning(
+                "FIXME: Rescheduling due to concurrency limits reached at task runtime. Attempt %s of "
+                "%s. State set to NONE.", self.try_number, self.max_tries + 1
+            )
+            self.log.warning(hr)
 
             self.queued_dttm = timezone.utcnow()
             self.log.info("Queuing into pool %s", self.pool)
@@ -1529,13 +1345,14 @@ def _check_and_change_state_before_execution(
         # Another worker might have started running this task instance while
         # the current worker process was blocked on refresh_from_db
         if self.state == State.RUNNING:
-            msg = "Task Instance already running {}".format(self)
-            self.log.warning(msg)
+            self.log.warning("Task Instance already running %s", self)
             session.commit()
             return False
 
         # print status message
-        self.log.info(hr + msg + hr)
+        self.log.info(hr)
+        self.log.info("Starting attempt %s of %s", self.try_number, self.max_tries + 1)
+        self.log.info(hr)
         self._try_number += 1
 
         if not test_mode:
@@ -1552,12 +1369,9 @@ def _check_and_change_state_before_execution(
         settings.engine.dispose()
         if verbose:
             if mark_success:
-                msg = "Marking success for {} on {}".format(self.task,
-                                                            self.execution_date)
-                self.log.info(msg)
+                self.log.info("Marking success for %s on %s", self.task, self.execution_date)
             else:
-                msg = "Executing {} on {}".format(self.task, self.execution_date)
-                self.log.info(msg)
+                self.log.info("Executing %s on %s", self.task, self.execution_date)
         return True
 
     @provide_session
@@ -1575,9 +1389,9 @@ def _run_raw_task(
         only after another function changes the state to running.
 
         :param mark_success: Don't run the task, mark its state as success
-        :type mark_success: boolean
+        :type mark_success: bool
         :param test_mode: Doesn't record success or failure in the DB
-        :type test_mode: boolean
+        :type test_mode: bool
         :param pool: specifies the pool to use to run the task instance
         :type pool: str
         """
@@ -1590,6 +1404,7 @@ def _run_raw_task(
         self.operator = task.__class__.__name__
 
         context = {}
+        actual_start_date = timezone.utcnow()
         try:
             if not mark_success:
                 context = self.get_template_context()
@@ -1606,6 +1421,8 @@ def signal_handler(signum, frame):
                 # Don't clear Xcom until the task is certain to execute
                 self.clear_xcom_data()
 
+                start_time = time.time()
+
                 self.render_templates()
                 task_copy.pre_execute(context=context)
 
@@ -1643,6 +1460,14 @@ def signal_handler(signum, frame):
                     else:
                         raise
 
+                end_time = time.time()
+                duration = end_time - start_time
+                Stats.timing(
+                    'dag.{dag_id}.{task_id}.duration'.format(
+                        dag_id=task_copy.dag_id,
+                        task_id=task_copy.task_id),
+                    duration)
+
                 Stats.incr('operator_successes_{}'.format(
                     self.task.__class__.__name__), 1, 1)
                 Stats.incr('ti_successes')
@@ -1651,11 +1476,15 @@ def signal_handler(signum, frame):
         except AirflowSkipException:
             self.refresh_from_db(lock_for_update=True)
             self.state = State.SKIPPED
+        except AirflowRescheduleException as reschedule_exception:
+            self.refresh_from_db()
+            self._handle_reschedule(actual_start_date, reschedule_exception, test_mode, context)
+            return
         except AirflowException as e:
             self.refresh_from_db()
-            # for case when task is marked as success externally
+            # for case when task is marked as success/failed externally
             # current behavior doesn't hit the success callback
-            if self.state == State.SUCCESS:
+            if self.state in {State.SUCCESS, State.FAILED}:
                 return
             else:
                 self.handle_failure(e, test_mode, context)
@@ -1664,14 +1493,6 @@ def signal_handler(signum, frame):
             self.handle_failure(e, test_mode, context)
             raise
 
-        # Recording SUCCESS
-        self.end_date = timezone.utcnow()
-        self.set_duration()
-        if not test_mode:
-            session.add(Log(self.state, self))
-            session.merge(self)
-        session.commit()
-
         # Success callback
         try:
             if task.on_success_callback:
@@ -1680,6 +1501,12 @@ def signal_handler(signum, frame):
             self.log.error("Failed when executing success callback")
             self.log.exception(e3)
 
+        # Recording SUCCESS
+        self.end_date = timezone.utcnow()
+        self.set_duration()
+        if not test_mode:
+            session.add(Log(self.state, self))
+            session.merge(self)
         session.commit()
 
     @provide_session
@@ -1722,6 +1549,32 @@ def dry_run(self):
         self.render_templates()
         task_copy.dry_run()
 
+    @provide_session
+    def _handle_reschedule(self, actual_start_date, reschedule_exception, test_mode=False, context=None,
+                           session=None):
+        # Don't record reschedule request in test mode
+        if test_mode:
+            return
+
+        self.end_date = timezone.utcnow()
+        self.set_duration()
+
+        # Log reschedule request
+        session.add(TaskReschedule(self.task, self.execution_date, self._try_number,
+                    actual_start_date, self.end_date,
+                    reschedule_exception.reschedule_date))
+
+        # set state
+        self.state = State.UP_FOR_RESCHEDULE
+
+        # Decrement try_number so subsequent runs will use the same try number and write
+        # to same log file.
+        self._try_number -= 1
+
+        session.merge(self)
+        session.commit()
+        self.log.info('Rescheduling task, marking task as UP_FOR_RESCHEDULE')
+
     @provide_session
     def handle_failure(self, error, test_mode=False, context=None, session=None):
         self.log.exception(error)
@@ -1736,6 +1589,9 @@ def handle_failure(self, error, test_mode=False, context=None, session=None):
         # Log failure duration
         session.add(TaskFail(task, self.execution_date, self.start_date, self.end_date))
 
+        if context is not None:
+            context['exception'] = error
+
         # Let's go deeper
         try:
             # Since this function is called only when the TI state is running,
@@ -1746,7 +1602,7 @@ def handle_failure(self, error, test_mode=False, context=None, session=None):
                 self.state = State.UP_FOR_RETRY
                 self.log.info('Marking task as UP_FOR_RETRY')
                 if task.email_on_retry and task.email:
-                    self.email_alert(error, is_retry=True)
+                    self.email_alert(error)
             else:
                 self.state = State.FAILED
                 if task.retries:
@@ -1754,7 +1610,7 @@ def handle_failure(self, error, test_mode=False, context=None, session=None):
                 else:
                     self.log.info('Marking task as FAILED.')
                 if task.email_on_failure and task.email:
-                    self.email_alert(error, is_retry=False)
+                    self.email_alert(error)
         except Exception as e2:
             self.log.error('Failed to send email to: %s', task.email)
             self.log.exception(e2)
@@ -1785,47 +1641,60 @@ def get_template_context(self, session=None):
         if 'tables' in task.params:
             tables = task.params['tables']
 
+        params = {}
+        run_id = ''
+        dag_run = None
+        if hasattr(task, 'dag'):
+            if task.dag.params:
+                params.update(task.dag.params)
+            dag_run = (
+                session.query(DagRun)
+                .filter_by(
+                    dag_id=task.dag.dag_id,
+                    execution_date=self.execution_date)
+                .first()
+            )
+            run_id = dag_run.run_id if dag_run else None
+            session.expunge_all()
+            session.commit()
+
         ds = self.execution_date.strftime('%Y-%m-%d')
         ts = self.execution_date.isoformat()
         yesterday_ds = (self.execution_date - timedelta(1)).strftime('%Y-%m-%d')
         tomorrow_ds = (self.execution_date + timedelta(1)).strftime('%Y-%m-%d')
 
-        prev_execution_date = task.dag.previous_schedule(self.execution_date)
-        next_execution_date = task.dag.following_schedule(self.execution_date)
+        # For manually triggered dagruns that aren't run on a schedule, next/previous
+        # schedule dates don't make sense, and should be set to execution date for
+        # consistency with how execution_date is set for manually triggered tasks, i.e.
+        # triggered_date == execution_date.
+        if dag_run and dag_run.external_trigger:
+            prev_execution_date = self.execution_date
+            next_execution_date = self.execution_date
+        else:
+            prev_execution_date = task.dag.previous_schedule(self.execution_date)
+            next_execution_date = task.dag.following_schedule(self.execution_date)
 
         next_ds = None
+        next_ds_nodash = None
         if next_execution_date:
             next_ds = next_execution_date.strftime('%Y-%m-%d')
+            next_ds_nodash = next_ds.replace('-', '')
 
         prev_ds = None
+        prev_ds_nodash = None
         if prev_execution_date:
             prev_ds = prev_execution_date.strftime('%Y-%m-%d')
+            prev_ds_nodash = prev_ds.replace('-', '')
 
         ds_nodash = ds.replace('-', '')
-        ts_nodash = ts.replace('-', '').replace(':', '')
+        ts_nodash = self.execution_date.strftime('%Y%m%dT%H%M%S')
+        ts_nodash_with_tz = ts.replace('-', '').replace(':', '')
         yesterday_ds_nodash = yesterday_ds.replace('-', '')
         tomorrow_ds_nodash = tomorrow_ds.replace('-', '')
 
         ti_key_str = "{task.dag_id}__{task.task_id}__{ds_nodash}"
         ti_key_str = ti_key_str.format(**locals())
 
-        params = {}
-        run_id = ''
-        dag_run = None
-        if hasattr(task, 'dag'):
-            if task.dag.params:
-                params.update(task.dag.params)
-            dag_run = (
-                session.query(DagRun)
-                .filter_by(
-                    dag_id=task.dag.dag_id,
-                    execution_date=self.execution_date)
-                .first()
-            )
-            run_id = dag_run.run_id if dag_run else None
-            session.expunge_all()
-            session.commit()
-
         if task.params:
             params.update(task.params)
 
@@ -1866,10 +1735,13 @@ def __repr__(self):
             'dag': task.dag,
             'ds': ds,
             'next_ds': next_ds,
+            'next_ds_nodash': next_ds_nodash,
             'prev_ds': prev_ds,
+            'prev_ds_nodash': prev_ds_nodash,
             'ds_nodash': ds_nodash,
             'ts': ts,
             'ts_nodash': ts_nodash,
+            'ts_nodash_with_tz': ts_nodash_with_tz,
             'yesterday_ds': yesterday_ds,
             'yesterday_ds_nodash': yesterday_ds_nodash,
             'tomorrow_ds': tomorrow_ds,
@@ -1918,22 +1790,43 @@ def render_templates(self):
                 rendered_content = rt(attr, content, jinja_context)
                 setattr(task, attr, rendered_content)
 
-    def email_alert(self, exception, is_retry=False):
-        task = self.task
-        title = "Airflow alert: {self}".format(**locals())
-        exception = str(exception).replace('\n', '<br>')
+    def email_alert(self, exception):
+        exception_html = str(exception).replace('\n', '<br>')
+        jinja_context = self.get_template_context()
+        # This function is called after changing the state
+        # from State.RUNNING so need to subtract 1 from self.try_number.
+        jinja_context.update(dict(
+            exception=exception,
+            exception_html=exception_html,
+            try_number=self.try_number - 1,
+            max_tries=self.max_tries))
+
+        jinja_env = self.task.get_template_env()
+
+        default_subject = 'Airflow alert: {{ti}}'
         # For reporting purposes, we report based on 1-indexed,
         # not 0-indexed lists (i.e. Try 1 instead of
         # Try 0 for the first attempt).
-        body = (
-            "Try {try_number} out of {max_tries}<br>"
-            "Exception:<br>{exception}<br>"
-            "Log: <a href='{self.log_url}'>Link</a><br>"
-            "Host: {self.hostname}<br>"
-            "Log file: {self.log_filepath}<br>"
-            "Mark success: <a href='{self.mark_success_url}'>Link</a><br>"
-        ).format(try_number=self.try_number, max_tries=self.max_tries + 1, **locals())
-        send_email(task.email, title, body)
+        default_html_content = (
+            'Try {{try_number}} out of {{max_tries + 1}}<br>'
+            'Exception:<br>{{exception_html}}<br>'
+            'Log: <a href="{{ti.log_url}}">Link</a><br>'
+            'Host: {{ti.hostname}}<br>'
+            'Log file: {{ti.log_filepath}}<br>'
+            'Mark success: <a href="{{ti.mark_success_url}}">Link</a><br>'
+        )
+
+        def render(key, content):
+            if configuration.has_option('email', key):
+                path = configuration.get('email', key)
+                with open(path) as f:
+                    content = f.read()
+
+            return jinja_env.from_string(content).render(**jinja_context)
+
+        subject = render('subject_template', default_subject)
+        html_content = render('html_content_template', default_html_content)
+        send_email(self.task.email, subject, html_content)
 
     def set_duration(self):
         if self.end_date and self.start_date:
@@ -1950,7 +1843,7 @@ def xcom_push(
         Make an XCom available for tasks to pull.
 
         :param key: A key for the XCom
-        :type key: string
+        :type key: str
         :param value: A value for the XCom. The value is pickled and stored
             in the database.
         :type value: any pickleable object
@@ -1996,13 +1889,13 @@ def xcom_pull(
             available as a constant XCOM_RETURN_KEY. This key is automatically
             given to XComs returned by tasks (as opposed to being pushed
             manually). To remove the filter, pass key=None.
-        :type key: string
+        :type key: str
         :param task_ids: Only XComs from tasks with matching ids will be
             pulled. Can pass None to remove the filter.
-        :type task_ids: string or iterable of strings (representing task_ids)
+        :type task_ids: str or iterable of strings (representing task_ids)
         :param dag_id: If provided, only pulls XComs from this DAG.
             If None (default), the DAG of the calling task is used.
-        :type dag_id: string
+        :type dag_id: str
         :param include_prior_dates: If False, only XComs from the current
             execution_date are returned. If True, XComs from previous dates
             are returned as well.
@@ -2041,114 +1934,6 @@ def init_run_context(self, raw=False):
         self._set_context(self)
 
 
-class TaskFail(Base):
-    """
-    TaskFail tracks the failed run durations of each task instance.
-    """
-
-    __tablename__ = "task_fail"
-
-    task_id = Column(String(ID_LEN), primary_key=True)
-    dag_id = Column(String(ID_LEN), primary_key=True)
-    execution_date = Column(UtcDateTime, primary_key=True)
-    start_date = Column(UtcDateTime)
-    end_date = Column(UtcDateTime)
-    duration = Column(Float)
-
-    def __init__(self, task, execution_date, start_date, end_date):
-        self.dag_id = task.dag_id
-        self.task_id = task.task_id
-        self.execution_date = execution_date
-        self.start_date = start_date
-        self.end_date = end_date
-        if self.end_date and self.start_date:
-            self.duration = (self.end_date - self.start_date).total_seconds()
-        else:
-            self.duration = None
-
-
-class Log(Base):
-    """
-    Used to actively log events to the database
-    """
-
-    __tablename__ = "log"
-
-    id = Column(Integer, primary_key=True)
-    dttm = Column(UtcDateTime)
-    dag_id = Column(String(ID_LEN))
-    task_id = Column(String(ID_LEN))
-    event = Column(String(30))
-    execution_date = Column(UtcDateTime)
-    owner = Column(String(500))
-    extra = Column(Text)
-
-    def __init__(self, event, task_instance, owner=None, extra=None, **kwargs):
-        self.dttm = timezone.utcnow()
-        self.event = event
-        self.extra = extra
-
-        task_owner = None
-
-        if task_instance:
-            self.dag_id = task_instance.dag_id
-            self.task_id = task_instance.task_id
-            self.execution_date = task_instance.execution_date
-            task_owner = task_instance.task.owner
-
-        if 'task_id' in kwargs:
-            self.task_id = kwargs['task_id']
-        if 'dag_id' in kwargs:
-            self.dag_id = kwargs['dag_id']
-        if 'execution_date' in kwargs:
-            if kwargs['execution_date']:
-                self.execution_date = kwargs['execution_date']
-
-        self.owner = owner or task_owner
-
-
-class SkipMixin(LoggingMixin):
-    @provide_session
-    def skip(self, dag_run, execution_date, tasks, session=None):
-        """
-        Sets tasks instances to skipped from the same dag run.
-
-        :param dag_run: the DagRun for which to set the tasks to skipped
-        :param execution_date: execution_date
-        :param tasks: tasks to skip (not task_ids)
-        :param session: db session to use
-        """
-        if not tasks:
-            return
-
-        task_ids = [d.task_id for d in tasks]
-        now = timezone.utcnow()
-
-        if dag_run:
-            session.query(TaskInstance).filter(
-                TaskInstance.dag_id == dag_run.dag_id,
-                TaskInstance.execution_date == dag_run.execution_date,
-                TaskInstance.task_id.in_(task_ids)
-            ).update({TaskInstance.state: State.SKIPPED,
-                      TaskInstance.start_date: now,
-                      TaskInstance.end_date: now},
-                     synchronize_session=False)
-            session.commit()
-        else:
-            assert execution_date is not None, "Execution date is None and no dag run"
-
-            self.log.warning("No DAG RUN present this should not happen")
-            # this is defensive against dag runs that are not complete
-            for task in tasks:
-                ti = TaskInstance(task, execution_date=execution_date)
-                ti.state = State.SKIPPED
-                ti.start_date = now
-                ti.end_date = now
-                session.merge(ti)
-
-            session.commit()
-
-
 @functools.total_ordering
 class BaseOperator(LoggingMixin):
     """
@@ -2171,20 +1956,20 @@ class derived from this one results in the creation of a task object,
     be set by using the set_upstream and/or set_downstream methods.
 
     :param task_id: a unique, meaningful id for the task
-    :type task_id: string
+    :type task_id: str
     :param owner: the owner of the task, using the unix username is recommended
-    :type owner: string
+    :type owner: str
     :param retries: the number of retries that should be performed before
         failing the task
     :type retries: int
     :param retry_delay: delay between retries
-    :type retry_delay: timedelta
+    :type retry_delay: datetime.timedelta
     :param retry_exponential_backoff: allow progressive longer waits between
         retries by using exponential backoff algorithm on retry delay (delay
         will be converted into seconds)
     :type retry_exponential_backoff: bool
     :param max_retry_delay: maximum delay interval between retries
-    :type max_retry_delay: timedelta
+    :type max_retry_delay: datetime.timedelta
     :param start_date: The ``start_date`` for the task, determines
         the ``execution_date`` for the first task instance. The best practice
         is to have the start_date rounded
@@ -2201,9 +1986,9 @@ class derived from this one results in the creation of a task object,
         ``TimeSensor`` and ``TimeDeltaSensor``. We advise against using
         dynamic ``start_date`` and recommend using fixed ones. Read the
         FAQ entry about start_date for more information.
-    :type start_date: datetime
+    :type start_date: datetime.datetime
     :param end_date: if specified, the scheduler won't go beyond this date
-    :type end_date: datetime
+    :type end_date: datetime.datetime
     :param depends_on_past: when set to true, task instances will run
         sequentially while relying on the previous task's schedule to
         succeed. The task instance for the start_date is allowed to run.
@@ -2220,7 +2005,7 @@ class derived from this one results in the creation of a task object,
         does support targeting specific queues.
     :type queue: str
     :param dag: a reference to the dag the task is attached to (if any)
-    :type dag: DAG
+    :type dag: airflow.models.DAG
     :param priority_weight: priority weight of this task against other task.
         This allows the executor to trigger higher priority tasks before
         others when things get backed up.
@@ -2274,13 +2059,14 @@ class derived from this one results in the creation of a task object,
     :type on_failure_callback: callable
     :param on_retry_callback: much like the ``on_failure_callback`` except
         that it is executed when retries occur.
+    :type on_retry_callback: callable
     :param on_success_callback: much like the ``on_failure_callback`` except
         that it is executed when the task succeeds.
     :type on_success_callback: callable
     :param trigger_rule: defines the rule by which dependencies are applied
         for the task to get triggered. Options are:
         ``{ all_success | all_failed | all_done | one_success |
-        one_failed | dummy}``
+        one_failed | none_failed | none_skipped | dummy}``
         default is ``all_success``. Options can be set as string or
         using the constants defined in the static class
         ``airflow.utils.TriggerRule``
@@ -2296,25 +2082,37 @@ class derived from this one results in the creation of a task object,
     :param executor_config: Additional task-level configuration parameters that are
         interpreted by a specific executor. Parameters are namespaced by the name of
         executor.
-        ``example: to run this task in a specific docker container through
-        the KubernetesExecutor
-        MyOperator(...,
-            executor_config={
-            "KubernetesExecutor":
-                {"image": "myCustomDockerImage"}
-                }
-        )``
+
+        **Example**: to run this task in a specific docker container through
+        the KubernetesExecutor ::
+
+            MyOperator(...,
+                executor_config={
+                "KubernetesExecutor":
+                    {"image": "myCustomDockerImage"}
+                    }
+            )
+
     :type executor_config: dict
     """
 
     # For derived classes to define which fields will get jinjaified
-    template_fields = []
+    template_fields = []  # type: Iterable[str]
     # Defines which files extensions to look for in the templated fields
-    template_ext = []
+    template_ext = []  # type: Iterable[str]
     # Defines the color in the UI
     ui_color = '#fff'
     ui_fgcolor = '#000'
 
+    # base list which includes all the attrs that don't need deep copy.
+    _base_operator_shallow_copy_attrs = ('user_defined_macros',
+                                         'user_defined_filters',
+                                         'params',
+                                         '_log',)
+
+    # each operator should override this class attr for shallow copy attrs.
+    shallow_copy_attrs = ()  # type: Iterable[str]
+
     @apply_defaults
     def __init__(
             self,
@@ -2335,7 +2133,6 @@ def __init__(
             dag=None,
             params=None,
             default_args=None,
-            adhoc=False,
             priority_weight=1,
             weight_rule=WeightRule.DOWNSTREAM,
             queue=configuration.conf.get('celery', 'default_queue'),
@@ -2358,29 +2155,36 @@ def __init__(
         if args or kwargs:
             # TODO remove *args and **kwargs in Airflow 2.0
             warnings.warn(
-                'Invalid arguments were passed to {c}. Support for '
-                'passing such arguments will be dropped in Airflow 2.0. '
-                'Invalid arguments were:'
+                'Invalid arguments were passed to {c} (task_id: {t}). '
+                'Support for passing such arguments will be dropped in '
+                'Airflow 2.0. Invalid arguments were:'
                 '\n*args: {a}\n**kwargs: {k}'.format(
-                    c=self.__class__.__name__, a=args, k=kwargs),
-                category=PendingDeprecationWarning
+                    c=self.__class__.__name__, a=args, k=kwargs, t=task_id),
+                category=PendingDeprecationWarning,
+                stacklevel=3
             )
-
         validate_key(task_id)
         self.task_id = task_id
         self.owner = owner
         self.email = email
         self.email_on_retry = email_on_retry
         self.email_on_failure = email_on_failure
+
         self.start_date = start_date
         if start_date and not isinstance(start_date, datetime):
             self.log.warning("start_date for %s isn't datetime.datetime", self)
+        elif start_date:
+            self.start_date = timezone.convert_to_utc(start_date)
+
         self.end_date = end_date
+        if end_date:
+            self.end_date = timezone.convert_to_utc(end_date)
+
         if not TriggerRule.is_valid(trigger_rule):
             raise AirflowException(
                 "The trigger_rule must be one of {all_triggers},"
                 "'{d}.{t}'; received '{tr}'."
-                .format(all_triggers=TriggerRule.all_triggers,
+                .format(all_triggers=TriggerRule.all_triggers(),
                         d=dag.dag_id if dag else "", t=task_id, tr=trigger_rule))
 
         self.trigger_rule = trigger_rule
@@ -2413,7 +2217,6 @@ def __init__(
         self.retry_exponential_backoff = retry_exponential_backoff
         self.max_retry_delay = max_retry_delay
         self.params = params or {}  # Available in templates!
-        self.adhoc = adhoc
         self.priority_weight = priority_weight
         if not WeightRule.is_valid(weight_rule):
             raise AirflowException(
@@ -2473,7 +2276,6 @@ def __init__(
             'schedule_interval',
             'depends_on_past',
             'wait_for_downstream',
-            'adhoc',
             'priority_weight',
             'sla',
             'execution_timeout',
@@ -2483,10 +2285,10 @@ def __init__(
         }
 
     def __eq__(self, other):
-        return (
-            type(self) == type(other) and
-            all(self.__dict__.get(c, None) == other.__dict__.get(c, None)
-                for c in self._comps))
+        if (type(self) == type(other) and
+                self.task_id == other.task_id):
+            return all(self.__dict__.get(c, None) == other.__dict__.get(c, None) for c in self._comps)
+        return False
 
     def __ne__(self, other):
         return not self == other
@@ -2681,17 +2483,13 @@ def __deepcopy__(self, memo):
         result = cls.__new__(cls)
         memo[id(self)] = result
 
+        shallow_copy = cls.shallow_copy_attrs + cls._base_operator_shallow_copy_attrs
+
         for k, v in list(self.__dict__.items()):
-            if k not in ('user_defined_macros', 'user_defined_filters',
-                         'params', '_log'):
+            if k not in shallow_copy:
                 setattr(result, k, copy.deepcopy(v, memo))
-        result.params = self.params
-        if hasattr(self, 'user_defined_macros'):
-            result.user_defined_macros = self.user_defined_macros
-        if hasattr(self, 'user_defined_filters'):
-            result.user_defined_filters = self.user_defined_filters
-        if hasattr(self, '_log'):
-            result._log = self._log
+            else:
+                setattr(result, k, copy.copy(v))
         return result
 
     def __getstate__(self):
@@ -2709,25 +2507,19 @@ def render_template_from_field(self, attr, content, context, jinja_env):
         Renders a template from a field. If the field is a string, it will
         simply render the string and return the result. If it is a collection or
         nested set of collections, it will traverse the structure and render
-        all strings in it.
+        all elements in it. If the field has another type, it will return it as it is.
         """
         rt = self.render_template
         if isinstance(content, six.string_types):
             result = jinja_env.from_string(content).render(**context)
         elif isinstance(content, (list, tuple)):
             result = [rt(attr, e, context) for e in content]
-        elif isinstance(content, numbers.Number):
-            result = content
         elif isinstance(content, dict):
             result = {
                 k: rt("{}[{}]".format(attr, k), v, context)
                 for k, v in list(content.items())}
         else:
-            param_type = type(content)
-            msg = (
-                "Type '{param_type}' used for parameter '{attr}' is "
-                "not supported for templating").format(**locals())
-            raise AirflowException(msg)
+            result = content
         return result
 
     def render_template(self, attr, content, context):
@@ -2735,9 +2527,7 @@ def render_template(self, attr, content, context):
         Renders a template either from a file or directly in a field, and returns
         the rendered result.
         """
-        jinja_env = self.dag.get_template_env() \
-            if hasattr(self, 'dag') \
-            else jinja2.Environment(cache_size=0)
+        jinja_env = self.get_template_env()
 
         exts = self.__class__.template_ext
         if (
@@ -2747,6 +2537,11 @@ def render_template(self, attr, content, context):
         else:
             return self.render_template_from_field(attr, content, context, jinja_env)
 
+    def get_template_env(self):
+        return self.dag.get_template_env() \
+            if hasattr(self, 'dag') \
+            else jinja2.Environment(cache_size=0)
+
     def prepare_template(self):
         """
         Hook that is triggered after the templated fields get replaced
@@ -2760,14 +2555,24 @@ def resolve_template_files(self):
         # Getting the content of files for template_field / template_ext
         for attr in self.template_fields:
             content = getattr(self, attr)
-            if content is not None and \
-                    isinstance(content, six.string_types) and \
+            if content is None:
+                continue
+            elif isinstance(content, six.string_types) and \
                     any([content.endswith(ext) for ext in self.template_ext]):
-                env = self.dag.get_template_env()
+                env = self.get_template_env()
                 try:
                     setattr(self, attr, env.loader.get_source(env, content)[0])
                 except Exception as e:
                     self.log.exception(e)
+            elif isinstance(content, list):
+                env = self.dag.get_template_env()
+                for i in range(len(content)):
+                    if isinstance(content[i], six.string_types) and \
+                            any([content[i].endswith(ext) for ext in self.template_ext]):
+                        try:
+                            content[i] = env.loader.get_source(env, content[i])[0]
+                        except Exception as e:
+                            self.log.exception(e)
         self.prepare_template()
 
     @property
@@ -2924,7 +2729,7 @@ def task_type(self):
 
     def add_only_new(self, item_set, item):
         if item in item_set:
-            raise AirflowException(
+            self.log.warning(
                 'Dependency {self}, {item} already registered'
                 ''.format(**locals()))
         else:
@@ -3048,15 +2853,83 @@ class DagModel(Base):
     fileloc = Column(String(2000))
     # String representing the owners
     owners = Column(String(2000))
+    # Description of the dag
+    description = Column(Text)
+    # Default view of the inside the webserver
+    default_view = Column(String(25))
+    # Schedule interval
+    schedule_interval = Column(Interval)
 
     def __repr__(self):
         return "<DAG: {self.dag_id}>".format(self=self)
 
+    @property
+    def timezone(self):
+        return settings.TIMEZONE
+
+    @staticmethod
+    @provide_session
+    def get_dagmodel(dag_id, session=None):
+        return session.query(DagModel).filter(DagModel.dag_id == dag_id).first()
+
     @classmethod
     @provide_session
     def get_current(cls, dag_id, session=None):
         return session.query(cls).filter(cls.dag_id == dag_id).first()
 
+    def get_default_view(self):
+        if self.default_view is None:
+            return configuration.conf.get('webserver', 'dag_default_view').lower()
+        else:
+            return self.default_view
+
+    @provide_session
+    def get_last_dagrun(self, session=None, include_externally_triggered=False):
+        return get_last_dagrun(self.dag_id, session=session,
+                               include_externally_triggered=include_externally_triggered)
+
+    @property
+    def safe_dag_id(self):
+        return self.dag_id.replace('.', '__dot__')
+
+    def get_dag(self):
+        return DagBag(dag_folder=self.fileloc).get_dag(self.dag_id)
+
+    @provide_session
+    def create_dagrun(self,
+                      run_id,
+                      state,
+                      execution_date,
+                      start_date=None,
+                      external_trigger=False,
+                      conf=None,
+                      session=None):
+        """
+        Creates a dag run from this dag including the tasks associated with this dag.
+        Returns the dag run.
+
+        :param run_id: defines the the run id for this dag run
+        :type run_id: str
+        :param execution_date: the execution date of this dag run
+        :type execution_date: datetime.datetime
+        :param state: the state of the dag run
+        :type state: airflow.utils.state.State
+        :param start_date: the date this dag run should be evaluated
+        :type start_date: datetime.datetime
+        :param external_trigger: whether this dag run is externally triggered
+        :type external_trigger: bool
+        :param session: database session
+        :type session: sqlalchemy.orm.session.Session
+        """
+
+        return self.get_dag().create_dagrun(run_id=run_id,
+                                            state=state,
+                                            execution_date=execution_date,
+                                            start_date=start_date,
+                                            external_trigger=external_trigger,
+                                            conf=conf,
+                                            session=session)
+
 
 @functools.total_ordering
 class DAG(BaseDag, LoggingMixin):
@@ -3072,9 +2945,9 @@ class DAG(BaseDag, LoggingMixin):
     added once to a DAG.
 
     :param dag_id: The id of the DAG
-    :type dag_id: string
+    :type dag_id: str
     :param description: The description for the DAG to e.g. be shown on the webserver
-    :type description: string
+    :type description: str
     :param schedule_interval: Defines how often that DAG runs, this
         timedelta object gets added to your latest task instance's
         execution_date to figure out the next schedule
@@ -3091,7 +2964,7 @@ class DAG(BaseDag, LoggingMixin):
         defines where jinja will look for your templates. Order matters.
         Note that jinja/airflow includes the path of your DAG file by
         default
-    :type template_searchpath: string or list of stings
+    :type template_searchpath: str or list[str]
     :param user_defined_macros: a dictionary of macros that will be exposed
         in your jinja templates. For example, passing ``dict(foo='bar')``
         to this argument allows you to ``{{ foo }}`` in all jinja
@@ -3130,9 +3003,9 @@ class DAG(BaseDag, LoggingMixin):
     :type sla_miss_callback: types.FunctionType
     :param default_view: Specify DAG default view (tree, graph, duration,
                                                    gantt, landing_times)
-    :type default_view: string
+    :type default_view: str
     :param orientation: Specify DAG orientation in graph view (LR, TB, RL, BT)
-    :type orientation: string
+    :type orientation: str
     :param catchup: Perform scheduler catchup (or only run latest)? Defaults to True
     :type catchup: bool
     :param on_failure_callback: A function to be called when a DagRun of this dag fails.
@@ -3158,10 +3031,11 @@ def __init__(
                 'core', 'max_active_runs_per_dag'),
             dagrun_timeout=None,
             sla_miss_callback=None,
-            default_view=configuration.conf.get('webserver', 'dag_default_view').lower(),
+            default_view=None,
             orientation=configuration.conf.get('webserver', 'dag_orientation'),
             catchup=configuration.conf.getboolean('scheduler', 'catchup_by_default'),
             on_success_callback=None, on_failure_callback=None,
+            doc_md=None,
             params=None):
 
         self.user_defined_macros = user_defined_macros
@@ -3196,7 +3070,8 @@ def __init__(
                     timezone.parse(self.default_args['start_date'])
                 )
             self.timezone = self.default_args['start_date'].tzinfo
-        else:
+
+        if not hasattr(self, 'timezone') or not self.timezone:
             self.timezone = settings.TIMEZONE
 
         self.start_date = timezone.convert_to_utc(start_date)
@@ -3213,7 +3088,7 @@ def __init__(
             )
 
         self.schedule_interval = schedule_interval
-        if schedule_interval in cron_presets:
+        if isinstance(schedule_interval, Hashable) and schedule_interval in cron_presets:
             self._schedule_interval = cron_presets.get(schedule_interval)
         elif schedule_interval == '@once':
             self._schedule_interval = None
@@ -3228,7 +3103,7 @@ def __init__(
         self.max_active_runs = max_active_runs
         self.dagrun_timeout = dagrun_timeout
         self.sla_miss_callback = sla_miss_callback
-        self.default_view = default_view
+        self._default_view = default_view
         self.orientation = orientation
         self.catchup = catchup
         self.is_subdag = False  # DagBag.bag_dag() will set this to True if appropriate
@@ -3236,6 +3111,9 @@ def __init__(
         self.partial = False
         self.on_success_callback = on_success_callback
         self.on_failure_callback = on_failure_callback
+        self.doc_md = doc_md
+
+        self._old_context_manager_dags = []
 
         self._comps = {
             'dag_id',
@@ -3252,12 +3130,13 @@ def __repr__(self):
         return "<DAG: {self.dag_id}>".format(self=self)
 
     def __eq__(self, other):
-        return (
-            type(self) == type(other) and
+        if (type(self) == type(other) and
+                self.dag_id == other.dag_id):
+
             # Use getattr() instead of __dict__ as __dict__ doesn't return
             # correct values for properties.
-            all(getattr(self, c, None) == getattr(other, c, None)
-                for c in self._comps))
+            return all(getattr(self, c, None) == getattr(other, c, None) for c in self._comps)
+        return False
 
     def __ne__(self, other):
         return not self == other
@@ -3284,16 +3163,23 @@ def __hash__(self):
 
     def __enter__(self):
         global _CONTEXT_MANAGER_DAG
-        self._old_context_manager_dag = _CONTEXT_MANAGER_DAG
+        self._old_context_manager_dags.append(_CONTEXT_MANAGER_DAG)
         _CONTEXT_MANAGER_DAG = self
         return self
 
     def __exit__(self, _type, _value, _tb):
         global _CONTEXT_MANAGER_DAG
-        _CONTEXT_MANAGER_DAG = self._old_context_manager_dag
+        _CONTEXT_MANAGER_DAG = self._old_context_manager_dags.pop()
 
     # /Context Manager ----------------------------------------------
 
+    def get_default_view(self):
+        """This is only there for backward compatible jinja2 templates"""
+        if self._default_view is None:
+            return configuration.conf.get('webserver', 'dag_default_view').lower()
+        else:
+            return self._default_view
+
     def date_range(self, start_date, num=None, end_date=timezone.utcnow()):
         if num:
             end_date = None
@@ -3301,34 +3187,77 @@ def date_range(self, start_date, num=None, end_date=timezone.utcnow()):
             start_date=start_date, end_date=end_date,
             num=num, delta=self._schedule_interval)
 
+    def is_fixed_time_schedule(self):
+        """
+        Figures out if the DAG schedule has a fixed time (e.g. 3 AM).
+
+        :return: True if the schedule has a fixed time, False if not.
+        """
+        now = datetime.now()
+        cron = croniter(self._schedule_interval, now)
+
+        start = cron.get_next(datetime)
+        cron_next = cron.get_next(datetime)
+
+        if cron_next.minute == start.minute and cron_next.hour == start.hour:
+            return True
+
+        return False
+
     def following_schedule(self, dttm):
         """
-        Calculates the following schedule for this dag in local time
+        Calculates the following schedule for this dag in UTC.
 
         :param dttm: utc datetime
         :return: utc datetime
         """
         if isinstance(self._schedule_interval, six.string_types):
-            dttm = timezone.make_naive(dttm, self.timezone)
-            cron = croniter(self._schedule_interval, dttm)
-            following = timezone.make_aware(cron.get_next(datetime), self.timezone)
+            # we don't want to rely on the transitions created by
+            # croniter as they are not always correct
+            dttm = pendulum.instance(dttm)
+            naive = timezone.make_naive(dttm, self.timezone)
+            cron = croniter(self._schedule_interval, naive)
+
+            # We assume that DST transitions happen on the minute/hour
+            if not self.is_fixed_time_schedule():
+                # relative offset (eg. every 5 minutes)
+                delta = cron.get_next(datetime) - naive
+                following = dttm.in_timezone(self.timezone).add_timedelta(delta)
+            else:
+                # absolute (e.g. 3 AM)
+                naive = cron.get_next(datetime)
+                tz = pendulum.timezone(self.timezone.name)
+                following = timezone.make_aware(naive, tz)
             return timezone.convert_to_utc(following)
-        elif isinstance(self._schedule_interval, timedelta):
+        elif self._schedule_interval is not None:
             return dttm + self._schedule_interval
 
     def previous_schedule(self, dttm):
         """
-        Calculates the previous schedule for this dag in local time
+        Calculates the previous schedule for this dag in UTC
 
         :param dttm: utc datetime
         :return: utc datetime
         """
         if isinstance(self._schedule_interval, six.string_types):
-            dttm = timezone.make_naive(dttm, self.timezone)
-            cron = croniter(self._schedule_interval, dttm)
-            prev = timezone.make_aware(cron.get_prev(datetime), self.timezone)
-            return timezone.convert_to_utc(prev)
-        elif isinstance(self._schedule_interval, timedelta):
+            # we don't want to rely on the transitions created by
+            # croniter as they are not always correct
+            dttm = pendulum.instance(dttm)
+            naive = timezone.make_naive(dttm, self.timezone)
+            cron = croniter(self._schedule_interval, naive)
+
+            # We assume that DST transitions happen on the minute/hour
+            if not self.is_fixed_time_schedule():
+                # relative offset (eg. every 5 minutes)
+                delta = naive - cron.get_prev(datetime)
+                previous = dttm.in_timezone(self.timezone).subtract_timedelta(delta)
+            else:
+                # absolute (e.g. 3 AM)
+                naive = cron.get_prev(datetime)
+                tz = pendulum.timezone(self.timezone.name)
+                previous = timezone.make_aware(naive, tz)
+            return timezone.convert_to_utc(previous)
+        elif self._schedule_interval is not None:
             return dttm - self._schedule_interval
 
     def get_run_dates(self, start_date, end_date=None):
@@ -3379,23 +3308,8 @@ def normalize_schedule(self, dttm):
 
     @provide_session
     def get_last_dagrun(self, session=None, include_externally_triggered=False):
-        """
-        Returns the last dag run for this dag, None if there was none.
-        Last dag run can be any type of run eg. scheduled or backfilled.
-        Overridden DagRuns are ignored
-        """
-        DR = DagRun
-        qry = session.query(DR).filter(
-            DR.dag_id == self.dag_id,
-        )
-        if not include_externally_triggered:
-            qry = qry.filter(DR.external_trigger.__eq__(False))
-
-        qry = qry.order_by(DR.execution_date.desc())
-
-        last = qry.first()
-
-        return last
+        return get_last_dagrun(self.dag_id, session=session,
+                               include_externally_triggered=include_externally_triggered)
 
     @property
     def dag_id(self):
@@ -3446,14 +3360,6 @@ def tasks(self, val):
     def task_ids(self):
         return list(self.task_dict.keys())
 
-    @property
-    def active_task_ids(self):
-        return list(k for k, v in self.task_dict.items() if not v.adhoc)
-
-    @property
-    def active_tasks(self):
-        return [t for t in self.tasks if not t.adhoc]
-
     @property
     def filepath(self):
         """
@@ -3474,13 +3380,8 @@ def folder(self):
     def owner(self):
         return ", ".join(list(set([t.owner for t in self.tasks])))
 
-    @property
     @provide_session
-    def concurrency_reached(self, session=None):
-        """
-        Returns a boolean indicating whether the concurrency limit for this DAG
-        has been reached
-        """
+    def _get_concurrency_reached(self, session=None):
         TI = TaskInstance
         qry = session.query(func.count(TI.task_id)).filter(
             TI.dag_id == self.dag_id,
@@ -3489,15 +3390,26 @@ def concurrency_reached(self, session=None):
         return qry.scalar() >= self.concurrency
 
     @property
-    @provide_session
-    def is_paused(self, session=None):
+    def concurrency_reached(self):
         """
-        Returns a boolean indicating whether this DAG is paused
+        Returns a boolean indicating whether the concurrency limit for this DAG
+        has been reached
         """
+        return self._get_concurrency_reached()
+
+    @provide_session
+    def _get_is_paused(self, session=None):
         qry = session.query(DagModel).filter(
             DagModel.dag_id == self.dag_id)
         return qry.value('is_paused')
 
+    @property
+    def is_paused(self):
+        """
+        Returns a boolean indicating whether this DAG is paused
+        """
+        return self._get_is_paused()
+
     @provide_session
     def handle_callback(self, dagrun, success=True, reason=None, session=None):
         """
@@ -3505,8 +3417,10 @@ def handle_callback(self, dagrun, success=True, reason=None, session=None):
         on_failure_callback or on_success_callback. This method gets the context of a
         single TaskInstance part of this DagRun and passes that to the callable along
         with a 'reason', primarily to differentiate DagRun failures.
-        .. note::
-            The logs end up in $AIRFLOW_HOME/logs/scheduler/latest/PROJECT/DAG_FILE.py.log
+
+        .. note: The logs end up in
+            ``$AIRFLOW_HOME/logs/scheduler/latest/PROJECT/DAG_FILE.py.log``
+
         :param dagrun: DagRun object
         :param success: Flag to specify if failure or success callback should be called
         :param reason: Completion reason
@@ -3577,24 +3491,25 @@ def get_dagrun(self, execution_date, session=None):
 
         return dagrun
 
-    @property
     @provide_session
-    def latest_execution_date(self, session=None):
+    def _get_latest_execution_date(self, session=None):
+        return session.query(func.max(DagRun.execution_date)).filter(
+            DagRun.dag_id == self.dag_id
+        ).scalar()
+
+    @property
+    def latest_execution_date(self):
         """
         Returns the latest date for which at least one dag run exists
         """
-        execution_date = session.query(func.max(DagRun.execution_date)).filter(
-            DagRun.dag_id == self.dag_id
-        ).scalar()
-        return execution_date
+        return self._get_latest_execution_date()
 
     @property
     def subdags(self):
         """
         Returns a list of the subdag objects associated to this DAG
         """
-        # Check SubDag for class but don't check class directly, see
-        # https://github.com/airbnb/airflow/issues/1168
+        # Check SubDag for class but don't check class directly
         from airflow.operators.subdag_operator import SubDagOperator
         subdag_lst = []
         for task in self.tasks:
@@ -3671,8 +3586,8 @@ def topological_sort(self):
         :return: list of tasks in topological order
         """
 
-        # copy the the tasks so we leave it unmodified
-        graph_unsorted = self.tasks[:]
+        # convert into an OrderedDict to speedup lookup while keeping order the same
+        graph_unsorted = OrderedDict((task.task_id, task) for task in self.tasks)
 
         graph_sorted = []
 
@@ -3695,14 +3610,14 @@ def topological_sort(self):
             # not, we need to bail out as the graph therefore can't be
             # sorted.
             acyclic = False
-            for node in list(graph_unsorted):
+            for node in list(graph_unsorted.values()):
                 for edge in node.upstream_list:
-                    if edge in graph_unsorted:
+                    if edge.task_id in graph_unsorted:
                         break
                 # no edges in upstream tasks
                 else:
                     acyclic = True
-                    graph_unsorted.remove(node)
+                    del graph_unsorted[node.task_id]
                     graph_sorted.append(node)
 
             if not acyclic:
@@ -3730,7 +3645,6 @@ def set_dag_runs_state(
         for dr in drs:
             dr.state = state
             dirty_ids.append(dr.dag_id)
-        DagStat.update(dirty_ids, session=session)
 
     @provide_session
     def clear(
@@ -3739,9 +3653,11 @@ def clear(
             only_running=False,
             confirm_prompt=False,
             include_subdags=True,
+            include_parentdag=True,
             reset_dag_runs=True,
             dry_run=False,
             session=None,
+            get_tis=False,
     ):
         """
         Clears a set of task instances associated with the current dag for
@@ -3762,15 +3678,39 @@ def clear(
             tis = session.query(TI).filter(TI.dag_id == self.dag_id)
             tis = tis.filter(TI.task_id.in_(self.task_ids))
 
+        if include_parentdag and self.is_subdag:
+
+            p_dag = self.parent_dag.sub_dag(
+                task_regex=self.dag_id.split('.')[1],
+                include_upstream=False,
+                include_downstream=True)
+
+            tis = tis.union(p_dag.clear(
+                start_date=start_date, end_date=end_date,
+                only_failed=only_failed,
+                only_running=only_running,
+                confirm_prompt=confirm_prompt,
+                include_subdags=include_subdags,
+                include_parentdag=False,
+                reset_dag_runs=reset_dag_runs,
+                get_tis=True,
+                session=session,
+            ))
+
         if start_date:
             tis = tis.filter(TI.execution_date >= start_date)
         if end_date:
             tis = tis.filter(TI.execution_date <= end_date)
         if only_failed:
-            tis = tis.filter(TI.state == State.FAILED)
+            tis = tis.filter(or_(
+                TI.state == State.FAILED,
+                TI.state == State.UPSTREAM_FAILED))
         if only_running:
             tis = tis.filter(TI.state == State.RUNNING)
 
+        if get_tis:
+            return tis
+
         if dry_run:
             tis = tis.all()
             session.expunge_all()
@@ -3814,6 +3754,7 @@ def clear_dags(
             only_running=False,
             confirm_prompt=False,
             include_subdags=True,
+            include_parentdag=False,
             reset_dag_runs=True,
             dry_run=False,
     ):
@@ -3826,6 +3767,7 @@ def clear_dags(
                 only_running=only_running,
                 confirm_prompt=False,
                 include_subdags=include_subdags,
+                include_parentdag=include_parentdag,
                 reset_dag_runs=reset_dag_runs,
                 dry_run=True)
             all_tis.extend(tis)
@@ -4012,16 +3954,6 @@ def add_tasks(self, tasks):
         for task in tasks:
             self.add_task(task)
 
-    @provide_session
-    def db_merge(self, session=None):
-        BO = BaseOperator
-        tasks = session.query(BO).filter(BO.dag_id == self.dag_id).all()
-        for t in tasks:
-            session.delete(t)
-        session.commit()
-        session.merge(self)
-        session.commit()
-
     def run(
             self,
             start_date=None,
@@ -4037,20 +3969,21 @@ def run(
             verbose=False,
             conf=None,
             rerun_failed_tasks=False,
+            run_backwards=False,
     ):
         """
         Runs the DAG.
 
         :param start_date: the start date of the range to run
-        :type start_date: datetime
+        :type start_date: datetime.datetime
         :param end_date: the end date of the range to run
-        :type end_date: datetime
+        :type end_date: datetime.datetime
         :param mark_success: True to mark jobs as succeeded without running them
         :type mark_success: bool
         :param local: True to run the tasks using the LocalExecutor
         :type local: bool
         :param executor: The executor instance to run the tasks
-        :type executor: BaseExecutor
+        :type executor: airflow.executor.BaseExecutor
         :param donot_pickle: True to avoid pickling DAG object and send to workers
         :type donot_pickle: bool
         :param ignore_task_deps: True to skip upstream tasks
@@ -4059,14 +3992,19 @@ def run(
             dependencies for the first set of tasks only
         :type ignore_first_depends_on_past: bool
         :param pool: Resource pool to use
-        :type pool: string
+        :type pool: str
         :param delay_on_limit_secs: Time in seconds to wait before next attempt to run
             dag run when max_active_runs limit has been reached
         :type delay_on_limit_secs: float
         :param verbose: Make logging output more verbose
-        :type verbose: boolean
+        :type verbose: bool
         :param conf: user defined dictionary passed from CLI
         :type conf: dict
+        :param rerun_failed_tasks:
+        :type: bool
+        :param run_backwards:
+        :type: bool
+
         """
         from airflow.jobs import BackfillJob
         if not executor and local:
@@ -4087,6 +4025,7 @@ def run(
             verbose=verbose,
             conf=conf,
             rerun_failed_tasks=rerun_failed_tasks,
+            run_backwards=run_backwards,
         )
         job.run()
 
@@ -4113,17 +4052,17 @@ def create_dagrun(self,
         Returns the dag run.
 
         :param run_id: defines the the run id for this dag run
-        :type run_id: string
+        :type run_id: str
         :param execution_date: the execution date of this dag run
-        :type execution_date: datetime
+        :type execution_date: datetime.datetime
         :param state: the state of the dag run
-        :type state: State
+        :type state: airflow.utils.state.State
         :param start_date: the date this dag run should be evaluated
         :type start_date: datetime
         :param external_trigger: whether this dag run is externally triggered
         :type external_trigger: bool
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         """
         run = DagRun(
             dag_id=self.dag_id,
@@ -4136,8 +4075,6 @@ def create_dagrun(self,
         )
         session.add(run)
 
-        DagStat.set_dirty(dag_id=self.dag_id, session=session)
-
         session.commit()
 
         run.dag = self
@@ -4158,7 +4095,7 @@ def sync_to_db(self, owner=None, sync_time=None, session=None):
         SubDagOperator.
 
         :param dag: the DAG object to save to the DB
-        :type dag: DAG
+        :type dag: airflow.models.DAG
         :param sync_time: The time that the DAG should be marked as sync'ed
         :type sync_time: datetime
         :return: None
@@ -4179,6 +4116,9 @@ def sync_to_db(self, owner=None, sync_time=None, session=None):
         orm_dag.owners = owner
         orm_dag.is_active = True
         orm_dag.last_scheduler_run = sync_time
+        orm_dag.default_view = self._default_view
+        orm_dag.description = self.description
+        orm_dag.schedule_interval = self.schedule_interval
         session.merge(orm_dag)
         session.commit()
 
@@ -4203,6 +4143,7 @@ def deactivate_unknown_dags(active_dag_ids, session=None):
                 DagModel).filter(~DagModel.dag_id.in_(active_dag_ids)).all():
             dag.is_active = False
             session.merge(dag)
+        session.commit()
 
     @staticmethod
     @provide_session
@@ -4351,6 +4292,7 @@ def __repr__(self):
 
 class Variable(Base, LoggingMixin):
     __tablename__ = "variable"
+    __NO_DEFAULT_SENTINEL = object()
 
     id = Column(Integer, primary_key=True)
     key = Column(String(ID_LEN), unique=True)
@@ -4366,32 +4308,23 @@ def get_val(self):
         if self._val and self.is_encrypted:
             try:
                 fernet = get_fernet()
-            except Exception:
-                log.error("Can't decrypt _val for key={}, FERNET_KEY "
-                          "configuration missing".format(self.key))
-                return None
-            try:
                 return fernet.decrypt(bytes(self._val, 'utf-8')).decode()
-            except cryptography.fernet.InvalidToken:
+            except InvalidFernetToken:
                 log.error("Can't decrypt _val for key={}, invalid token "
                           "or value".format(self.key))
                 return None
+            except Exception:
+                log.error("Can't decrypt _val for key={}, FERNET_KEY "
+                          "configuration missing".format(self.key))
+                return None
         else:
             return self._val
 
     def set_val(self, value):
         if value:
-            try:
-                fernet = get_fernet()
-                self._val = fernet.encrypt(bytes(value, 'utf-8')).decode()
-                self.is_encrypted = True
-            except AirflowException:
-                self.log.exception(
-                    "Failed to load fernet while encrypting value, "
-                    "using non-encrypted value."
-                )
-                self._val = value
-                self.is_encrypted = False
+            fernet = get_fernet()
+            self._val = fernet.encrypt(bytes(value, 'utf-8')).decode()
+            self.is_encrypted = fernet.is_encrypted
 
     @declared_attr
     def val(cls):
@@ -4405,18 +4338,17 @@ def setdefault(cls, key, default, deserialize_json=False):
         for a key, and if it isn't there, stores the default value and returns it.
 
         :param key: Dict key for this Variable
-        :type key: String
+        :type key: str
         :param default: Default value to set and return if the variable
-        isn't already in the DB
+            isn't already in the DB
         :type default: Mixed
         :param deserialize_json: Store this as a JSON encoded value in the DB
-         and un-encode it when retrieving a value
+            and un-encode it when retrieving a value
         :return: Mixed
         """
-        default_sentinel = object()
-        obj = Variable.get(key, default_var=default_sentinel,
+        obj = Variable.get(key, default_var=None,
                            deserialize_json=deserialize_json)
-        if obj is default_sentinel:
+        if obj is None:
             if default is not None:
                 Variable.set(key, default, serialize_json=deserialize_json)
                 return default
@@ -4427,10 +4359,10 @@ def setdefault(cls, key, default, deserialize_json=False):
 
     @classmethod
     @provide_session
-    def get(cls, key, default_var=None, deserialize_json=False, session=None):
+    def get(cls, key, default_var=__NO_DEFAULT_SENTINEL, deserialize_json=False, session=None):
         obj = session.query(cls).filter(cls.key == key).first()
         if obj is None:
-            if default_var is not None:
+            if default_var is not cls.__NO_DEFAULT_SENTINEL:
                 return default_var
             else:
                 raise KeyError('Variable {} does not exist'.format(key))
@@ -4453,6 +4385,11 @@ def set(cls, key, value, serialize_json=False, session=None):
         session.add(Variable(key=key, val=stored_value))
         session.flush()
 
+    def rotate_fernet_key(self):
+        fernet = get_fernet()
+        if self._val and self.is_encrypted:
+            self._val = fernet.rotate(self._val.encode('utf-8')).decode()
+
 
 class XCom(Base, LoggingMixin):
     """
@@ -4464,7 +4401,7 @@ class XCom(Base, LoggingMixin):
     key = Column(String(512))
     value = Column(LargeBinary)
     timestamp = Column(
-        DateTime, default=timezone.utcnow, nullable=False)
+        UtcDateTime, default=timezone.utcnow, nullable=False)
     execution_date = Column(UtcDateTime, nullable=False)
 
     # source information
@@ -4512,7 +4449,8 @@ def set(
         """
         Store an XCom value.
         TODO: "pickling" has been deprecated and JSON is preferred.
-              "pickling" will be removed in Airflow 2.0.
+        "pickling" will be removed in Airflow 2.0.
+
         :return: None
         """
         session.expunge_all()
@@ -4562,7 +4500,8 @@ def get_one(cls,
         """
         Retrieve an XCom value, optionally meeting certain criteria.
         TODO: "pickling" has been deprecated and JSON is preferred.
-              "pickling" will be removed in Airflow 2.0.
+        "pickling" will be removed in Airflow 2.0.
+
         :return: XCom value
         """
         filters = []
@@ -4610,7 +4549,7 @@ def get_many(cls,
         """
         Retrieve an XCom value, optionally meeting certain criteria
         TODO: "pickling" has been deprecated and JSON is preferred.
-              "pickling" will be removed in Airflow 2.0.
+        "pickling" will be removed in Airflow 2.0.
         """
         filters = []
         if key:
@@ -4645,122 +4584,6 @@ def delete(cls, xcoms, session=None):
         session.commit()
 
 
-class DagStat(Base):
-    __tablename__ = "dag_stats"
-
-    dag_id = Column(String(ID_LEN), primary_key=True)
-    state = Column(String(50), primary_key=True)
-    count = Column(Integer, default=0)
-    dirty = Column(Boolean, default=False)
-
-    def __init__(self, dag_id, state, count=0, dirty=False):
-        self.dag_id = dag_id
-        self.state = state
-        self.count = count
-        self.dirty = dirty
-
-    @staticmethod
-    @provide_session
-    def set_dirty(dag_id, session=None):
-        """
-        :param dag_id: the dag_id to mark dirty
-        :param session: database session
-        :return:
-        """
-        DagStat.create(dag_id=dag_id, session=session)
-
-        try:
-            stats = session.query(DagStat).filter(
-                DagStat.dag_id == dag_id
-            ).with_for_update().all()
-
-            for stat in stats:
-                stat.dirty = True
-            session.commit()
-        except Exception as e:
-            session.rollback()
-            log = LoggingMixin().log
-            log.warning("Could not update dag stats for %s", dag_id)
-            log.exception(e)
-
-    @staticmethod
-    @provide_session
-    def update(dag_ids=None, dirty_only=True, session=None):
-        """
-        Updates the stats for dirty/out-of-sync dags
-
-        :param dag_ids: dag_ids to be updated
-        :type dag_ids: list
-        :param dirty_only: only updated for marked dirty, defaults to True
-        :type dirty_only: bool
-        :param session: db session to use
-        :type session: Session
-        """
-        try:
-            qry = session.query(DagStat)
-            if dag_ids:
-                qry = qry.filter(DagStat.dag_id.in_(set(dag_ids)))
-            if dirty_only:
-                qry = qry.filter(DagStat.dirty == True) # noqa
-
-            qry = qry.with_for_update().all()
-
-            ids = set([dag_stat.dag_id for dag_stat in qry])
-
-            # avoid querying with an empty IN clause
-            if len(ids) == 0:
-                session.commit()
-                return
-
-            dagstat_states = set(itertools.product(ids, State.dag_states))
-            qry = (
-                session.query(DagRun.dag_id, DagRun.state, func.count('*'))
-                .filter(DagRun.dag_id.in_(ids))
-                .group_by(DagRun.dag_id, DagRun.state)
-            )
-
-            counts = {(dag_id, state): count for dag_id, state, count in qry}
-            for dag_id, state in dagstat_states:
-                count = 0
-                if (dag_id, state) in counts:
-                    count = counts[(dag_id, state)]
-
-                session.merge(
-                    DagStat(dag_id=dag_id, state=state, count=count, dirty=False)
-                )
-
-            session.commit()
-        except Exception as e:
-            session.rollback()
-            log = LoggingMixin().log
-            log.warning("Could not update dag stat table")
-            log.exception(e)
-
-    @staticmethod
-    @provide_session
-    def create(dag_id, session=None):
-        """
-        Creates the missing states the stats table for the dag specified
-
-        :param dag_id: dag id of the dag to create stats for
-        :param session: database session
-        :return:
-        """
-        # unfortunately sqlalchemy does not know upsert
-        qry = session.query(DagStat).filter(DagStat.dag_id == dag_id).all()
-        states = [dag_stat.state for dag_stat in qry]
-        for state in State.dag_states:
-            if state not in states:
-                try:
-                    session.merge(DagStat(dag_id=dag_id, state=state))
-                    session.commit()
-                except Exception as e:
-                    session.rollback()
-                    log = LoggingMixin().log
-                    log.warning("Could not create stat record")
-                    log.exception(e)
-
-
 class DagRun(Base, LoggingMixin):
     """
     DagRun describes an instance of a Dag. It can be created
@@ -4784,7 +4607,9 @@ class DagRun(Base, LoggingMixin):
     dag = None
 
     __table_args__ = (
-        Index('dr_run_id', dag_id, run_id, unique=True),
+        Index('dag_id_state', dag_id, _state),
+        UniqueConstraint('dag_id', 'execution_date'),
+        UniqueConstraint('dag_id', 'run_id'),
     )
 
     def __repr__(self):
@@ -4803,12 +4628,7 @@ def get_state(self):
     def set_state(self, state):
         if self._state != state:
             self._state = state
-            if self.dag_id is not None:
-                # FIXME: Due to the scoped_session factor we we don't get a clean
-                # session here, so something really weird goes on:
-                # if you try to close the session dag runs will end up detached
-                session = settings.Session()
-                DagStat.set_dirty(self.dag_id, session=session)
+            self.end_date = timezone.utcnow() if self._state in State.finished() else None
 
     @declared_attr
     def state(self):
@@ -4847,20 +4667,20 @@ def find(dag_id=None, run_id=None, execution_date=None,
         Returns a set of dag runs for the given search criteria.
 
         :param dag_id: the dag_id to find dag runs for
-        :type dag_id: integer, list
+        :type dag_id: int, list
         :param run_id: defines the the run id for this dag run
-        :type run_id: string
+        :type run_id: str
         :param execution_date: the execution date
-        :type execution_date: datetime
+        :type execution_date: datetime.datetime
         :param state: the state of the dag run
-        :type state: State
+        :type state: airflow.utils.state.State
         :param external_trigger: whether this dag run is externally triggered
         :type external_trigger: bool
         :param no_backfills: return no backfills (True), return all (False).
-        Defaults to False
+            Defaults to False
         :type no_backfills: bool
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         """
         DR = DagRun
 
@@ -4977,7 +4797,6 @@ def update_state(self, session=None):
         dag = self.get_dag()
 
         tis = self.get_task_instances(session=session)
-
         self.log.debug("Updating state for %s considering %s task(s)", self, len(tis))
 
         for ti in list(tis):
@@ -5003,12 +4822,13 @@ def update_state(self, session=None):
             no_dependencies_met = True
             for ut in unfinished_tasks:
                 # We need to flag upstream and check for changes because upstream
-                # failures can result in deadlock false positives
+                # failures/re-schedules can result in deadlock false positives
                 old_state = ut.state
                 deps_met = ut.are_dependencies_met(
                     dep_context=DepContext(
                         flag_upstream_failed=True,
-                        ignore_in_retry_period=True),
+                        ignore_in_retry_period=True,
+                        ignore_in_reschedule_period=True),
                     session=session)
                 if deps_met or old_state != ut.current_state(session=session):
                     no_dependencies_met = False
@@ -5017,37 +4837,35 @@ def update_state(self, session=None):
         duration = (timezone.utcnow() - start_dttm).total_seconds() * 1000
         Stats.timing("dagrun.dependency-check.{}".format(self.dag_id), duration)
 
-        # future: remove the check on adhoc tasks (=active_tasks)
-        if len(tis) == len(dag.active_tasks):
-            root_ids = [t.task_id for t in dag.roots]
-            roots = [t for t in tis if t.task_id in root_ids]
-
-            # if all roots finished and at least one failed, the run failed
-            if (not unfinished_tasks and
-                    any(r.state in (State.FAILED, State.UPSTREAM_FAILED) for r in roots)):
-                self.log.info('Marking run %s failed', self)
-                self.state = State.FAILED
-                dag.handle_callback(self, success=False, reason='task_failure',
-                                    session=session)
-
-            # if all roots succeeded and no unfinished tasks, the run succeeded
-            elif not unfinished_tasks and all(r.state in (State.SUCCESS, State.SKIPPED)
-                                              for r in roots):
-                self.log.info('Marking run %s successful', self)
-                self.state = State.SUCCESS
-                dag.handle_callback(self, success=True, reason='success', session=session)
-
-            # if *all tasks* are deadlocked, the run failed
-            elif (unfinished_tasks and none_depends_on_past and
-                  none_task_concurrency and no_dependencies_met):
-                self.log.info('Deadlock; marking run %s failed', self)
-                self.state = State.FAILED
-                dag.handle_callback(self, success=False, reason='all_tasks_deadlocked',
-                                    session=session)
-
-            # finally, if the roots aren't done, the dag is still running
-            else:
-                self.state = State.RUNNING
+        root_ids = [t.task_id for t in dag.roots]
+        roots = [t for t in tis if t.task_id in root_ids]
+
+        # if all roots finished and at least one failed, the run failed
+        if (not unfinished_tasks and
+                any(r.state in (State.FAILED, State.UPSTREAM_FAILED) for r in roots)):
+            self.log.info('Marking run %s failed', self)
+            self.set_state(State.FAILED)
+            dag.handle_callback(self, success=False, reason='task_failure',
+                                session=session)
+
+        # if all roots succeeded and no unfinished tasks, the run succeeded
+        elif not unfinished_tasks and all(r.state in (State.SUCCESS, State.SKIPPED)
+                                          for r in roots):
+            self.log.info('Marking run %s successful', self)
+            self.set_state(State.SUCCESS)
+            dag.handle_callback(self, success=True, reason='success', session=session)
+
+        # if *all tasks* are deadlocked, the run failed
+        elif (unfinished_tasks and none_depends_on_past and
+              none_task_concurrency and no_dependencies_met):
+            self.log.info('Deadlock; marking run %s failed', self)
+            self.set_state(State.FAILED)
+            dag.handle_callback(self, success=False, reason='all_tasks_deadlocked',
+                                session=session)
+
+        # finally, if the roots aren't done, the dag is still running
+        else:
+            self.set_state(State.RUNNING)
 
         # todo: determine we want to use with_for_update to make sure to lock the run
         session.merge(self)
@@ -5091,10 +4909,13 @@ def verify_integrity(self, session=None):
 
         # check for missing tasks
         for task in six.itervalues(dag.task_dict):
-            if task.adhoc:
+            if task.start_date > self.execution_date and not self.is_backfill:
                 continue
 
             if task.task_id not in task_ids:
+                Stats.incr(
+                    "task_instance_created-{}".format(task.__class__.__name__),
+                    1, 1)
                 ti = TaskInstance(task, self.execution_date)
                 session.add(ti)
 
@@ -5108,7 +4929,7 @@ def get_run(session, dag_id, execution_date):
         :param execution_date: execution date
         :type execution_date: datetime
         :return: DagRun corresponding to the given dag_id and execution date
-        if one exists. None otherwise.
+            if one exists. None otherwise.
         :rtype: DagRun
         """
         qry = session.query(DagRun).filter(
@@ -5121,7 +4942,10 @@ def get_run(session, dag_id, execution_date):
     @property
     def is_backfill(self):
         from airflow.jobs import BackfillJob
-        return self.run_id.startswith(BackfillJob.ID_PREFIX)
+        return (
+            self.run_id is not None and
+            self.run_id.startswith(BackfillJob.ID_PREFIX)
+        )
 
     @classmethod
     @provide_session
@@ -5202,84 +5026,6 @@ def open_slots(self, session):
         return self.slots - used_slots - queued_slots
 
 
-class SlaMiss(Base):
-    """
-    Model that stores a history of the SLA that have been missed.
-    It is used to keep track of SLA failures over time and to avoid double
-    triggering alert emails.
-    """
-    __tablename__ = "sla_miss"
-
-    task_id = Column(String(ID_LEN), primary_key=True)
-    dag_id = Column(String(ID_LEN), primary_key=True)
-    execution_date = Column(UtcDateTime, primary_key=True)
-    email_sent = Column(Boolean, default=False)
-    timestamp = Column(UtcDateTime)
-    description = Column(Text)
-    notification_sent = Column(Boolean, default=False)
-
-    def __repr__(self):
-        return str((
-            self.dag_id, self.task_id, self.execution_date.isoformat()))
-
-
-class ImportError(Base):
-    __tablename__ = "import_error"
-    id = Column(Integer, primary_key=True)
-    timestamp = Column(UtcDateTime)
-    filename = Column(String(1024))
-    stacktrace = Column(Text)
-
-
-class KubeResourceVersion(Base):
-    __tablename__ = "kube_resource_version"
-    one_row_id = Column(Boolean, server_default=sqltrue(), primary_key=True)
-    resource_version = Column(String(255))
-
-    @staticmethod
-    @provide_session
-    def get_current_resource_version(session=None):
-        (resource_version,) = session.query(KubeResourceVersion.resource_version).one()
-        return resource_version
-
-    @staticmethod
-    @provide_session
-    def checkpoint_resource_version(resource_version, session=None):
-        if resource_version:
-            session.query(KubeResourceVersion).update({
-                KubeResourceVersion.resource_version: resource_version
-            })
-            session.commit()
-
-    @staticmethod
-    @provide_session
-    def reset_resource_version(session=None):
-        session.query(KubeResourceVersion).update({
-            KubeResourceVersion.resource_version: '0'
-        })
-        session.commit()
-        return '0'
-
-
-class KubeWorkerIdentifier(Base):
-    __tablename__ = "kube_worker_uuid"
-    one_row_id = Column(Boolean, server_default=sqltrue(), primary_key=True)
-    worker_uuid = Column(String(255))
-
-    @staticmethod
-    @provide_session
-    def get_or_create_current_kube_worker_uuid(session=None):
-        (worker_uuid,) = session.query(KubeWorkerIdentifier.worker_uuid).one()
-        if worker_uuid == '':
-            worker_uuid = str(uuid.uuid4())
-            KubeWorkerIdentifier.checkpoint_kube_worker_uuid(worker_uuid, session)
-        return worker_uuid
-
-    @staticmethod
-    @provide_session
-    def checkpoint_kube_worker_uuid(worker_uuid, session=None):
-        if worker_uuid:
-            session.query(KubeWorkerIdentifier).update({
-                KubeWorkerIdentifier.worker_uuid: worker_uuid
-            })
-            session.commit()
+# To avoid circular import on Python2.7 we need to define this at the _bottom_
+from airflow.models.connection import Connection  # noqa: E402,F401
+from airflow.models.skipmixin import SkipMixin  # noqa: F401
diff --git a/airflow/models/base.py b/airflow/models/base.py
new file mode 100644
index 0000000000000..97c6b777984d8
--- /dev/null
+++ b/airflow/models/base.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Any
+from sqlalchemy import MetaData
+from sqlalchemy.ext.declarative import declarative_base
+
+import airflow
+
+SQL_ALCHEMY_SCHEMA = airflow.configuration.get("core", "SQL_ALCHEMY_SCHEMA")
+
+metadata = (
+    None
+    if not SQL_ALCHEMY_SCHEMA or SQL_ALCHEMY_SCHEMA.isspace()
+    else MetaData(schema=SQL_ALCHEMY_SCHEMA)
+)
+Base = declarative_base(metadata=metadata)  # type: Any
+
+ID_LEN = 250
diff --git a/airflow/models/connection.py b/airflow/models/connection.py
new file mode 100644
index 0000000000000..1ac53553a9609
--- /dev/null
+++ b/airflow/models/connection.py
@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from builtins import bytes
+from urllib.parse import urlparse, unquote, parse_qsl
+
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import synonym
+
+from airflow.exceptions import AirflowException
+from airflow import LoggingMixin
+from airflow.models import get_fernet
+from airflow.models.base import Base, ID_LEN
+
+
+# Python automatically converts all letters to lowercase in hostname
+# See: https://issues.apache.org/jira/browse/AIRFLOW-3615
+def parse_netloc_to_hostname(uri_parts):
+    hostname = unquote(uri_parts.hostname or '')
+    if '/' in hostname:
+        hostname = uri_parts.netloc
+        if "@" in hostname:
+            hostname = hostname.rsplit("@", 1)[1]
+        if ":" in hostname:
+            hostname = hostname.split(":", 1)[0]
+        hostname = unquote(hostname)
+    return hostname
+
+
+class Connection(Base, LoggingMixin):
+    """
+    Placeholder to store information about different database instances
+    connection information. The idea here is that scripts use references to
+    database instances (conn_id) instead of hard coding hostname, logins and
+    passwords when using operators or hooks.
+    """
+    __tablename__ = "connection"
+
+    id = Column(Integer(), primary_key=True)
+    conn_id = Column(String(ID_LEN))
+    conn_type = Column(String(500))
+    host = Column(String(500))
+    schema = Column(String(500))
+    login = Column(String(500))
+    _password = Column('password', String(5000))
+    port = Column(Integer())
+    is_encrypted = Column(Boolean, unique=False, default=False)
+    is_extra_encrypted = Column(Boolean, unique=False, default=False)
+    _extra = Column('extra', String(5000))
+
+    _types = [
+        ('docker', 'Docker Registry',),
+        ('fs', 'File (path)'),
+        ('ftp', 'FTP',),
+        ('google_cloud_platform', 'Google Cloud Platform'),
+        ('hdfs', 'HDFS',),
+        ('http', 'HTTP',),
+        ('hive_cli', 'Hive Client Wrapper',),
+        ('hive_metastore', 'Hive Metastore Thrift',),
+        ('hiveserver2', 'Hive Server 2 Thrift',),
+        ('jdbc', 'Jdbc Connection',),
+        ('jenkins', 'Jenkins'),
+        ('mysql', 'MySQL',),
+        ('postgres', 'Postgres',),
+        ('oracle', 'Oracle',),
+        ('vertica', 'Vertica',),
+        ('presto', 'Presto',),
+        ('s3', 'S3',),
+        ('samba', 'Samba',),
+        ('sqlite', 'Sqlite',),
+        ('ssh', 'SSH',),
+        ('cloudant', 'IBM Cloudant',),
+        ('mssql', 'Microsoft SQL Server'),
+        ('mesos_framework-id', 'Mesos Framework ID'),
+        ('jira', 'JIRA',),
+        ('redis', 'Redis',),
+        ('wasb', 'Azure Blob Storage'),
+        ('databricks', 'Databricks',),
+        ('aws', 'Amazon Web Services',),
+        ('emr', 'Elastic MapReduce',),
+        ('snowflake', 'Snowflake',),
+        ('segment', 'Segment',),
+        ('azure_data_lake', 'Azure Data Lake'),
+        ('azure_container_instances', 'Azure Container Instances'),
+        ('azure_cosmos', 'Azure CosmosDB'),
+        ('cassandra', 'Cassandra',),
+        ('qubole', 'Qubole'),
+        ('mongo', 'MongoDB'),
+        ('gcpcloudsql', 'Google Cloud SQL'),
+    ]
+
+    def __init__(
+            self, conn_id=None, conn_type=None,
+            host=None, login=None, password=None,
+            schema=None, port=None, extra=None,
+            uri=None):
+        self.conn_id = conn_id
+        if uri:
+            self.parse_from_uri(uri)
+        else:
+            self.conn_type = conn_type
+            self.host = host
+            self.login = login
+            self.password = password
+            self.schema = schema
+            self.port = port
+            self.extra = extra
+
+    def parse_from_uri(self, uri):
+        uri_parts = urlparse(uri)
+        conn_type = uri_parts.scheme
+        if conn_type == 'postgresql':
+            conn_type = 'postgres'
+        elif '-' in conn_type:
+            conn_type = conn_type.replace('-', '_')
+        self.conn_type = conn_type
+        self.host = parse_netloc_to_hostname(uri_parts)
+        quoted_schema = uri_parts.path[1:]
+        self.schema = unquote(quoted_schema) if quoted_schema else quoted_schema
+        self.login = unquote(uri_parts.username) \
+            if uri_parts.username else uri_parts.username
+        self.password = unquote(uri_parts.password) \
+            if uri_parts.password else uri_parts.password
+        self.port = uri_parts.port
+        if uri_parts.query:
+            self.extra = json.dumps(dict(parse_qsl(uri_parts.query)))
+
+    def get_password(self):
+        if self._password and self.is_encrypted:
+            fernet = get_fernet()
+            if not fernet.is_encrypted:
+                raise AirflowException(
+                    "Can't decrypt encrypted password for login={}, \
+                    FERNET_KEY configuration is missing".format(self.login))
+            return fernet.decrypt(bytes(self._password, 'utf-8')).decode()
+        else:
+            return self._password
+
+    def set_password(self, value):
+        if value:
+            fernet = get_fernet()
+            self._password = fernet.encrypt(bytes(value, 'utf-8')).decode()
+            self.is_encrypted = fernet.is_encrypted
+
+    @declared_attr
+    def password(cls):
+        return synonym('_password',
+                       descriptor=property(cls.get_password, cls.set_password))
+
+    def get_extra(self):
+        if self._extra and self.is_extra_encrypted:
+            fernet = get_fernet()
+            if not fernet.is_encrypted:
+                raise AirflowException(
+                    "Can't decrypt `extra` params for login={},\
+                    FERNET_KEY configuration is missing".format(self.login))
+            return fernet.decrypt(bytes(self._extra, 'utf-8')).decode()
+        else:
+            return self._extra
+
+    def set_extra(self, value):
+        if value:
+            fernet = get_fernet()
+            self._extra = fernet.encrypt(bytes(value, 'utf-8')).decode()
+            self.is_extra_encrypted = fernet.is_encrypted
+        else:
+            self._extra = value
+            self.is_extra_encrypted = False
+
+    @declared_attr
+    def extra(cls):
+        return synonym('_extra',
+                       descriptor=property(cls.get_extra, cls.set_extra))
+
+    def rotate_fernet_key(self):
+        fernet = get_fernet()
+        if self._password and self.is_encrypted:
+            self._password = fernet.rotate(self._password.encode('utf-8')).decode()
+        if self._extra and self.is_extra_encrypted:
+            self._extra = fernet.rotate(self._extra.encode('utf-8')).decode()
+
+    def get_hook(self):
+        try:
+            if self.conn_type == 'mysql':
+                from airflow.hooks.mysql_hook import MySqlHook
+                return MySqlHook(mysql_conn_id=self.conn_id)
+            elif self.conn_type == 'google_cloud_platform':
+                from airflow.contrib.hooks.bigquery_hook import BigQueryHook
+                return BigQueryHook(bigquery_conn_id=self.conn_id)
+            elif self.conn_type == 'postgres':
+                from airflow.hooks.postgres_hook import PostgresHook
+                return PostgresHook(postgres_conn_id=self.conn_id)
+            elif self.conn_type == 'hive_cli':
+                from airflow.hooks.hive_hooks import HiveCliHook
+                return HiveCliHook(hive_cli_conn_id=self.conn_id)
+            elif self.conn_type == 'presto':
+                from airflow.hooks.presto_hook import PrestoHook
+                return PrestoHook(presto_conn_id=self.conn_id)
+            elif self.conn_type == 'hiveserver2':
+                from airflow.hooks.hive_hooks import HiveServer2Hook
+                return HiveServer2Hook(hiveserver2_conn_id=self.conn_id)
+            elif self.conn_type == 'sqlite':
+                from airflow.hooks.sqlite_hook import SqliteHook
+                return SqliteHook(sqlite_conn_id=self.conn_id)
+            elif self.conn_type == 'jdbc':
+                from airflow.hooks.jdbc_hook import JdbcHook
+                return JdbcHook(jdbc_conn_id=self.conn_id)
+            elif self.conn_type == 'mssql':
+                from airflow.hooks.mssql_hook import MsSqlHook
+                return MsSqlHook(mssql_conn_id=self.conn_id)
+            elif self.conn_type == 'oracle':
+                from airflow.hooks.oracle_hook import OracleHook
+                return OracleHook(oracle_conn_id=self.conn_id)
+            elif self.conn_type == 'vertica':
+                from airflow.contrib.hooks.vertica_hook import VerticaHook
+                return VerticaHook(vertica_conn_id=self.conn_id)
+            elif self.conn_type == 'cloudant':
+                from airflow.contrib.hooks.cloudant_hook import CloudantHook
+                return CloudantHook(cloudant_conn_id=self.conn_id)
+            elif self.conn_type == 'jira':
+                from airflow.contrib.hooks.jira_hook import JiraHook
+                return JiraHook(jira_conn_id=self.conn_id)
+            elif self.conn_type == 'redis':
+                from airflow.contrib.hooks.redis_hook import RedisHook
+                return RedisHook(redis_conn_id=self.conn_id)
+            elif self.conn_type == 'wasb':
+                from airflow.contrib.hooks.wasb_hook import WasbHook
+                return WasbHook(wasb_conn_id=self.conn_id)
+            elif self.conn_type == 'docker':
+                from airflow.hooks.docker_hook import DockerHook
+                return DockerHook(docker_conn_id=self.conn_id)
+            elif self.conn_type == 'azure_data_lake':
+                from airflow.contrib.hooks.azure_data_lake_hook import AzureDataLakeHook
+                return AzureDataLakeHook(azure_data_lake_conn_id=self.conn_id)
+            elif self.conn_type == 'azure_cosmos':
+                from airflow.contrib.hooks.azure_cosmos_hook import AzureCosmosDBHook
+                return AzureCosmosDBHook(azure_cosmos_conn_id=self.conn_id)
+            elif self.conn_type == 'cassandra':
+                from airflow.contrib.hooks.cassandra_hook import CassandraHook
+                return CassandraHook(cassandra_conn_id=self.conn_id)
+            elif self.conn_type == 'mongo':
+                from airflow.contrib.hooks.mongo_hook import MongoHook
+                return MongoHook(conn_id=self.conn_id)
+            elif self.conn_type == 'gcpcloudsql':
+                from airflow.contrib.hooks.gcp_sql_hook import CloudSqlDatabaseHook
+                return CloudSqlDatabaseHook(gcp_cloudsql_conn_id=self.conn_id)
+        except Exception:
+            pass
+
+    def __repr__(self):
+        return self.conn_id
+
+    def debug_info(self):
+        return ("id: {}. Host: {}, Port: {}, Schema: {}, "
+                "Login: {}, Password: {}, extra: {}".
+                format(self.conn_id,
+                       self.host,
+                       self.port,
+                       self.schema,
+                       self.login,
+                       "XXXXXXXX" if self.password else None,
+                       self.extra_dejson))
+
+    @property
+    def extra_dejson(self):
+        """Returns the extra property by deserializing json."""
+        obj = {}
+        if self.extra:
+            try:
+                obj = json.loads(self.extra)
+            except Exception as e:
+                self.log.exception(e)
+                self.log.error("Failed parsing the json for conn_id %s", self.conn_id)
+
+        return obj
diff --git a/airflow/models/dagpickle.py b/airflow/models/dagpickle.py
new file mode 100644
index 0000000000000..e261ab1ac009d
--- /dev/null
+++ b/airflow/models/dagpickle.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import dill
+from sqlalchemy import Column, Integer, PickleType, Text
+
+from airflow.models.base import Base
+from airflow.utils import timezone
+from airflow.utils.sqlalchemy import UtcDateTime
+
+
+class DagPickle(Base):
+    """
+    Dags can originate from different places (user repos, master repo, ...)
+    and also get executed in different places (different executors). This
+    object represents a version of a DAG and becomes a source of truth for
+    a BackfillJob execution. A pickle is a native python serialized object,
+    and in this case gets stored in the database for the duration of the job.
+
+    The executors pick up the DagPickle id and read the dag definition from
+    the database.
+    """
+
+    id = Column(Integer, primary_key=True)
+    pickle = Column(PickleType(pickler=dill))
+    created_dttm = Column(UtcDateTime, default=timezone.utcnow)
+    pickle_hash = Column(Text)
+
+    __tablename__ = "dag_pickle"
+
+    def __init__(self, dag):
+        self.dag_id = dag.dag_id
+        if hasattr(dag, 'template_env'):
+            dag.template_env = None
+        self.pickle_hash = hash(dag)
+        self.pickle = dag
diff --git a/airflow/models/errors.py b/airflow/models/errors.py
new file mode 100644
index 0000000000000..6a3797ca3df9b
--- /dev/null
+++ b/airflow/models/errors.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import Integer, Column, String, Text
+
+from airflow.models.base import Base
+from airflow.utils.sqlalchemy import UtcDateTime
+
+
+class ImportError(Base):
+    __tablename__ = "import_error"
+    id = Column(Integer, primary_key=True)
+    timestamp = Column(UtcDateTime)
+    filename = Column(String(1024))
+    stacktrace = Column(Text)
diff --git a/airflow/models/kubernetes.py b/airflow/models/kubernetes.py
new file mode 100644
index 0000000000000..a18689eefd316
--- /dev/null
+++ b/airflow/models/kubernetes.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import uuid
+
+from sqlalchemy import Column, Boolean, true as sqltrue, String
+
+from airflow.models.base import Base
+from airflow.utils.db import provide_session
+
+
+class KubeResourceVersion(Base):
+    __tablename__ = "kube_resource_version"
+    one_row_id = Column(Boolean, server_default=sqltrue(), primary_key=True)
+    resource_version = Column(String(255))
+
+    @staticmethod
+    @provide_session
+    def get_current_resource_version(session=None):
+        (resource_version,) = session.query(KubeResourceVersion.resource_version).one()
+        return resource_version
+
+    @staticmethod
+    @provide_session
+    def checkpoint_resource_version(resource_version, session=None):
+        if resource_version:
+            session.query(KubeResourceVersion).update({
+                KubeResourceVersion.resource_version: resource_version
+            })
+            session.commit()
+
+    @staticmethod
+    @provide_session
+    def reset_resource_version(session=None):
+        session.query(KubeResourceVersion).update({
+            KubeResourceVersion.resource_version: '0'
+        })
+        session.commit()
+        return '0'
+
+
+class KubeWorkerIdentifier(Base):
+    __tablename__ = "kube_worker_uuid"
+    one_row_id = Column(Boolean, server_default=sqltrue(), primary_key=True)
+    worker_uuid = Column(String(255))
+
+    @staticmethod
+    @provide_session
+    def get_or_create_current_kube_worker_uuid(session=None):
+        (worker_uuid,) = session.query(KubeWorkerIdentifier.worker_uuid).one()
+        if worker_uuid == '':
+            worker_uuid = str(uuid.uuid4())
+            KubeWorkerIdentifier.checkpoint_kube_worker_uuid(worker_uuid, session)
+        return worker_uuid
+
+    @staticmethod
+    @provide_session
+    def checkpoint_kube_worker_uuid(worker_uuid, session=None):
+        if worker_uuid:
+            session.query(KubeWorkerIdentifier).update({
+                KubeWorkerIdentifier.worker_uuid: worker_uuid
+            })
+            session.commit()
diff --git a/airflow/models/log.py b/airflow/models/log.py
new file mode 100644
index 0000000000000..fa3fed60ee240
--- /dev/null
+++ b/airflow/models/log.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import Column, Integer, String, Text, Index
+
+from airflow.models.base import Base, ID_LEN
+from airflow.utils import timezone
+from airflow.utils.sqlalchemy import UtcDateTime
+
+
+class Log(Base):
+    """
+    Used to actively log events to the database
+    """
+
+    __tablename__ = "log"
+
+    id = Column(Integer, primary_key=True)
+    dttm = Column(UtcDateTime)
+    dag_id = Column(String(ID_LEN))
+    task_id = Column(String(ID_LEN))
+    event = Column(String(30))
+    execution_date = Column(UtcDateTime)
+    owner = Column(String(500))
+    extra = Column(Text)
+
+    __table_args__ = (
+        Index('idx_log_dag', dag_id),
+    )
+
+    def __init__(self, event, task_instance, owner=None, extra=None, **kwargs):
+        self.dttm = timezone.utcnow()
+        self.event = event
+        self.extra = extra
+
+        task_owner = None
+
+        if task_instance:
+            self.dag_id = task_instance.dag_id
+            self.task_id = task_instance.task_id
+            self.execution_date = task_instance.execution_date
+            task_owner = task_instance.task.owner
+
+        if 'task_id' in kwargs:
+            self.task_id = kwargs['task_id']
+        if 'dag_id' in kwargs:
+            self.dag_id = kwargs['dag_id']
+        if 'execution_date' in kwargs:
+            if kwargs['execution_date']:
+                self.execution_date = kwargs['execution_date']
+
+        self.owner = owner or task_owner
diff --git a/airflow/models/skipmixin.py b/airflow/models/skipmixin.py
new file mode 100644
index 0000000000000..c0adbd20aa4a4
--- /dev/null
+++ b/airflow/models/skipmixin.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.models import TaskInstance
+from airflow.utils import timezone
+from airflow.utils.db import provide_session
+from airflow.utils.log.logging_mixin import LoggingMixin
+from airflow.utils.state import State
+
+
+class SkipMixin(LoggingMixin):
+    @provide_session
+    def skip(self, dag_run, execution_date, tasks, session=None):
+        """
+        Sets tasks instances to skipped from the same dag run.
+
+        :param dag_run: the DagRun for which to set the tasks to skipped
+        :param execution_date: execution_date
+        :param tasks: tasks to skip (not task_ids)
+        :param session: db session to use
+        """
+        if not tasks:
+            return
+
+        task_ids = [d.task_id for d in tasks]
+        now = timezone.utcnow()
+
+        if dag_run:
+            session.query(TaskInstance).filter(
+                TaskInstance.dag_id == dag_run.dag_id,
+                TaskInstance.execution_date == dag_run.execution_date,
+                TaskInstance.task_id.in_(task_ids)
+            ).update({TaskInstance.state: State.SKIPPED,
+                      TaskInstance.start_date: now,
+                      TaskInstance.end_date: now},
+                     synchronize_session=False)
+            session.commit()
+        else:
+            assert execution_date is not None, "Execution date is None and no dag run"
+
+            self.log.warning("No DAG RUN present this should not happen")
+            # this is defensive against dag runs that are not complete
+            for task in tasks:
+                ti = TaskInstance(task, execution_date=execution_date)
+                ti.state = State.SKIPPED
+                ti.start_date = now
+                ti.end_date = now
+                session.merge(ti)
+
+            session.commit()
diff --git a/airflow/models/slamiss.py b/airflow/models/slamiss.py
new file mode 100644
index 0000000000000..0981be886ea01
--- /dev/null
+++ b/airflow/models/slamiss.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import Boolean, Column, String, Index, Text
+
+from airflow.models.base import Base, ID_LEN
+from airflow.utils.sqlalchemy import UtcDateTime
+
+
+class SlaMiss(Base):
+    """
+    Model that stores a history of the SLA that have been missed.
+    It is used to keep track of SLA failures over time and to avoid double
+    triggering alert emails.
+    """
+    __tablename__ = "sla_miss"
+
+    task_id = Column(String(ID_LEN), primary_key=True)
+    dag_id = Column(String(ID_LEN), primary_key=True)
+    execution_date = Column(UtcDateTime, primary_key=True)
+    email_sent = Column(Boolean, default=False)
+    timestamp = Column(UtcDateTime)
+    description = Column(Text)
+    notification_sent = Column(Boolean, default=False)
+
+    __table_args__ = (
+        Index('sm_dag', dag_id, unique=False),
+    )
+
+    def __repr__(self):
+        return str((
+            self.dag_id, self.task_id, self.execution_date.isoformat()))
diff --git a/airflow/models/taskfail.py b/airflow/models/taskfail.py
new file mode 100755
index 0000000000000..d9cc7af922730
--- /dev/null
+++ b/airflow/models/taskfail.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import Column, Index, Integer, String
+
+from airflow.models.base import Base, ID_LEN
+from airflow.utils.sqlalchemy import UtcDateTime
+
+
+class TaskFail(Base):
+    """
+    TaskFail tracks the failed run durations of each task instance.
+    """
+
+    __tablename__ = "task_fail"
+
+    id = Column(Integer, primary_key=True)
+    task_id = Column(String(ID_LEN), nullable=False)
+    dag_id = Column(String(ID_LEN), nullable=False)
+    execution_date = Column(UtcDateTime, nullable=False)
+    start_date = Column(UtcDateTime)
+    end_date = Column(UtcDateTime)
+    duration = Column(Integer)
+
+    __table_args__ = (
+        Index('idx_task_fail_dag_task_date', dag_id, task_id, execution_date,
+              unique=False),
+    )
+
+    def __init__(self, task, execution_date, start_date, end_date):
+        self.dag_id = task.dag_id
+        self.task_id = task.task_id
+        self.execution_date = execution_date
+        self.start_date = start_date
+        self.end_date = end_date
+        if self.end_date and self.start_date:
+            self.duration = (self.end_date - self.start_date).total_seconds()
+        else:
+            self.duration = None
diff --git a/airflow/models/taskreschedule.py b/airflow/models/taskreschedule.py
new file mode 100755
index 0000000000000..4bdb6431f1ae4
--- /dev/null
+++ b/airflow/models/taskreschedule.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import Column, ForeignKeyConstraint, Index, Integer, String, asc
+
+from airflow.models.base import Base, ID_LEN
+from airflow.utils.db import provide_session
+from airflow.utils.sqlalchemy import UtcDateTime
+
+
+class TaskReschedule(Base):
+    """
+    TaskReschedule tracks rescheduled task instances.
+    """
+
+    __tablename__ = "task_reschedule"
+
+    id = Column(Integer, primary_key=True)
+    task_id = Column(String(ID_LEN), nullable=False)
+    dag_id = Column(String(ID_LEN), nullable=False)
+    execution_date = Column(UtcDateTime, nullable=False)
+    try_number = Column(Integer, nullable=False)
+    start_date = Column(UtcDateTime, nullable=False)
+    end_date = Column(UtcDateTime, nullable=False)
+    duration = Column(Integer, nullable=False)
+    reschedule_date = Column(UtcDateTime, nullable=False)
+
+    __table_args__ = (
+        Index('idx_task_reschedule_dag_task_date', dag_id, task_id, execution_date,
+              unique=False),
+        ForeignKeyConstraint([task_id, dag_id, execution_date],
+                             ['task_instance.task_id', 'task_instance.dag_id',
+                              'task_instance.execution_date'],
+                             name='task_reschedule_dag_task_date_fkey',
+                             ondelete='CASCADE')
+    )
+
+    def __init__(self, task, execution_date, try_number, start_date, end_date,
+                 reschedule_date):
+        self.dag_id = task.dag_id
+        self.task_id = task.task_id
+        self.execution_date = execution_date
+        self.try_number = try_number
+        self.start_date = start_date
+        self.end_date = end_date
+        self.reschedule_date = reschedule_date
+        self.duration = (self.end_date - self.start_date).total_seconds()
+
+    @staticmethod
+    @provide_session
+    def find_for_task_instance(task_instance, session):
+        """
+        Returns all task reschedules for the task instance and try number,
+        in ascending order.
+
+        :param task_instance: the task instance to find task reschedules for
+        :type task_instance: airflow.models.TaskInstance
+        """
+        TR = TaskReschedule
+        return (
+            session
+            .query(TR)
+            .filter(TR.dag_id == task_instance.dag_id,
+                    TR.task_id == task_instance.task_id,
+                    TR.execution_date == task_instance.execution_date,
+                    TR.try_number == task_instance.try_number)
+            .order_by(asc(TR.id))
+            .all()
+        )
diff --git a/airflow/operators/__init__.py b/airflow/operators/__init__.py
index efdfd3ebe2dc1..00f34d038ccfd 100644
--- a/airflow/operators/__init__.py
+++ b/airflow/operators/__init__.py
@@ -19,7 +19,7 @@
 
 import sys
 import os
-from airflow.models import BaseOperator
+from airflow.models import BaseOperator  # noqa: F401
 
 # ------------------------------------------------------------------------
 #
diff --git a/airflow/operators/bash_operator.py b/airflow/operators/bash_operator.py
index 37a19dbe9bc31..54c2dfba8907a 100644
--- a/airflow/operators/bash_operator.py
+++ b/airflow/operators/bash_operator.py
@@ -18,25 +18,31 @@
 # under the License.
 
 
-from builtins import bytes
 import os
 import signal
 from subprocess import Popen, STDOUT, PIPE
 from tempfile import gettempdir, NamedTemporaryFile
 
+from builtins import bytes
+
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
 from airflow.utils.file import TemporaryDirectory
+from airflow.utils.operator_helpers import context_to_airflow_vars
 
 
 class BashOperator(BaseOperator):
     """
     Execute a Bash script, command or set of commands.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:BashOperator`
+
     :param bash_command: The command, set of commands or reference to a
         bash script (must be '.sh') to be executed. (templated)
-    :type bash_command: string
+    :type bash_command: str
     :param xcom_push: If xcom_push is True, the last line written to stdout
         will also be pushed to an XCom when the bash command completes.
     :type xcom_push: bool
@@ -45,7 +51,8 @@ class BashOperator(BaseOperator):
         of inheriting the current process environment, which is the default
         behavior. (templated)
     :type env: dict
-    :type output_encoding: output encoding of bash command
+    :param output_encoding: Output encoding of bash command
+    :type output_encoding: str
     """
     template_fields = ('bash_command', 'env')
     template_ext = ('.sh', '.bash',)
@@ -73,6 +80,16 @@ def execute(self, context):
         """
         self.log.info("Tmp dir root location: \n %s", gettempdir())
 
+        # Prepare env for child process.
+        if self.env is None:
+            self.env = os.environ.copy()
+        airflow_context_vars = context_to_airflow_vars(context, in_env_var_format=True)
+        self.log.info('Exporting the following env vars:\n%s',
+                      '\n'.join(["{}={}".format(k, v)
+                                 for k, v in
+                                 airflow_context_vars.items()]))
+        self.env.update(airflow_context_vars)
+
         self.lineage_data = self.bash_command
 
         with TemporaryDirectory(prefix='airflowtmp') as tmp_dir:
diff --git a/airflow/operators/check_operator.py b/airflow/operators/check_operator.py
index 5a31737fd5a6c..e611ce0af683d 100644
--- a/airflow/operators/check_operator.py
+++ b/airflow/operators/check_operator.py
@@ -19,6 +19,7 @@
 
 from builtins import zip
 from builtins import str
+from typing import Iterable
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
@@ -59,11 +60,11 @@ class CheckOperator(BaseOperator):
     single record from an external source.
 
     :param sql: the sql to be executed. (templated)
-    :type sql: string
+    :type sql: str
     """
 
-    template_fields = ('sql',)
-    template_ext = ('.hql', '.sql',)
+    template_fields = ('sql',)  # type: Iterable[str]
+    template_ext = ('.hql', '.sql',)  # type: Iterable[str]
     ui_color = '#fff7e6'
 
     @apply_defaults
@@ -114,14 +115,14 @@ class ValueCheckOperator(BaseOperator):
     single record from an external source.
 
     :param sql: the sql to be executed. (templated)
-    :type sql: string
+    :type sql: str
     """
 
     __mapper_args__ = {
         'polymorphic_identity': 'ValueCheckOperator'
     }
-    template_fields = ('sql', 'pass_value',)
-    template_ext = ('.hql', '.sql',)
+    template_fields = ('sql', 'pass_value',)  # type: Iterable[str]
+    template_ext = ('.hql', '.sql',)  # type: Iterable[str]
     ui_color = '#fff7e6'
 
     @apply_defaults
@@ -147,7 +148,7 @@ def execute(self, context=None):
         is_numeric_value_check = isinstance(pass_value_conv, float)
 
         tolerance_pct_str = None
-        if (self.tol is not None):
+        if self.tol is not None:
             tolerance_pct_str = str(self.tol * 100) + '%'
 
         except_temp = ("Test failed.\nPass value:{pass_value_conv}\n"
@@ -158,7 +159,7 @@ def execute(self, context=None):
         elif is_numeric_value_check:
             try:
                 num_rec = [float(r) for r in records]
-            except (ValueError, TypeError) as e:
+            except (ValueError, TypeError):
                 cvestr = "Converting a result to float failed.\n"
                 raise AirflowException(cvestr + except_temp.format(**locals()))
             if self.has_tolerance:
@@ -196,8 +197,8 @@ class IntervalCheckOperator(BaseOperator):
     __mapper_args__ = {
         'polymorphic_identity': 'IntervalCheckOperator'
     }
-    template_fields = ('sql1', 'sql2')
-    template_ext = ('.hql', '.sql',)
+    template_fields = ('sql1', 'sql2')  # type: Iterable[str]
+    template_ext = ('.hql', '.sql',)  # type: Iterable[str]
     ui_color = '#fff7e6'
 
     @apply_defaults
@@ -233,29 +234,25 @@ def execute(self, context=None):
         reference = dict(zip(self.metrics_sorted, row2))
         ratios = {}
         test_results = {}
-        rlog = "Ratio for {0}: {1} \n Ratio threshold : {2}"
-        fstr = "'{k}' check failed. {r} is above {tr}"
-        estr = "The following tests have failed:\n {0}"
-        countstr = "The following {j} tests out of {n} failed:"
         for m in self.metrics_sorted:
             if current[m] == 0 or reference[m] == 0:
                 ratio = None
             else:
                 ratio = float(max(current[m], reference[m])) / \
                     min(current[m], reference[m])
-            self.log.info(rlog.format(m, ratio, self.metrics_thresholds[m]))
+            self.log.info("Ratio for %s: %s \n Ratio threshold : %s", m, ratio, self.metrics_thresholds[m])
             ratios[m] = ratio
             test_results[m] = ratio < self.metrics_thresholds[m]
         if not all(test_results.values()):
             failed_tests = [it[0] for it in test_results.items() if not it[1]]
             j = len(failed_tests)
             n = len(self.metrics_sorted)
-            self.log.warning(countstr.format(**locals()))
+            self.log.warning("The following %s tests out of %s failed:", j, n)
             for k in failed_tests:
                 self.log.warning(
-                    fstr.format(k=k, r=ratios[k], tr=self.metrics_thresholds[k])
+                    "'%s' check failed. %s is above %s", k, ratios[k], self.metrics_thresholds[k]
                 )
-            raise AirflowException(estr.format(", ".join(failed_tests)))
+            raise AirflowException("The following tests have failed:\n {0}".format(", ".join(failed_tests)))
         self.log.info("All tests have passed")
 
     def get_db_hook(self):
diff --git a/airflow/operators/dagrun_operator.py b/airflow/operators/dagrun_operator.py
index 53814af41024d..1bc1979a1afb4 100644
--- a/airflow/operators/dagrun_operator.py
+++ b/airflow/operators/dagrun_operator.py
@@ -17,6 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import datetime
+import six
 from airflow.models import BaseOperator
 from airflow.utils import timezone
 from airflow.utils.decorators import apply_defaults
@@ -35,7 +37,7 @@ class TriggerDagRunOperator(BaseOperator):
     """
     Triggers a DAG run for a specified ``dag_id``
 
-    :param trigger_dag_id: the dag_id to trigger
+    :param trigger_dag_id: the dag_id to trigger (templated)
     :type trigger_dag_id: str
     :param python_callable: a reference to a python function that will be
         called while passing it the ``context`` object and a placeholder
@@ -47,11 +49,10 @@ class TriggerDagRunOperator(BaseOperator):
         to your tasks while executing that DAG run. Your function header
         should look like ``def foo(context, dag_run_obj):``
     :type python_callable: python callable
-    :param execution_date: Execution date for the dag
-    :type execution_date: datetime.datetime
+    :param execution_date: Execution date for the dag (templated)
+    :type execution_date: str or datetime.datetime
     """
-    template_fields = tuple()
-    template_ext = tuple()
+    template_fields = ('trigger_dag_id', 'execution_date')
     ui_color = '#ffefeb'
 
     @apply_defaults
@@ -64,10 +65,26 @@ def __init__(
         super(TriggerDagRunOperator, self).__init__(*args, **kwargs)
         self.python_callable = python_callable
         self.trigger_dag_id = trigger_dag_id
-        self.execution_date = execution_date
+
+        if isinstance(execution_date, datetime.datetime):
+            self.execution_date = execution_date.isoformat()
+        elif isinstance(execution_date, six.string_types):
+            self.execution_date = execution_date
+        elif execution_date is None:
+            self.execution_date = execution_date
+        else:
+            raise TypeError(
+                'Expected str or datetime.datetime type '
+                'for execution_date. Got {}'.format(
+                    type(execution_date)))
 
     def execute(self, context):
-        dro = DagRunOrder(run_id='trig__' + timezone.utcnow().isoformat())
+        if self.execution_date is not None:
+            run_id = 'trig__{}'.format(self.execution_date)
+            self.execution_date = timezone.parse(self.execution_date)
+        else:
+            run_id = 'trig__' + timezone.utcnow().isoformat()
+        dro = DagRunOrder(run_id=run_id)
         if self.python_callable is not None:
             dro = self.python_callable(context, dro)
         if dro:
diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py
index 69dc1ebef71e6..1d550cec8804a 100644
--- a/airflow/operators/docker_operator.py
+++ b/airflow/operators/docker_operator.py
@@ -43,22 +43,31 @@ class DockerOperator(BaseOperator):
     be provided with the parameter ``docker_conn_id``.
 
     :param image: Docker image from which to create the container.
+        If image tag is omitted, "latest" will be used.
     :type image: str
     :param api_version: Remote API version. Set to ``auto`` to automatically
         detect the server's version.
     :type api_version: str
+    :param auto_remove: Auto-removal of the container on daemon side when the
+        container's process exits.
+        The default is False.
+    :type auto_remove: bool
     :param command: Command to be run in the container. (templated)
     :type command: str or list
     :param cpus: Number of CPUs to assign to the container.
         This value gets multiplied with 1024. See
         https://docs.docker.com/engine/reference/run/#cpu-share-constraint
     :type cpus: float
+    :param dns: Docker custom DNS servers
+    :type dns: list[str]
+    :param dns_search: Docker custom DNS search domain
+    :type dns_search: list[str]
     :param docker_url: URL of the host running the docker daemon.
         Default is unix://var/run/docker.sock
     :type docker_url: str
     :param environment: Environment variables to set in the container. (templated)
     :type environment: dict
-    :param force_pull: Pull the docker image on every run. Default is false.
+    :param force_pull: Pull the docker image on every run. Default is False.
     :type force_pull: bool
     :param mem_limit: Maximum amount of memory the container can use.
         Either a float value, which represents the limit in bytes,
@@ -99,6 +108,9 @@ class DockerOperator(BaseOperator):
     :type xcom_all: bool
     :param docker_conn_id: ID of the Airflow connection to use
     :type docker_conn_id: str
+    :param shm_size: Size of ``/dev/shm`` in bytes. The size must be
+        greater than 0. If omitted uses system default.
+    :type shm_size: int
     """
     template_fields = ('command', 'environment',)
     template_ext = ('.sh', '.bash',)
@@ -127,13 +139,20 @@ def __init__(
             xcom_push=False,
             xcom_all=False,
             docker_conn_id=None,
+            dns=None,
+            dns_search=None,
+            auto_remove=False,
+            shm_size=None,
             *args,
             **kwargs):
 
         super(DockerOperator, self).__init__(*args, **kwargs)
         self.api_version = api_version
+        self.auto_remove = auto_remove
         self.command = command
         self.cpus = cpus
+        self.dns = dns
+        self.dns_search = dns_search
         self.docker_url = docker_url
         self.environment = environment or {}
         self.force_pull = force_pull
@@ -152,7 +171,7 @@ def __init__(
         self.xcom_push_flag = xcom_push
         self.xcom_all = xcom_all
         self.docker_conn_id = docker_conn_id
-        self.shm_size = kwargs.get('shm_size')
+        self.shm_size = shm_size
 
         self.cli = None
         self.container = None
@@ -179,18 +198,12 @@ def execute(self, context):
                 tls=tls_config
             )
 
-        if ':' not in self.image:
-            image = self.image + ':latest'
-        else:
-            image = self.image
-
-        if self.force_pull or len(self.cli.images(name=image)) == 0:
-            self.log.info('Pulling docker image %s', image)
-            for l in self.cli.pull(image, stream=True):
-                output = json.loads(l.decode('utf-8'))
-                self.log.info("%s", output['status'])
-
-        cpu_shares = int(round(self.cpus * 1024))
+        if self.force_pull or len(self.cli.images(name=self.image)) == 0:
+            self.log.info('Pulling docker image %s', self.image)
+            for l in self.cli.pull(self.image, stream=True):
+                output = json.loads(l.decode('utf-8').strip())
+                if 'status' in output:
+                    self.log.info("%s", output['status'])
 
         with TemporaryDirectory(prefix='airflowtmp') as host_tmp_dir:
             self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir
@@ -198,14 +211,17 @@ def execute(self, context):
 
             self.container = self.cli.create_container(
                 command=self.get_command(),
-                cpu_shares=cpu_shares,
                 environment=self.environment,
                 host_config=self.cli.create_host_config(
+                    auto_remove=self.auto_remove,
                     binds=self.volumes,
                     network_mode=self.network_mode,
-                    shm_size=self.shm_size),
-                image=image,
-                mem_limit=self.mem_limit,
+                    shm_size=self.shm_size,
+                    dns=self.dns,
+                    dns_search=self.dns_search,
+                    cpu_shares=int(round(self.cpus * 1024)),
+                    mem_limit=self.mem_limit),
+                image=self.image,
                 user=self.user,
                 working_dir=self.working_dir
             )
@@ -218,9 +234,9 @@ def execute(self, context):
                     line = line.decode('utf-8')
                 self.log.info(line)
 
-            exit_code = self.cli.wait(self.container['Id'])
-            if exit_code != 0:
-                raise AirflowException('docker container failed')
+            result = self.cli.wait(self.container['Id'])
+            if result['StatusCode'] != 0:
+                raise AirflowException('docker container failed: ' + repr(result))
 
             if self.xcom_push_flag:
                 return self.cli.logs(container=self.container['Id']) \
diff --git a/airflow/operators/druid_check_operator.py b/airflow/operators/druid_check_operator.py
index 73f7915ca5f40..ac18f0567d4ad 100644
--- a/airflow/operators/druid_check_operator.py
+++ b/airflow/operators/druid_check_operator.py
@@ -47,13 +47,13 @@ class DruidCheckOperator(CheckOperator):
     This operator can be used as a data quality check in your pipeline, and
     depending on where you put it in your DAG, you have the choice to
     stop the critical path, preventing from
-    publishing dubious data, or on the side and receive email alterts
+    publishing dubious data, or on the side and receive email alerts
     without stopping the progress of the DAG.
 
     :param sql: the sql to be executed
-    :type sql: string
+    :type sql: str
     :param druid_broker_conn_id: reference to the druid broker
-    :type druid_broker_conn_id: string
+    :type druid_broker_conn_id: str
     """
 
     @apply_defaults
@@ -83,9 +83,9 @@ def get_first(self, sql):
             return cur.fetchone()
 
     def execute(self, context=None):
-        self.log.info('Executing SQL check: {}'.format(self.sql))
+        self.log.info('Executing SQL check: %s', self.sql)
         record = self.get_first(self.sql)
-        self.log.info("Record: {}".format(str(record)))
+        self.log.info("Record: %s", str(record))
         if not record:
             raise AirflowException("The query returned None")
         self.log.info("Success.")
diff --git a/airflow/operators/dummy_operator.py b/airflow/operators/dummy_operator.py
index 025a242fd1b94..222c853b1d1fb 100644
--- a/airflow/operators/dummy_operator.py
+++ b/airflow/operators/dummy_operator.py
@@ -27,7 +27,6 @@ class DummyOperator(BaseOperator):
     DAG.
     """
 
-    template_fields = tuple()
     ui_color = '#e8f7e4'
 
     @apply_defaults
diff --git a/airflow/operators/email_operator.py b/airflow/operators/email_operator.py
index ae176caecff2a..0ea7fc0cc476a 100644
--- a/airflow/operators/email_operator.py
+++ b/airflow/operators/email_operator.py
@@ -29,10 +29,10 @@ class EmailOperator(BaseOperator):
     :param to: list of emails to send the email to. (templated)
     :type to: list or string (comma or semicolon delimited)
     :param subject: subject line for the email. (templated)
-    :type subject: string
+    :type subject: str
     :param html_content: content of the email, html markup
         is allowed. (templated)
-    :type html_content: string
+    :type html_content: str
     :param files: file names to attach in email
     :type files: list
     :param cc: list of recipients to be added in CC field
@@ -40,10 +40,10 @@ class EmailOperator(BaseOperator):
     :param bcc: list of recipients to be added in BCC field
     :type bcc: list or string (comma or semicolon delimited)
     :param mime_subtype: MIME sub content type
-    :type mime_subtype: string
+    :type mime_subtype: str
     :param mime_charset: character set parameter added to the Content-Type
         header.
-    :type mime_charset: string
+    :type mime_charset: str
     """
 
     template_fields = ('to', 'subject', 'html_content')
diff --git a/airflow/operators/generic_transfer.py b/airflow/operators/generic_transfer.py
index 7b1a64ee753d5..61d6692e5ff6b 100644
--- a/airflow/operators/generic_transfer.py
+++ b/airflow/operators/generic_transfer.py
@@ -41,7 +41,7 @@ class GenericTransfer(BaseOperator):
     :type destination_conn_id: str
     :param preoperator: sql statement or list of statements to be
         executed prior to loading the data. (templated)
-    :type preoperator: str or list of str
+    :type preoperator: str or list[str]
     """
 
     template_fields = ('sql', 'destination_table', 'preoperator')
diff --git a/airflow/operators/hive_operator.py b/airflow/operators/hive_operator.py
index bd727039b9f52..9cebc386c3192 100644
--- a/airflow/operators/hive_operator.py
+++ b/airflow/operators/hive_operator.py
@@ -34,9 +34,9 @@ class HiveOperator(BaseOperator):
     :param hql: the hql to be executed. Note that you may also use
         a relative path from the dag file of a (template) hive
         script. (templated)
-    :type hql: string
+    :type hql: str
     :param hive_cli_conn_id: reference to the Hive database. (templated)
-    :type hive_cli_conn_id: string
+    :type hive_cli_conn_id: str
     :param hiveconfs: if defined, these key value pairs will be passed
         to hive as ``-hiveconf "key"="value"``
     :type hiveconfs: dict
@@ -46,18 +46,18 @@ class HiveOperator(BaseOperator):
         Note that you may want to use this along with the
         ``DAG(user_defined_macros=myargs)`` parameter. View the DAG
         object documentation for more details.
-    :type hiveconf_jinja_translate: boolean
+    :type hiveconf_jinja_translate: bool
     :param script_begin_tag: If defined, the operator will get rid of the
         part of the script before the first occurrence of `script_begin_tag`
     :type script_begin_tag: str
     :param mapred_queue: queue used by the Hadoop CapacityScheduler. (templated)
-    :type  mapred_queue: string
+    :type  mapred_queue: str
     :param mapred_queue_priority: priority within CapacityScheduler queue.
         Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW
-    :type  mapred_queue_priority: string
+    :type  mapred_queue_priority: str
     :param mapred_job_name: This name will appear in the jobtracker.
         This can make monitoring easier.
-    :type  mapred_job_name: string
+    :type  mapred_job_name: str
     """
 
     template_fields = ('hql', 'schema', 'hive_cli_conn_id', 'mapred_queue',
@@ -110,7 +110,7 @@ def get_hook(self):
     def prepare_template(self):
         if self.hiveconf_jinja_translate:
             self.hql = re.sub(
-                "(\$\{(hiveconf:)?([ a-zA-Z0-9_]*)\})", "{{ \g<3> }}", self.hql)
+                r"(\$\{(hiveconf:)?([ a-zA-Z0-9_]*)\})", r"{{ \g<3> }}", self.hql)
         if self.script_begin_tag and self.script_begin_tag in self.hql:
             self.hql = "\n".join(self.hql.split(self.script_begin_tag)[1:])
 
diff --git a/airflow/operators/hive_stats_operator.py b/airflow/operators/hive_stats_operator.py
index fe83284193757..b0bb874956a92 100644
--- a/airflow/operators/hive_stats_operator.py
+++ b/airflow/operators/hive_stats_operator.py
@@ -91,8 +91,7 @@ def __init__(
     def get_default_exprs(self, col, col_type):
         if col in self.col_blacklist:
             return {}
-        d = {}
-        d[(col, 'non_null')] = "COUNT({col})"
+        d = {(col, 'non_null'): "COUNT({col})"}
         if col_type in ['double', 'int', 'bigint', 'float', 'double']:
             d[(col, 'sum')] = 'SUM({col})'
             d[(col, 'min')] = 'MIN({col})'
diff --git a/airflow/operators/hive_to_druid.py b/airflow/operators/hive_to_druid.py
index 9b4a881c5c3d4..7c2fa19c033ed 100644
--- a/airflow/operators/hive_to_druid.py
+++ b/airflow/operators/hive_to_druid.py
@@ -48,13 +48,15 @@ class HiveToDruidTransfer(BaseOperator):
     :type metastore_conn_id: str
     :param hadoop_dependency_coordinates: list of coordinates to squeeze
         int the ingest json
-    :type hadoop_dependency_coordinates: list of str
+    :type hadoop_dependency_coordinates: list[str]
     :param intervals: list of time intervals that defines segments,
         this is passed as is to the json object. (templated)
     :type intervals: list
     :param hive_tblproperties: additional properties for tblproperties in
         hive for the staging table
     :type hive_tblproperties: dict
+    :param job_properties: additional properties for job
+    :type job_properties: dict
     """
 
     template_fields = ('sql', 'intervals')
@@ -77,6 +79,7 @@ def __init__(
             query_granularity="NONE",
             segment_granularity="DAY",
             hive_tblproperties=None,
+            job_properties=None,
             *args, **kwargs):
         super(HiveToDruidTransfer, self).__init__(*args, **kwargs)
         self.sql = sql
@@ -95,6 +98,7 @@ def __init__(
         self.druid_ingest_conn_id = druid_ingest_conn_id
         self.metastore_conn_id = metastore_conn_id
         self.hive_tblproperties = hive_tblproperties
+        self.job_properties = job_properties
 
     def execute(self, context):
         hive = HiveCliHook(hive_cli_conn_id=self.hive_cli_conn_id)
@@ -162,7 +166,7 @@ def construct_ingest_query(self, static_path, columns):
         :type columns: list
         """
 
-        # backward compatibilty for num_shards,
+        # backward compatibility for num_shards,
         # but target_partition_size is the default setting
         # and overwrites the num_shards
         num_shards = self.num_shards
@@ -231,6 +235,10 @@ def construct_ingest_query(self, static_path, columns):
             }
         }
 
+        if self.job_properties:
+            ingest_query_dict['spec']['tuningConfig']['jobProperties'] \
+                .update(self.job_properties)
+
         if self.hadoop_dependency_coordinates:
             ingest_query_dict['hadoopDependencyCoordinates'] \
                 = self.hadoop_dependency_coordinates
diff --git a/airflow/operators/hive_to_mysql.py b/airflow/operators/hive_to_mysql.py
index 4dc25a6e8f8c5..882a9d854016a 100644
--- a/airflow/operators/hive_to_mysql.py
+++ b/airflow/operators/hive_to_mysql.py
@@ -17,12 +17,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from tempfile import NamedTemporaryFile
+
 from airflow.hooks.hive_hooks import HiveServer2Hook
 from airflow.hooks.mysql_hook import MySqlHook
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
-
-from tempfile import NamedTemporaryFile
+from airflow.utils.operator_helpers import context_to_airflow_vars
 
 
 class HiveToMySqlTransfer(BaseOperator):
@@ -88,7 +89,8 @@ def execute(self, context):
         if self.bulk_load:
             tmpfile = NamedTemporaryFile()
             hive.to_csv(self.sql, tmpfile.name, delimiter='\t',
-                        lineterminator='\n', output_header=False)
+                        lineterminator='\n', output_header=False,
+                        hive_conf=context_to_airflow_vars(context))
         else:
             results = hive.get_records(self.sql)
 
diff --git a/airflow/operators/hive_to_samba_operator.py b/airflow/operators/hive_to_samba_operator.py
index f6978ac30e2e6..7963524a106e3 100644
--- a/airflow/operators/hive_to_samba_operator.py
+++ b/airflow/operators/hive_to_samba_operator.py
@@ -23,6 +23,7 @@
 from airflow.hooks.samba_hook import SambaHook
 from airflow.models import BaseOperator
 from airflow.utils.decorators import apply_defaults
+from airflow.utils.operator_helpers import context_to_airflow_vars
 
 
 class Hive2SambaOperator(BaseOperator):
@@ -31,11 +32,11 @@ class Hive2SambaOperator(BaseOperator):
     results of the query as a csv to a Samba location.
 
     :param hql: the hql to be exported. (templated)
-    :type hql: string
+    :type hql: str
     :param hiveserver2_conn_id: reference to the hiveserver2 service
-    :type hiveserver2_conn_id: string
+    :type hiveserver2_conn_id: str
     :param samba_conn_id: reference to the samba destination
-    :type samba_conn_id: string
+    :type samba_conn_id: str
     """
 
     template_fields = ('hql', 'destination_filepath')
@@ -60,6 +61,7 @@ def execute(self, context):
         hive = HiveServer2Hook(hiveserver2_conn_id=self.hiveserver2_conn_id)
         tmpfile = tempfile.NamedTemporaryFile()
         self.log.info("Fetching file from Hive")
-        hive.to_csv(hql=self.hql, csv_filepath=tmpfile.name)
+        hive.to_csv(hql=self.hql, csv_filepath=tmpfile.name,
+                    hive_conf=context_to_airflow_vars(context))
         self.log.info("Pushing to samba")
         samba.push_from_local(self.destination_filepath, tmpfile.name)
diff --git a/airflow/operators/http_operator.py b/airflow/operators/http_operator.py
index 2cfc9c0057c7c..d382b012ab090 100644
--- a/airflow/operators/http_operator.py
+++ b/airflow/operators/http_operator.py
@@ -27,12 +27,12 @@ class SimpleHttpOperator(BaseOperator):
     """
     Calls an endpoint on an HTTP system to execute an action
 
-    :param http_conn_id: The connection to run the sensor against
-    :type http_conn_id: string
+    :param http_conn_id: The connection to run the operator against
+    :type http_conn_id: str
     :param endpoint: The relative part of the full url. (templated)
-    :type endpoint: string
+    :type endpoint: str
     :param method: The HTTP method to use, default = "POST"
-    :type method: string
+    :type method: str
     :param data: The data to pass. POST-data in POST/PUT and params
         in the URL for a GET request. (templated)
     :type data: For POST/PUT, depends on the content-type parameter,
@@ -46,6 +46,12 @@ class SimpleHttpOperator(BaseOperator):
         'requests' documentation (options to modify timeout, ssl, etc.)
     :type extra_options: A dictionary of options, where key is string and value
         depends on the option that's being modified.
+    :param xcom_push: Push the response to Xcom (default: False).
+        If xcom_push is True, response of an HTTP request will also
+        be pushed to an XCom.
+    :type xcom_push: bool
+    :param log_response: Log the response (default: False)
+    :type log_response: bool
     """
 
     template_fields = ('endpoint', 'data',)
@@ -61,11 +67,9 @@ def __init__(self,
                  response_check=None,
                  extra_options=None,
                  xcom_push=False,
-                 http_conn_id='http_default', *args, **kwargs):
-        """
-        If xcom_push is True, response of an HTTP request will also
-        be pushed to an XCom.
-        """
+                 http_conn_id='http_default',
+                 log_response=False,
+                 *args, **kwargs):
         super(SimpleHttpOperator, self).__init__(*args, **kwargs)
         self.http_conn_id = http_conn_id
         self.method = method
@@ -75,6 +79,7 @@ def __init__(self,
         self.response_check = response_check
         self.extra_options = extra_options or {}
         self.xcom_push_flag = xcom_push
+        self.log_response = log_response
 
     def execute(self, context):
         http = HttpHook(self.method, http_conn_id=self.http_conn_id)
@@ -85,6 +90,8 @@ def execute(self, context):
                             self.data,
                             self.headers,
                             self.extra_options)
+        if self.log_response:
+            self.log.info(response.text)
         if self.response_check:
             if not self.response_check(response):
                 raise AirflowException("Response check returned False.")
diff --git a/airflow/operators/jdbc_operator.py b/airflow/operators/jdbc_operator.py
index 9e7f24dbf6fa3..1c9423d5abd8b 100644
--- a/airflow/operators/jdbc_operator.py
+++ b/airflow/operators/jdbc_operator.py
@@ -28,12 +28,17 @@ class JdbcOperator(BaseOperator):
 
     Requires jaydebeapi.
 
-    :param jdbc_conn_id: reference to a predefined database
-    :type jdbc_conn_id: string
     :param sql: the sql code to be executed. (templated)
     :type sql: Can receive a str representing a sql statement,
         a list of str (sql statements), or reference to a template file.
         Template reference are recognized by str ending in '.sql'
+    :param jdbc_conn_id: reference to a predefined database
+    :type jdbc_conn_id: str
+    :param autocommit: if True, each command is automatically committed.
+        (default value: False)
+    :type autocommit: bool
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
     """
 
     template_fields = ('sql',)
diff --git a/airflow/operators/latest_only_operator.py b/airflow/operators/latest_only_operator.py
index 113977491cc4b..6fee50965d42a 100644
--- a/airflow/operators/latest_only_operator.py
+++ b/airflow/operators/latest_only_operator.py
@@ -17,7 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from airflow.models import BaseOperator, SkipMixin
+from airflow.models import BaseOperator
+from airflow.models.skipmixin import SkipMixin
 from airflow.utils import timezone
 
 
diff --git a/airflow/operators/mssql_operator.py b/airflow/operators/mssql_operator.py
index 1309be94e4520..086d1e39a9d28 100644
--- a/airflow/operators/mssql_operator.py
+++ b/airflow/operators/mssql_operator.py
@@ -26,13 +26,18 @@ class MsSqlOperator(BaseOperator):
     """
     Executes sql code in a specific Microsoft SQL database
 
-    :param mssql_conn_id: reference to a specific mssql database
-    :type mssql_conn_id: string
     :param sql: the sql code to be executed
-    :type sql: string or string pointing to a template file with .sql
+    :type sql: str or string pointing to a template file with .sql
         extension. (templated)
+    :param mssql_conn_id: reference to a specific mssql database
+    :type mssql_conn_id: str
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
+    :param autocommit: if True, each command is automatically committed.
+        (default value: False)
+    :type autocommit: bool
     :param database: name of database which overwrite defined one in connection
-    :type database: string
+    :type database: str
     """
 
     template_fields = ('sql',)
diff --git a/airflow/operators/mysql_operator.py b/airflow/operators/mysql_operator.py
index 2b940c785be35..905a43d64b964 100644
--- a/airflow/operators/mysql_operator.py
+++ b/airflow/operators/mysql_operator.py
@@ -26,14 +26,20 @@ class MySqlOperator(BaseOperator):
     """
     Executes sql code in a specific MySQL database
 
-    :param mysql_conn_id: reference to a specific mysql database
-    :type mysql_conn_id: string
-    :param sql: the sql code to be executed. (templated)
-    :type sql: Can receive a str representing a sql statement,
-        a list of str (sql statements), or reference to a template file.
+    :param sql: the sql code to be executed. Can receive a str representing a
+        sql statement, a list of str (sql statements), or reference to a template file.
         Template reference are recognized by str ending in '.sql'
+        (templated)
+    :type sql: str or list[str]
+    :param mysql_conn_id: reference to a specific mysql database
+    :type mysql_conn_id: str
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
+    :param autocommit: if True, each command is automatically committed.
+        (default value: False)
+    :type autocommit: bool
     :param database: name of database which overwrite defined one in connection
-    :type database: string
+    :type database: str
     """
 
     template_fields = ('sql',)
diff --git a/airflow/operators/mysql_to_hive.py b/airflow/operators/mysql_to_hive.py
index 94d660801ff82..edffe9164e577 100644
--- a/airflow/operators/mysql_to_hive.py
+++ b/airflow/operators/mysql_to_hive.py
@@ -101,6 +101,7 @@ def type_map(cls, mysql_type):
         d = {
             t.BIT: 'INT',
             t.DECIMAL: 'DOUBLE',
+            t.NEWDECIMAL: 'DOUBLE',
             t.DOUBLE: 'DOUBLE',
             t.FLOAT: 'DOUBLE',
             t.INT24: 'INT',
@@ -122,7 +123,8 @@ def execute(self, context):
         cursor = conn.cursor()
         cursor.execute(self.sql)
         with NamedTemporaryFile("wb") as f:
-            csv_writer = csv.writer(f, delimiter=self.delimiter, encoding="utf-8")
+            csv_writer = csv.writer(f, delimiter=self.delimiter,
+                                    encoding="utf-8")
             field_dict = OrderedDict()
             for field in cursor.description:
                 field_dict[field[0]] = self.type_map(field[1])
diff --git a/airflow/operators/oracle_operator.py b/airflow/operators/oracle_operator.py
index 84820c079013e..5cb62ef8a3455 100644
--- a/airflow/operators/oracle_operator.py
+++ b/airflow/operators/oracle_operator.py
@@ -25,12 +25,19 @@
 class OracleOperator(BaseOperator):
     """
     Executes sql code in a specific Oracle database
-    :param oracle_conn_id: reference to a specific Oracle database
-    :type oracle_conn_id: string
-    :param sql: the sql code to be executed. (templated)
-    :type sql: Can receive a str representing a sql statement,
+
+    :param sql: the sql code to be executed. Can receive a str representing a sql statement,
         a list of str (sql statements), or reference to a template file.
         Template reference are recognized by str ending in '.sql'
+        (templated)
+    :type sql: str or list[str]
+    :param oracle_conn_id: reference to a specific Oracle database
+    :type oracle_conn_id: str
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
+    :param autocommit: if True, each command is automatically committed.
+        (default value: False)
+    :type autocommit: bool
     """
 
     template_fields = ('sql',)
diff --git a/airflow/operators/pig_operator.py b/airflow/operators/pig_operator.py
index d22f19f70921e..3426ec522e4b6 100644
--- a/airflow/operators/pig_operator.py
+++ b/airflow/operators/pig_operator.py
@@ -29,15 +29,15 @@ class PigOperator(BaseOperator):
     Executes pig script.
 
     :param pig: the pig latin script to be executed. (templated)
-    :type pig: string
+    :type pig: str
     :param pig_cli_conn_id: reference to the Hive database
-    :type pig_cli_conn_id: string
+    :type pig_cli_conn_id: str
     :param pigparams_jinja_translate: when True, pig params-type templating
         ${var} gets translated into jinja-type templating {{ var }}. Note that
         you may want to use this along with the
         ``DAG(user_defined_macros=myargs)`` parameter. View the DAG
         object documentation for more details.
-    :type pigparams_jinja_translate: boolean
+    :type pigparams_jinja_translate: bool
     """
 
     template_fields = ('pig',)
@@ -62,7 +62,7 @@ def get_hook(self):
     def prepare_template(self):
         if self.pigparams_jinja_translate:
             self.pig = re.sub(
-                "(\$([a-zA-Z_][a-zA-Z0-9_]*))", "{{ \g<2> }}", self.pig)
+                r"(\$([a-zA-Z_][a-zA-Z0-9_]*))", r"{{ \g<2> }}", self.pig)
 
     def execute(self, context):
         self.log.info('Executing: %s', self.pig)
diff --git a/airflow/operators/postgres_operator.py b/airflow/operators/postgres_operator.py
index 5ff6e9e643278..8f2aaf553dc1c 100644
--- a/airflow/operators/postgres_operator.py
+++ b/airflow/operators/postgres_operator.py
@@ -25,14 +25,19 @@ class PostgresOperator(BaseOperator):
     """
     Executes sql code in a specific Postgres database
 
-    :param postgres_conn_id: reference to a specific postgres database
-    :type postgres_conn_id: string
     :param sql: the sql code to be executed. (templated)
     :type sql: Can receive a str representing a sql statement,
         a list of str (sql statements), or reference to a template file.
         Template reference are recognized by str ending in '.sql'
+    :param postgres_conn_id: reference to a specific postgres database
+    :type postgres_conn_id: str
+    :param autocommit: if True, each command is automatically committed.
+        (default value: False)
+    :type autocommit: bool
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
     :param database: name of database which overwrite defined one in connection
-    :type database: string
+    :type database: str
     """
 
     template_fields = ('sql',)
diff --git a/airflow/operators/presto_check_operator.py b/airflow/operators/presto_check_operator.py
index 608aebfe2966a..d70dcaa7d25af 100644
--- a/airflow/operators/presto_check_operator.py
+++ b/airflow/operators/presto_check_operator.py
@@ -48,13 +48,13 @@ class PrestoCheckOperator(CheckOperator):
     This operator can be used as a data quality check in your pipeline, and
     depending on where you put it in your DAG, you have the choice to
     stop the critical path, preventing from
-    publishing dubious data, or on the side and receive email alterts
+    publishing dubious data, or on the side and receive email alerts
     without stopping the progress of the DAG.
 
     :param sql: the sql to be executed
-    :type sql: string
+    :type sql: str
     :param presto_conn_id: reference to the Presto database
-    :type presto_conn_id: string
+    :type presto_conn_id: str
     """
 
     @apply_defaults
@@ -76,9 +76,9 @@ class PrestoValueCheckOperator(ValueCheckOperator):
     Performs a simple value check using sql code.
 
     :param sql: the sql to be executed
-    :type sql: string
+    :type sql: str
     :param presto_conn_id: reference to the Presto database
-    :type presto_conn_id: string
+    :type presto_conn_id: str
     """
 
     @apply_defaults
@@ -108,7 +108,7 @@ class PrestoIntervalCheckOperator(IntervalCheckOperator):
     :param metrics_threshold: a dictionary of ratios indexed by metrics
     :type metrics_threshold: dict
     :param presto_conn_id: reference to the Presto database
-    :type presto_conn_id: string
+    :type presto_conn_id: str
     """
 
     @apply_defaults
diff --git a/airflow/operators/python_operator.py b/airflow/operators/python_operator.py
index 88f3b1a8a6347..67cf5492656e2 100644
--- a/airflow/operators/python_operator.py
+++ b/airflow/operators/python_operator.py
@@ -17,35 +17,42 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from builtins import str
-import dill
 import inspect
 import os
 import pickle
 import subprocess
 import sys
 import types
+from builtins import str
+from textwrap import dedent
+
+import dill
+import six
 
 from airflow.exceptions import AirflowException
-from airflow.models import BaseOperator, SkipMixin
+from airflow.models import BaseOperator
+from airflow.models.skipmixin import SkipMixin
 from airflow.utils.decorators import apply_defaults
 from airflow.utils.file import TemporaryDirectory
-
-from textwrap import dedent
+from airflow.utils.operator_helpers import context_to_airflow_vars
 
 
 class PythonOperator(BaseOperator):
     """
     Executes a Python callable
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:PythonOperator`
+
     :param python_callable: A reference to an object that is callable
     :type python_callable: python callable
     :param op_kwargs: a dictionary of keyword arguments that will get unpacked
         in your function
-    :type op_kwargs: dict
+    :type op_kwargs: dict (templated)
     :param op_args: a list of positional arguments that will get unpacked when
         calling your callable
-    :type op_args: list
+    :type op_args: list (templated)
     :param provide_context: if set to true, Airflow will pass a set of
         keyword arguments that can be used in your function. This set of
         kwargs correspond exactly to what you can use in your jinja
@@ -56,15 +63,18 @@ class PythonOperator(BaseOperator):
         will get templated by the Airflow engine sometime between
         ``__init__`` and ``execute`` takes place and are made available
         in your callable's context after the template has been applied. (templated)
-    :type templates_dict: dict of str
+    :type templates_dict: dict[str]
     :param templates_exts: a list of file extensions to resolve while
         processing templated fields, for examples ``['.sql', '.hql']``
-    :type templates_exts: list(str)
+    :type templates_exts: list[str]
     """
-    template_fields = ('templates_dict',)
-    template_ext = tuple()
+    template_fields = ('templates_dict', 'op_args', 'op_kwargs')
     ui_color = '#ffefeb'
 
+    # since we won't mutate the arguments, we should just do the shallow copy
+    # there are some cases we can't deepcopy the objects(e.g protobuf).
+    shallow_copy_attrs = ('python_callable', 'op_kwargs',)
+
     @apply_defaults
     def __init__(
             self,
@@ -87,6 +97,13 @@ def __init__(
             self.template_ext = templates_exts
 
     def execute(self, context):
+        # Export context to make it available for callables to use.
+        airflow_context_vars = context_to_airflow_vars(context, in_env_var_format=True)
+        self.log.info("Exporting the following env vars:\n%s",
+                      '\n'.join(["{}={}".format(k, v)
+                                 for k, v in airflow_context_vars.items()]))
+        os.environ.update(airflow_context_vars)
+
         if self.provide_context:
             context.update(self.op_kwargs)
             context['templates_dict'] = self.templates_dict
@@ -102,14 +119,14 @@ def execute_callable(self):
 
 class BranchPythonOperator(PythonOperator, SkipMixin):
     """
-    Allows a workflow to "branch" or follow a single path following the
-    execution of this task.
+    Allows a workflow to "branch" or follow a path following the execution
+    of this task.
 
     It derives the PythonOperator and expects a Python function that returns
-    the task_id to follow. The task_id returned should point to a task
-    directly downstream from {self}. All other "branches" or
-    directly downstream tasks are marked with a state of ``skipped`` so that
-    these paths can't move forward. The ``skipped`` states are propageted
+    a single task_id or list of task_ids to follow. The task_id(s) returned
+    should point to a task directly downstream from {self}. All other "branches"
+    or directly downstream tasks are marked with a state of ``skipped`` so that
+    these paths can't move forward. The ``skipped`` states are propagated
     downstream to allow for the DAG state to fill up and the DAG run's state
     to be inferred.
 
@@ -121,14 +138,26 @@ class BranchPythonOperator(PythonOperator, SkipMixin):
     """
     def execute(self, context):
         branch = super(BranchPythonOperator, self).execute(context)
+        if isinstance(branch, six.string_types):
+            branch = [branch]
         self.log.info("Following branch %s", branch)
         self.log.info("Marking other directly downstream tasks as skipped")
 
         downstream_tasks = context['task'].downstream_list
         self.log.debug("Downstream task_ids %s", downstream_tasks)
 
-        skip_tasks = [t for t in downstream_tasks if t.task_id != branch]
         if downstream_tasks:
+            # Also check downstream tasks of the branch task. In case the task to skip
+            # is a downstream task of the branch task, we exclude it from skipping.
+            branch_downstream_task_ids = set()
+            for b in branch:
+                branch_downstream_task_ids.update(context["dag"].
+                                                  get_task(b).
+                                                  get_flat_relative_ids(upstream=False))
+            skip_tasks = [t
+                          for t in downstream_tasks
+                          if t.task_id not in branch and
+                          t.task_id not in branch_downstream_task_ids]
             self.skip(context['dag_run'], context['ti'].execution_date, skip_tasks)
 
         self.log.info("Done.")
@@ -176,7 +205,6 @@ class PythonVirtualenvOperator(PythonOperator):
     variable named virtualenv_string_args will be available (populated by
     string_args). In addition, one can pass stuff through op_args and op_kwargs, and one
     can use a return value.
-
     Note that if your virtualenv runs in a different Python major version than Airflow,
     you cannot use return values, op_args, or op_kwargs. You can use string_args though.
 
@@ -184,7 +212,7 @@ class PythonVirtualenvOperator(PythonOperator):
         defined with def, which will be run in a virtualenv
     :type python_callable: function
     :param requirements: A list of requirements as specified in a pip install command
-    :type requirements: list(str)
+    :type requirements: list[str]
     :param python_version: The Python version to run the virtualenv with. Note that
         both 2 and 2.7 are acceptable forms.
     :type python_version: str
@@ -200,10 +228,16 @@ class PythonVirtualenvOperator(PythonOperator):
     :type op_kwargs: list
     :param op_kwargs: A dict of keyword arguments to pass to python_callable.
     :type op_kwargs: dict
+    :param provide_context: if set to true, Airflow will pass a set of
+        keyword arguments that can be used in your function. This set of
+        kwargs correspond exactly to what you can use in your jinja
+        templates. For this to work, you need to define `**kwargs` in your
+        function header.
+    :type provide_context: bool
     :param string_args: Strings that are present in the global var virtualenv_string_args,
-        available to python_callable at runtime as a list(str). Note that args are split
+        available to python_callable at runtime as a list[str]. Note that args are split
         by newline.
-    :type string_args: list(str)
+    :type string_args: list[str]
     :param templates_dict: a dictionary where the values are templates that
         will get templated by the Airflow engine sometime between
         ``__init__`` and ``execute`` takes place and are made available
@@ -211,21 +245,23 @@ class PythonVirtualenvOperator(PythonOperator):
     :type templates_dict: dict of str
     :param templates_exts: a list of file extensions to resolve while
         processing templated fields, for examples ``['.sql', '.hql']``
-    :type templates_exts: list(str)
+    :type templates_exts: list[str]
     """
+    @apply_defaults
     def __init__(self, python_callable,
                  requirements=None,
                  python_version=None, use_dill=False,
                  system_site_packages=True,
-                 op_args=None, op_kwargs=None, string_args=None,
-                 templates_dict=None, templates_exts=None, *args, **kwargs):
+                 op_args=None, op_kwargs=None, provide_context=False,
+                 string_args=None, templates_dict=None, templates_exts=None,
+                 *args, **kwargs):
         super(PythonVirtualenvOperator, self).__init__(
             python_callable=python_callable,
             op_args=op_args,
             op_kwargs=op_kwargs,
             templates_dict=templates_dict,
             templates_exts=templates_exts,
-            provide_context=False,
+            provide_context=provide_context,
             *args,
             **kwargs)
         self.requirements = requirements or []
@@ -289,14 +325,14 @@ def _pass_op_args(self):
 
     def _execute_in_subprocess(self, cmd):
         try:
-            self.log.info("Executing cmd\n{}".format(cmd))
+            self.log.info("Executing cmd\n%s", cmd)
             output = subprocess.check_output(cmd,
                                              stderr=subprocess.STDOUT,
                                              close_fds=True)
             if output:
-                self.log.info("Got output\n{}".format(output))
+                self.log.info("Got output\n%s", output)
         except subprocess.CalledProcessError as e:
-            self.log.info("Got error output\n{}".format(e.output))
+            self.log.info("Got error output\n%s", e.output)
             raise
 
     def _write_string_args(self, filename):
@@ -351,7 +387,8 @@ def _generate_pip_install_cmd(self, tmp_dir):
             cmd = ['{}/bin/pip'.format(tmp_dir), 'install']
             return cmd + self.requirements
 
-    def _generate_python_cmd(self, tmp_dir, script_filename,
+    @staticmethod
+    def _generate_python_cmd(tmp_dir, script_filename,
                              input_filename, output_filename, string_args_filename):
         # direct path alleviates need to activate
         return ['{}/bin/python'.format(tmp_dir), script_filename,
diff --git a/airflow/operators/redshift_to_s3_operator.py b/airflow/operators/redshift_to_s3_operator.py
index 9c1b621dae965..c0c4db3503467 100644
--- a/airflow/operators/redshift_to_s3_operator.py
+++ b/airflow/operators/redshift_to_s3_operator.py
@@ -28,17 +28,28 @@ class RedshiftToS3Transfer(BaseOperator):
     Executes an UNLOAD command to s3 as a CSV with headers
 
     :param schema: reference to a specific schema in redshift database
-    :type schema: string
+    :type schema: str
     :param table: reference to a specific table in redshift database
-    :type table: string
+    :type table: str
     :param s3_bucket: reference to a specific S3 bucket
-    :type s3_bucket: string
+    :type s3_bucket: str
     :param s3_key: reference to a specific S3 key
-    :type s3_key: string
+    :type s3_key: str
     :param redshift_conn_id: reference to a specific redshift database
-    :type redshift_conn_id: string
+    :type redshift_conn_id: str
     :param aws_conn_id: reference to a specific S3 connection
-    :type aws_conn_id: string
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
     :param unload_options: reference to a list of UNLOAD options
     :type unload_options: list
     """
@@ -56,9 +67,9 @@ def __init__(
             s3_key,
             redshift_conn_id='redshift_default',
             aws_conn_id='aws_default',
+            verify=None,
             unload_options=tuple(),
             autocommit=False,
-            parameters=None,
             include_header=False,
             *args, **kwargs):
         super(RedshiftToS3Transfer, self).__init__(*args, **kwargs)
@@ -68,9 +79,9 @@ def __init__(
         self.s3_key = s3_key
         self.redshift_conn_id = redshift_conn_id
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
         self.unload_options = unload_options
         self.autocommit = autocommit
-        self.parameters = parameters
         self.include_header = include_header
 
         if self.include_header and \
@@ -79,7 +90,7 @@ def __init__(
 
     def execute(self, context):
         self.hook = PostgresHook(postgres_conn_id=self.redshift_conn_id)
-        self.s3 = S3Hook(aws_conn_id=self.aws_conn_id)
+        self.s3 = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
         credentials = self.s3.get_credentials()
         unload_options = '\n\t\t\t'.join(self.unload_options)
 
diff --git a/airflow/operators/s3_file_transform_operator.py b/airflow/operators/s3_file_transform_operator.py
index 84a6eda0c8101..74c351db86356 100644
--- a/airflow/operators/s3_file_transform_operator.py
+++ b/airflow/operators/s3_file_transform_operator.py
@@ -19,6 +19,7 @@
 
 from tempfile import NamedTemporaryFile
 import subprocess
+import sys
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.S3_hook import S3Hook
@@ -47,6 +48,19 @@ class S3FileTransformOperator(BaseOperator):
     :type source_s3_key: str
     :param source_aws_conn_id: source s3 connection
     :type source_aws_conn_id: str
+    :param source_verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+             (unless use_ssl is False), but SSL certificates will not be
+             verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+             You can specify this argument if you want to use a different
+             CA cert bundle than the one used by botocore.
+
+        This is also applicable to ``dest_verify``.
+    :type source_verify: bool or str
     :param dest_s3_key: The key to be written from S3. (templated)
     :type dest_s3_key: str
     :param dest_aws_conn_id: destination s3 connection
@@ -71,25 +85,32 @@ def __init__(
             transform_script=None,
             select_expression=None,
             source_aws_conn_id='aws_default',
+            source_verify=None,
             dest_aws_conn_id='aws_default',
+            dest_verify=None,
             replace=False,
             *args, **kwargs):
         super(S3FileTransformOperator, self).__init__(*args, **kwargs)
         self.source_s3_key = source_s3_key
         self.source_aws_conn_id = source_aws_conn_id
+        self.source_verify = source_verify
         self.dest_s3_key = dest_s3_key
         self.dest_aws_conn_id = dest_aws_conn_id
+        self.dest_verify = dest_verify
         self.replace = replace
         self.transform_script = transform_script
         self.select_expression = select_expression
+        self.output_encoding = sys.getdefaultencoding()
 
     def execute(self, context):
         if self.transform_script is None and self.select_expression is None:
             raise AirflowException(
                 "Either transform_script or select_expression must be specified")
 
-        source_s3 = S3Hook(aws_conn_id=self.source_aws_conn_id)
-        dest_s3 = S3Hook(aws_conn_id=self.dest_aws_conn_id)
+        source_s3 = S3Hook(aws_conn_id=self.source_aws_conn_id,
+                           verify=self.source_verify)
+        dest_s3 = S3Hook(aws_conn_id=self.dest_aws_conn_id,
+                         verify=self.dest_verify)
 
         self.log.info("Downloading source S3 file %s", self.source_s3_key)
         if not source_s3.check_for_key(self.source_s3_key):
@@ -114,15 +135,23 @@ def execute(self, context):
             f_source.flush()
 
             if self.transform_script is not None:
-                transform_script_process = subprocess.Popen(
+                process = subprocess.Popen(
                     [self.transform_script, f_source.name, f_dest.name],
-                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
-                (transform_script_stdoutdata, transform_script_stderrdata) = \
-                    transform_script_process.communicate()
-                self.log.info("Transform script stdout %s", transform_script_stdoutdata)
-                if transform_script_process.returncode > 0:
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    close_fds=True
+                )
+
+                self.log.info("Output:")
+                for line in iter(process.stdout.readline, b''):
+                    self.log.info(line.decode(self.output_encoding).rstrip())
+
+                process.wait()
+
+                if process.returncode > 0:
                     raise AirflowException(
-                        "Transform script failed %s", transform_script_stderrdata)
+                        "Transform script failed: {0}".format(process.returncode)
+                    )
                 else:
                     self.log.info(
                         "Transform script successful. Output temporarily located at %s",
diff --git a/airflow/operators/s3_to_hive_operator.py b/airflow/operators/s3_to_hive_operator.py
index 09eb8363c0dc5..33aa5b0103627 100644
--- a/airflow/operators/s3_to_hive_operator.py
+++ b/airflow/operators/s3_to_hive_operator.py
@@ -78,6 +78,17 @@ class S3ToHiveTransfer(BaseOperator):
     :type delimiter: str
     :param aws_conn_id: source s3 connection
     :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
     :param hive_cli_conn_id: destination hive connection
     :type hive_cli_conn_id: str
     :param input_compressed: Boolean to determine if file decompression is
@@ -107,6 +118,7 @@ def __init__(
             check_headers=False,
             wildcard_match=False,
             aws_conn_id='aws_default',
+            verify=None,
             hive_cli_conn_id='hive_cli_default',
             input_compressed=False,
             tblproperties=None,
@@ -125,6 +137,7 @@ def __init__(
         self.wildcard_match = wildcard_match
         self.hive_cli_conn_id = hive_cli_conn_id
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
         self.input_compressed = input_compressed
         self.tblproperties = tblproperties
         self.select_expression = select_expression
@@ -136,7 +149,7 @@ def __init__(
 
     def execute(self, context):
         # Downloading file from S3
-        self.s3 = S3Hook(aws_conn_id=self.aws_conn_id)
+        self.s3 = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
         self.hive = HiveCliHook(hive_cli_conn_id=self.hive_cli_conn_id)
         self.log.info("Downloading S3 file")
 
@@ -153,7 +166,7 @@ def execute(self, context):
 
         root, file_ext = os.path.splitext(s3_key_object.key)
         if (self.select_expression and self.input_compressed and
-                file_ext != '.gz'):
+                file_ext.lower() != '.gz'):
             raise AirflowException("GZIP is the only compression " +
                                    "format Amazon S3 Select supports")
 
@@ -161,8 +174,9 @@ def execute(self, context):
                 NamedTemporaryFile(mode="wb",
                                    dir=tmp_dir,
                                    suffix=file_ext) as f:
-            self.log.info("Dumping S3 key {0} contents to local file {1}"
-                          .format(s3_key_object.key, f.name))
+            self.log.info(
+                "Dumping S3 key %s contents to local file %s", s3_key_object.key, f.name
+            )
             if self.select_expression:
                 option = {}
                 if self.headers:
@@ -245,24 +259,23 @@ def _match_headers(self, header_list):
             raise AirflowException("Unable to retrieve header row from file")
         field_names = self.field_dict.keys()
         if len(field_names) != len(header_list):
-            self.log.warning("Headers count mismatch"
-                             "File headers:\n {header_list}\n"
-                             "Field names: \n {field_names}\n"
-                             .format(**locals()))
+            self.log.warning(
+                "Headers count mismatch File headers:\n %s\nField names: \n %s\n", header_list, field_names
+            )
             return False
         test_field_match = [h1.lower() == h2.lower()
                             for h1, h2 in zip(header_list, field_names)]
         if not all(test_field_match):
-            self.log.warning("Headers do not match field names"
-                             "File headers:\n {header_list}\n"
-                             "Field names: \n {field_names}\n"
-                             .format(**locals()))
+            self.log.warning(
+                "Headers do not match field names File headers:\n %s\nField names: \n %s\n",
+                header_list, field_names
+            )
             return False
         else:
             return True
 
+    @staticmethod
     def _delete_top_row_and_compress(
-            self,
             input_file_name,
             output_file_ext,
             dest_dir):
@@ -275,7 +288,7 @@ def _delete_top_row_and_compress(
 
         os_fh_output, fn_output = \
             tempfile.mkstemp(suffix=output_file_ext, dir=dest_dir)
-        with open(input_file_name, 'rb') as f_in,\
+        with open(input_file_name, 'rb') as f_in, \
                 open_fn(fn_output, 'wb') as f_out:
             f_in.seek(0)
             next(f_in)
diff --git a/airflow/operators/s3_to_redshift_operator.py b/airflow/operators/s3_to_redshift_operator.py
index 0d7921e9ed0f1..65bc679f243ab 100644
--- a/airflow/operators/s3_to_redshift_operator.py
+++ b/airflow/operators/s3_to_redshift_operator.py
@@ -28,17 +28,28 @@ class S3ToRedshiftTransfer(BaseOperator):
     Executes an COPY command to load files from s3 to Redshift
 
     :param schema: reference to a specific schema in redshift database
-    :type schema: string
+    :type schema: str
     :param table: reference to a specific table in redshift database
-    :type table: string
+    :type table: str
     :param s3_bucket: reference to a specific S3 bucket
-    :type s3_bucket: string
+    :type s3_bucket: str
     :param s3_key: reference to a specific S3 key
-    :type s3_key: string
+    :type s3_key: str
     :param redshift_conn_id: reference to a specific redshift database
-    :type redshift_conn_id: string
+    :type redshift_conn_id: str
     :param aws_conn_id: reference to a specific S3 connection
-    :type aws_conn_id: string
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
     :param copy_options: reference to a list of COPY options
     :type copy_options: list
     """
@@ -56,6 +67,7 @@ def __init__(
             s3_key,
             redshift_conn_id='redshift_default',
             aws_conn_id='aws_default',
+            verify=None,
             copy_options=tuple(),
             autocommit=False,
             parameters=None,
@@ -67,13 +79,14 @@ def __init__(
         self.s3_key = s3_key
         self.redshift_conn_id = redshift_conn_id
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
         self.copy_options = copy_options
         self.autocommit = autocommit
         self.parameters = parameters
 
     def execute(self, context):
         self.hook = PostgresHook(postgres_conn_id=self.redshift_conn_id)
-        self.s3 = S3Hook(aws_conn_id=self.aws_conn_id)
+        self.s3 = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
         credentials = self.s3.get_credentials()
         copy_options = '\n\t\t\t'.join(self.copy_options)
 
diff --git a/airflow/operators/slack_operator.py b/airflow/operators/slack_operator.py
index c5a69456fbe86..b6fb75a2360e3 100644
--- a/airflow/operators/slack_operator.py
+++ b/airflow/operators/slack_operator.py
@@ -32,11 +32,11 @@ class SlackAPIOperator(BaseOperator):
     In the future additional Slack API Operators will be derived from this class as well
 
     :param slack_conn_id: Slack connection ID which its password is Slack API token
-    :type slack_conn_id: string
+    :type slack_conn_id: str
     :param token: Slack API token (https://api.slack.com/web)
-    :type token: string
+    :type token: str
     :param method: The Slack API Method to Call (https://api.slack.com/methods)
-    :type method: string
+    :type method: str
     :param api_params: API Method call parameters (https://api.slack.com/methods)
     :type api_params: dict
     """
@@ -93,13 +93,13 @@ class SlackAPIPostOperator(SlackAPIOperator):
 
     :param channel: channel in which to post message on slack name (#general) or
         ID (C12318391). (templated)
-    :type channel: string
+    :type channel: str
     :param username: Username that airflow will be posting to Slack as. (templated)
-    :type username: string
+    :type username: str
     :param text: message to send to slack. (templated)
-    :type text: string
+    :type text: str
     :param icon_url: url to icon used for this message
-    :type icon_url: string
+    :type icon_url: str
     :param attachments: extra formatting details. (templated)
         - see https://api.slack.com/docs/attachments.
     :type attachments: array of hashes
@@ -115,8 +115,8 @@ def __init__(self,
                  text='No message has been set.\n'
                       'Here is a cat video instead\n'
                       'https://www.youtube.com/watch?v=J---aiyznGQ',
-                 icon_url='https://raw.githubusercontent.com'
-                          '/airbnb/airflow/master/airflow/www/static/pin_100.png',
+                 icon_url='https://raw.githubusercontent.com/apache/'
+                          'airflow/master/airflow/www/static/pin_100.jpg',
                  attachments=None,
                  *args, **kwargs):
         self.method = 'chat.postMessage'
diff --git a/airflow/operators/sqlite_operator.py b/airflow/operators/sqlite_operator.py
index 5b7213ed816d8..fb4a30e350250 100644
--- a/airflow/operators/sqlite_operator.py
+++ b/airflow/operators/sqlite_operator.py
@@ -26,11 +26,13 @@ class SqliteOperator(BaseOperator):
     """
     Executes sql code in a specific Sqlite database
 
-    :param sqlite_conn_id: reference to a specific sqlite database
-    :type sqlite_conn_id: string
     :param sql: the sql code to be executed. (templated)
-    :type sql: string or string pointing to a template file. File must have
+    :type sql: str or string pointing to a template file. File must have
         a '.sql' extensions.
+    :param sqlite_conn_id: reference to a specific sqlite database
+    :type sqlite_conn_id: str
+    :param parameters: (optional) the parameters to render the SQL query with.
+    :type parameters: mapping or iterable
     """
 
     template_fields = ('sql',)
diff --git a/airflow/operators/subdag_operator.py b/airflow/operators/subdag_operator.py
index 052095e2a618f..812f486e16428 100644
--- a/airflow/operators/subdag_operator.py
+++ b/airflow/operators/subdag_operator.py
@@ -25,8 +25,19 @@
 
 
 class SubDagOperator(BaseOperator):
+    """
+    This runs a sub dag. By convention, a sub dag's dag_id
+    should be prefixed by its parent and a dot. As in `parent.child`.
+
+    :param subdag: the DAG object to run as a subdag of the current DAG.
+    :type subdag: airflow.models.DAG
+    :param dag: the parent DAG for the subdag.
+    :type dag: airflow.models.DAG
+    :param executor: the executor for this subdag. Default to use SequentialExecutor.
+        Please find AIRFLOW-74 for more details.
+    :type executor: airflow.executors.base_executor.BaseExecutor
+    """
 
-    template_fields = tuple()
     ui_color = '#555'
     ui_fgcolor = '#fff'
 
@@ -37,18 +48,6 @@ def __init__(
             subdag,
             executor=SequentialExecutor(),
             *args, **kwargs):
-        """
-        This runs a sub dag. By convention, a sub dag's dag_id
-        should be prefixed by its parent and a dot. As in `parent.child`.
-
-        :param subdag: the DAG object to run as a subdag of the current DAG.
-        :type subdag: airflow.DAG.
-        :param dag: the parent DAG for the subdag.
-        :type dag: airflow.DAG.
-        :param executor: the executor for this subdag. Default to use SequentialExecutor.
-                         Please find AIRFLOW-74 for more details.
-        :type executor: airflow.executors.
-        """
         import airflow.models
         dag = kwargs.get('dag') or airflow.models._CONTEXT_MANAGER_DAG
         if not dag:
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 735f2de1e8c79..4277b33432077 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -27,48 +27,99 @@
 import inspect
 import os
 import re
-import sys
+import pkg_resources
+from typing import List, Any
 
-from airflow import configuration
+from airflow import settings
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 log = LoggingMixin().log
 
+import_errors = {}
+
+
 class AirflowPluginException(Exception):
     pass
 
 
 class AirflowPlugin(object):
-    name = None
-    operators = []
-    sensors = []
-    hooks = []
-    executors = []
-    macros = []
-    admin_views = []
-    flask_blueprints = []
-    menu_links = []
+    name = None  # type: str
+    operators = []  # type: List[Any]
+    sensors = []  # type: List[Any]
+    hooks = []  # type: List[Any]
+    executors = []  # type: List[Any]
+    macros = []  # type: List[Any]
+    admin_views = []  # type: List[Any]
+    flask_blueprints = []  # type: List[Any]
+    menu_links = []  # type: List[Any]
+    appbuilder_views = []  # type: List[Any]
+    appbuilder_menu_items = []  # type: List[Any]
 
     @classmethod
     def validate(cls):
         if not cls.name:
             raise AirflowPluginException("Your plugin needs a name.")
 
-
-plugins_folder = configuration.conf.get('core', 'plugins_folder')
-if not plugins_folder:
-    plugins_folder = configuration.conf.get('core', 'airflow_home') + '/plugins'
-plugins_folder = os.path.expanduser(plugins_folder)
-
-if plugins_folder not in sys.path:
-    sys.path.append(plugins_folder)
-
-plugins = []
+    @classmethod
+    def on_load(cls, *args, **kwargs):
+        """
+        Executed when the plugin is loaded.
+        This method is only called once during runtime.
+
+        :param args: If future arguments are passed in on call.
+        :param kwargs: If future arguments are passed in on call.
+        """
+        pass
+
+
+def load_entrypoint_plugins(entry_points, airflow_plugins):
+    """
+    Load AirflowPlugin subclasses from the entrypoints
+    provided. The entry_point group should be 'airflow.plugins'.
+
+    :param entry_points: A collection of entrypoints to search for plugins
+    :type entry_points: Generator[setuptools.EntryPoint, None, None]
+    :param airflow_plugins: A collection of existing airflow plugins to
+        ensure we don't load duplicates
+    :type airflow_plugins: list[type[airflow.plugins_manager.AirflowPlugin]]
+    :rtype: list[airflow.plugins_manager.AirflowPlugin]
+    """
+    for entry_point in entry_points:
+        log.debug('Importing entry_point plugin %s', entry_point.name)
+        plugin_obj = entry_point.load()
+        if is_valid_plugin(plugin_obj, airflow_plugins):
+            if callable(getattr(plugin_obj, 'on_load', None)):
+                plugin_obj.on_load()
+                airflow_plugins.append(plugin_obj)
+    return airflow_plugins
+
+
+def is_valid_plugin(plugin_obj, existing_plugins):
+    """
+    Check whether a potential object is a subclass of
+    the AirflowPlugin class.
+
+    :param plugin_obj: potential subclass of AirflowPlugin
+    :param existing_plugins: Existing list of AirflowPlugin subclasses
+    :return: Whether or not the obj is a valid subclass of
+        AirflowPlugin
+    """
+    if (
+        inspect.isclass(plugin_obj) and
+        issubclass(plugin_obj, AirflowPlugin) and
+        (plugin_obj is not AirflowPlugin)
+    ):
+        plugin_obj.validate()
+        return plugin_obj not in existing_plugins
+    return False
+
+
+plugins = []  # type: List[AirflowPlugin]
 
 norm_pattern = re.compile(r'[/|.]')
 
 # Crawl through the plugins folder to find AirflowPlugin derivatives
-for root, dirs, files in os.walk(plugins_folder, followlinks=True):
+for root, dirs, files in os.walk(settings.PLUGINS_FOLDER, followlinks=True):
     for f in files:
         try:
             filepath = os.path.join(root, f)
@@ -85,17 +136,18 @@ def validate(cls):
 
             m = imp.load_source(namespace, filepath)
             for obj in list(m.__dict__.values()):
-                if (
-                        inspect.isclass(obj) and
-                        issubclass(obj, AirflowPlugin) and
-                        obj is not AirflowPlugin):
-                    obj.validate()
-                    if obj not in plugins:
-                        plugins.append(obj)
+                if is_valid_plugin(obj, plugins):
+                    plugins.append(obj)
 
         except Exception as e:
             log.exception(e)
             log.error('Failed to import plugin %s', filepath)
+            import_errors[filepath] = str(e)
+
+plugins = load_entrypoint_plugins(
+    pkg_resources.iter_entry_points('airflow.plugins'),
+    plugins
+)
 
 
 def make_module(name, objects):
@@ -107,6 +159,7 @@ def make_module(name, objects):
     module.__dict__.update((o.__name__, o) for o in objects)
     return module
 
+
 # Plugin components to integrate as modules
 operators_modules = []
 sensors_modules = []
@@ -115,9 +168,11 @@ def make_module(name, objects):
 macros_modules = []
 
 # Plugin components to integrate directly
-admin_views = []
-flask_blueprints = []
-menu_links = []
+admin_views = []  # type: List[Any]
+flask_blueprints = []  # type: List[Any]
+menu_links = []  # type: List[Any]
+flask_appbuilder_views = []  # type: List[Any]
+flask_appbuilder_menu_links = []  # type: List[Any]
 
 for p in plugins:
     operators_modules.append(
@@ -131,5 +186,10 @@ def make_module(name, objects):
     macros_modules.append(make_module('airflow.macros.' + p.name, p.macros))
 
     admin_views.extend(p.admin_views)
-    flask_blueprints.extend(p.flask_blueprints)
     menu_links.extend(p.menu_links)
+    flask_appbuilder_views.extend(p.appbuilder_views)
+    flask_appbuilder_menu_links.extend(p.appbuilder_menu_items)
+    flask_blueprints.extend([{
+        'name': p.name,
+        'blueprint': bp
+    } for bp in p.flask_blueprints])
diff --git a/airflow/security/kerberos.py b/airflow/security/kerberos.py
index 43c9fcccdca85..3fc6aa345d79f 100644
--- a/airflow/security/kerberos.py
+++ b/airflow/security/kerberos.py
@@ -27,11 +27,13 @@
 log = LoggingMixin().log
 
 
-def renew_from_kt():
+def renew_from_kt(principal, keytab):
     # The config is specified in seconds. But we ask for that same amount in
     # minutes to give ourselves a large renewal buffer.
+
     renewal_lifetime = "%sm" % configuration.conf.getint('kerberos', 'reinit_frequency')
-    principal = configuration.conf.get('kerberos', 'principal').replace(
+
+    cmd_principal = principal or configuration.conf.get('kerberos', 'principal').replace(
         "_HOST", socket.getfqdn()
     )
 
@@ -39,11 +41,11 @@ def renew_from_kt():
         configuration.conf.get('kerberos', 'kinit_path'),
         "-r", renewal_lifetime,
         "-k",  # host ticket
-        "-t", configuration.conf.get('kerberos', 'keytab'),  # specify keytab
+        "-t", keytab,  # specify keytab
         "-c", configuration.conf.get('kerberos', 'ccache'),  # specify credentials cache
-        principal
+        cmd_principal
     ]
-    log.info("Reinitting kerberos from keytab: " + " ".join(cmdv))
+    log.info("Reinitting kerberos from keytab: %s", " ".join(cmdv))
 
     subp = subprocess.Popen(cmdv,
                             stdout=subprocess.PIPE,
@@ -53,10 +55,10 @@ def renew_from_kt():
                             universal_newlines=True)
     subp.wait()
     if subp.returncode != 0:
-        log.error("Couldn't reinit from keytab! `kinit' exited with %s.\n%s\n%s" % (
-            subp.returncode,
-            b"\n".join(subp.stdout.readlines()),
-            b"\n".join(subp.stderr.readlines())))
+        log.error(
+            "Couldn't reinit from keytab! `kinit' exited with %s.\n%s\n%s",
+            subp.returncode, "\n".join(subp.stdout.readlines()), "\n".join(subp.stderr.readlines())
+        )
         sys.exit(subp.returncode)
 
     global NEED_KRB181_WORKAROUND
@@ -66,34 +68,32 @@ def renew_from_kt():
         # (From: HUE-640). Kerberos clock have seconds level granularity. Make sure we
         # renew the ticket after the initial valid time.
         time.sleep(1.5)
-        perform_krb181_workaround()
+        perform_krb181_workaround(principal)
 
 
-def perform_krb181_workaround():
+def perform_krb181_workaround(principal):
     cmdv = [configuration.conf.get('kerberos', 'kinit_path'),
             "-c", configuration.conf.get('kerberos', 'ccache'),
             "-R"]  # Renew ticket_cache
 
-    log.info("Renewing kerberos ticket to work around kerberos 1.8.1: " +
-             " ".join(cmdv))
+    log.info(
+        "Renewing kerberos ticket to work around kerberos 1.8.1: %s", " ".join(cmdv)
+    )
 
     ret = subprocess.call(cmdv, close_fds=True)
 
     if ret != 0:
-        principal = "%s/%s" % (
-            configuration.conf.get('kerberos', 'principal'),
-            socket.getfqdn()
+        principal = "%s/%s" % (principal or configuration.conf.get('kerberos', 'principal'),
+                               socket.getfqdn())
+        princ = principal
+        ccache = configuration.conf.get('kerberos', 'principal')
+        log.error(
+            "Couldn't renew kerberos ticket in order to work around Kerberos 1.8.1 issue. Please check that "
+            "the ticket for '%s' is still renewable:\n  $ kinit -f -c %s\nIf the 'renew until' date is the "
+            "same as the 'valid starting' date, the ticket cannot be renewed. Please check your KDC "
+            "configuration, and the ticket renewal policy (maxrenewlife) for the '%s' and `krbtgt' "
+            "principals.", princ, ccache, princ
         )
-        fmt_dict = dict(princ=principal,
-                        ccache=configuration.conf.get('kerberos', 'principal'))
-        log.error("Couldn't renew kerberos ticket in order to work around "
-                  "Kerberos 1.8.1 issue. Please check that the ticket for "
-                  "'%(princ)s' is still renewable:\n"
-                  "  $ kinit -f -c %(ccache)s\n"
-                  "If the 'renew until' date is the same as the 'valid starting' "
-                  "date, the ticket cannot be renewed. Please check your KDC "
-                  "configuration, and the ticket renewal policy (maxrenewlife) "
-                  "for the '%(princ)s' and `krbtgt' principals." % fmt_dict)
         sys.exit(ret)
 
 
@@ -110,11 +110,11 @@ def detect_conf_var():
         return b'X-CACHECONF:' in f.read()
 
 
-def run():
-    if configuration.conf.get('kerberos', 'keytab') is None:
+def run(principal, keytab):
+    if not keytab:
         log.debug("Keytab renewer not starting, no keytab configured")
         sys.exit(0)
 
     while True:
-        renew_from_kt()
+        renew_from_kt(principal, keytab)
         time.sleep(configuration.conf.getint('kerberos', 'reinit_frequency'))
diff --git a/airflow/security/utils.py b/airflow/security/utils.py
index 8e4fcbd4bfc0e..e10510e24eb71 100644
--- a/airflow/security/utils.py
+++ b/airflow/security/utils.py
@@ -18,24 +18,9 @@
 
 import re
 import socket
-import airflow.configuration as conf
 
 from airflow.utils.net import get_hostname
 
-# Pattern to replace with hostname
-HOSTNAME_PATTERN = '_HOST'
-
-
-def get_kerberos_principal(principal, host):
-    components = get_components(principal)
-    if not components or len(components) != 3 or components[1] != HOSTNAME_PATTERN:
-        return principal
-    else:
-        if not host:
-            raise IOError("Can't replace %s pattern "
-                          "since host is null." % HOSTNAME_PATTERN)
-        return replace_hostname_pattern(components, host)
-
 
 def get_components(principal):
     """
@@ -45,39 +30,33 @@ def get_components(principal):
     """
     if not principal:
         return None
-    return re.split('[\/@]', str(principal))
+    return re.split(r'[\/@]', str(principal))
 
 
 def replace_hostname_pattern(components, host=None):
     fqdn = host
     if not fqdn or fqdn == '0.0.0.0':
-        fqdn = get_localhost_name()
+        fqdn = get_hostname()
     return '%s/%s@%s' % (components[0], fqdn.lower(), components[2])
 
 
-def get_localhost_name():
-    return get_hostname()
-
-
 def get_fqdn(hostname_or_ip=None):
     # Get hostname
     try:
         if hostname_or_ip:
             fqdn = socket.gethostbyaddr(hostname_or_ip)[0]
+            if fqdn == 'localhost':
+                fqdn = get_hostname()
         else:
-            fqdn = get_localhost_name()
+            fqdn = get_hostname()
     except IOError:
         fqdn = hostname_or_ip
 
-    if fqdn == 'localhost':
-        fqdn = get_localhost_name()
-
     return fqdn
 
 
-def principal_from_username(username):
-    realm = conf.get("security", "default_realm")
-    if '@' not in username and realm:
+def principal_from_username(username, realm):
+    if ('@' not in username) and realm:
         username = "{}@{}".format(username, realm)
 
     return username
diff --git a/airflow/sensors/base_sensor_operator.py b/airflow/sensors/base_sensor_operator.py
index 74b0e0fe1ca5b..fe96a36d8b3ec 100644
--- a/airflow/sensors/base_sensor_operator.py
+++ b/airflow/sensors/base_sensor_operator.py
@@ -19,20 +19,24 @@
 
 
 from time import sleep
+from datetime import timedelta
 
 from airflow.exceptions import AirflowException, AirflowSensorTimeout, \
-    AirflowSkipException
-from airflow.models import BaseOperator, SkipMixin
+    AirflowSkipException, AirflowRescheduleException
+from airflow.models import BaseOperator
+from airflow.models.skipmixin import SkipMixin
+from airflow.models.taskreschedule import TaskReschedule
 from airflow.utils import timezone
 from airflow.utils.decorators import apply_defaults
+from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep
 
 
 class BaseSensorOperator(BaseOperator, SkipMixin):
     """
-    Sensor operators are derived from this class an inherit these attributes.
+    Sensor operators are derived from this class and inherit these attributes.
 
     Sensor operators keep executing at a time interval and succeed when
-        a criteria is met and fail if and when they time out.
+    a criteria is met and fail if and when they time out.
 
     :param soft_fail: Set to true to mark the task as SKIPPED on failure
     :type soft_fail: bool
@@ -41,20 +45,51 @@ class BaseSensorOperator(BaseOperator, SkipMixin):
     :type poke_interval: int
     :param timeout: Time, in seconds before the task times out and fails.
     :type timeout: int
+    :param mode: How the sensor operates.
+        Options are: ``{ poke | reschedule }``, default is ``poke``.
+        When set to ``poke`` the sensor is taking up a worker slot for its
+        whole execution time and sleeps between pokes. Use this mode if the
+        expected runtime of the sensor is short or if a short poke interval
+        is requried.
+        When set to ``reschedule`` the sensor task frees the worker slot when
+        the criteria is not yet met and it's rescheduled at a later time. Use
+        this mode if the expected time until the criteria is met is. The poke
+        interval should be more than one minute to prevent too much load on
+        the scheduler.
+    :type mode: str
     """
     ui_color = '#e6f1f2'
+    valid_modes = ['poke', 'reschedule']
 
     @apply_defaults
     def __init__(self,
                  poke_interval=60,
                  timeout=60 * 60 * 24 * 7,
                  soft_fail=False,
+                 mode='poke',
                  *args,
                  **kwargs):
         super(BaseSensorOperator, self).__init__(*args, **kwargs)
         self.poke_interval = poke_interval
         self.soft_fail = soft_fail
         self.timeout = timeout
+        self.mode = mode
+        self._validate_input_values()
+
+    def _validate_input_values(self):
+        if not isinstance(self.poke_interval, (int, float)) or self.poke_interval < 0:
+            raise AirflowException(
+                "The poke_interval must be a non-negative number")
+        if not isinstance(self.timeout, (int, float)) or self.timeout < 0:
+            raise AirflowException(
+                "The timeout must be a non-negative number")
+        if self.mode not in self.valid_modes:
+            raise AirflowException(
+                "The mode must be one of {valid_modes},"
+                "'{d}.{t}'; received '{m}'."
+                .format(valid_modes=self.valid_modes,
+                        d=self.dag.dag_id if self.dag else "",
+                        t=self.task_id, m=self.mode))
 
     def poke(self, context):
         """
@@ -65,6 +100,11 @@ def poke(self, context):
 
     def execute(self, context):
         started_at = timezone.utcnow()
+        if self.reschedule:
+            # If reschedule, use first start date of current try
+            task_reschedules = TaskReschedule.find_for_task_instance(context['ti'])
+            if task_reschedules:
+                started_at = task_reschedules[0].start_date
         while not self.poke(context):
             if (timezone.utcnow() - started_at).total_seconds() > self.timeout:
                 # If sensor is in soft fail mode but will be retried then
@@ -75,7 +115,12 @@ def execute(self, context):
                     raise AirflowSkipException('Snap. Time is OUT.')
                 else:
                     raise AirflowSensorTimeout('Snap. Time is OUT.')
-            sleep(self.poke_interval)
+            if self.reschedule:
+                reschedule_date = timezone.utcnow() + timedelta(
+                    seconds=self.poke_interval)
+                raise AirflowRescheduleException(reschedule_date)
+            else:
+                sleep(self.poke_interval)
         self.log.info("Success criteria met. Exiting.")
 
     def _do_skip_downstream_tasks(self, context):
@@ -83,3 +128,15 @@ def _do_skip_downstream_tasks(self, context):
         self.log.debug("Downstream task_ids %s", downstream_tasks)
         if downstream_tasks:
             self.skip(context['dag_run'], context['ti'].execution_date, downstream_tasks)
+
+    @property
+    def reschedule(self):
+        return self.mode == 'reschedule'
+
+    @property
+    def deps(self):
+        """
+        Adds one additional dependency for all sensor operators that
+        checks if a sensor task instance can be rescheduled.
+        """
+        return BaseOperator.deps.fget(self) | {ReadyToRescheduleDep()}
diff --git a/airflow/sensors/external_task_sensor.py b/airflow/sensors/external_task_sensor.py
index eda1a2d9d2d28..ca98cbf50e736 100644
--- a/airflow/sensors/external_task_sensor.py
+++ b/airflow/sensors/external_task_sensor.py
@@ -17,7 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from airflow.models import TaskInstance
+import os
+
+from airflow.exceptions import AirflowException
+from airflow.models import TaskInstance, DagBag, DagModel, DagRun
 from airflow.sensors.base_sensor_operator import BaseSensorOperator
 from airflow.utils.db import provide_session
 from airflow.utils.decorators import apply_defaults
@@ -26,18 +29,19 @@
 
 class ExternalTaskSensor(BaseSensorOperator):
     """
-    Waits for a task to complete in a different DAG
+    Waits for a different DAG or a task in a different DAG to complete for a
+    specific execution_date
 
     :param external_dag_id: The dag_id that contains the task you want to
         wait for
-    :type external_dag_id: string
+    :type external_dag_id: str
     :param external_task_id: The task_id that contains the task you want to
-        wait for
-    :type external_task_id: string
+        wait for. If ``None`` the sensor waits for the DAG
+    :type external_task_id: str
     :param allowed_states: list of allowed states, default is ``['success']``
     :type allowed_states: list
     :param execution_delta: time difference with the previous execution to
-        look at, the default is the same execution_date as the current task.
+        look at, the default is the same execution_date as the current task or DAG.
         For yesterday, use [positive!] datetime.timedelta(days=1). Either
         execution_delta or execution_date_fn can be passed to
         ExternalTaskSensor, but not both.
@@ -46,6 +50,11 @@ class ExternalTaskSensor(BaseSensorOperator):
         and returns the desired execution dates to query. Either execution_delta
         or execution_date_fn can be passed to ExternalTaskSensor, but not both.
     :type execution_date_fn: callable
+    :param check_existence: Set to `True` to check if the external task exists (when
+        external_task_id is not None) or check if the DAG to wait for exists (when
+        external_task_id is None), and immediately cease waiting if the external task
+        or DAG does not exist (default value: False).
+    :type check_existence: bool
     """
     template_fields = ['external_dag_id', 'external_task_id']
     ui_color = '#19647e'
@@ -57,19 +66,34 @@ def __init__(self,
                  allowed_states=None,
                  execution_delta=None,
                  execution_date_fn=None,
+                 check_existence=False,
                  *args,
                  **kwargs):
         super(ExternalTaskSensor, self).__init__(*args, **kwargs)
         self.allowed_states = allowed_states or [State.SUCCESS]
+        if external_task_id:
+            if not set(self.allowed_states) <= set(State.task_states):
+                raise ValueError(
+                    'Valid values for `allowed_states` '
+                    'when `external_task_id` is not `None`: {}'.format(State.task_states)
+                )
+        else:
+            if not set(self.allowed_states) <= set(State.dag_states):
+                raise ValueError(
+                    'Valid values for `allowed_states` '
+                    'when `external_task_id` is `None`: {}'.format(State.dag_states)
+                )
+
         if execution_delta is not None and execution_date_fn is not None:
             raise ValueError(
-                'Only one of `execution_date` or `execution_date_fn` may'
+                'Only one of `execution_delta` or `execution_date_fn` may '
                 'be provided to ExternalTaskSensor; not both.')
 
         self.execution_delta = execution_delta
         self.execution_date_fn = execution_date_fn
         self.external_dag_id = external_dag_id
         self.external_task_id = external_task_id
+        self.check_existence = check_existence
 
     @provide_session
     def poke(self, context, session=None):
@@ -85,17 +109,46 @@ def poke(self, context, session=None):
             [datetime.isoformat() for datetime in dttm_filter])
 
         self.log.info(
-            'Poking for '
-            '{self.external_dag_id}.'
-            '{self.external_task_id} on '
-            '{} ... '.format(serialized_dttm_filter, **locals()))
+            'Poking for %s.%s on %s ... ',
+            self.external_dag_id, self.external_task_id, serialized_dttm_filter
+        )
+
+        DM = DagModel
         TI = TaskInstance
+        DR = DagRun
+        if self.check_existence:
+            dag_to_wait = session.query(DM).filter(
+                DM.dag_id == self.external_dag_id
+            ).first()
+
+            if not dag_to_wait:
+                raise AirflowException('The external DAG '
+                                       '{} does not exist.'.format(self.external_dag_id))
+            else:
+                if not os.path.exists(dag_to_wait.fileloc):
+                    raise AirflowException('The external DAG '
+                                           '{} was deleted.'.format(self.external_dag_id))
+
+            if self.external_task_id:
+                refreshed_dag_info = DagBag(dag_to_wait.fileloc).get_dag(self.external_dag_id)
+                if not refreshed_dag_info.has_task(self.external_task_id):
+                    raise AirflowException('The external task'
+                                           '{} in DAG {} does not exist.'.format(self.external_task_id,
+                                                                                 self.external_dag_id))
+
+        if self.external_task_id:
+            count = session.query(TI).filter(
+                TI.dag_id == self.external_dag_id,
+                TI.task_id == self.external_task_id,
+                TI.state.in_(self.allowed_states),
+                TI.execution_date.in_(dttm_filter),
+            ).count()
+        else:
+            count = session.query(DR).filter(
+                DR.dag_id == self.external_dag_id,
+                DR.state.in_(self.allowed_states),
+                DR.execution_date.in_(dttm_filter),
+            ).count()
 
-        count = session.query(TI).filter(
-            TI.dag_id == self.external_dag_id,
-            TI.task_id == self.external_task_id,
-            TI.state.in_(self.allowed_states),
-            TI.execution_date.in_(dttm_filter),
-        ).count()
         session.commit()
         return count == len(dttm_filter)
diff --git a/airflow/sensors/hdfs_sensor.py b/airflow/sensors/hdfs_sensor.py
index c9bac08ecbfe7..bc3a3c63fd64e 100644
--- a/airflow/sensors/hdfs_sensor.py
+++ b/airflow/sensors/hdfs_sensor.py
@@ -39,13 +39,15 @@ class HdfsSensor(BaseSensorOperator):
     def __init__(self,
                  filepath,
                  hdfs_conn_id='hdfs_default',
-                 ignored_ext=['_COPYING_'],
+                 ignored_ext=None,
                  ignore_copying=True,
                  file_size=None,
                  hook=HDFSHook,
                  *args,
                  **kwargs):
         super(HdfsSensor, self).__init__(*args, **kwargs)
+        if ignored_ext is None:
+            ignored_ext = ['_COPYING_']
         self.filepath = filepath
         self.hdfs_conn_id = hdfs_conn_id
         self.file_size = file_size
@@ -78,26 +80,30 @@ def filter_for_ignored_ext(result, ignored_ext, ignore_copying):
         """
         Will filter if instructed to do so the result to remove matching criteria
 
-        :param result: (list) of dicts returned by Snakebite ls
-        :param ignored_ext: (list) of ignored extensions
-        :param ignore_copying: (bool) shall we ignore ?
-        :return: (list) of dicts which were not removed
+        :param result: list of dicts returned by Snakebite ls
+        :type result: list[dict]
+        :param ignored_ext: list of ignored extensions
+        :type ignored_ext: list
+        :param ignore_copying: shall we ignore ?
+        :type ignore_copying: bool
+        :return: list of dicts which were not removed
+        :rtype: list[dict]
         """
         if ignore_copying:
             log = LoggingMixin().log
-            regex_builder = "^.*\.(%s$)$" % '$|'.join(ignored_ext)
-            ignored_extentions_regex = re.compile(regex_builder)
+            regex_builder = r"^.*\.(%s$)$" % '$|'.join(ignored_ext)
+            ignored_extensions_regex = re.compile(regex_builder)
             log.debug(
                 'Filtering result for ignored extensions: %s in files %s',
-                ignored_extentions_regex.pattern, map(lambda x: x['path'], result)
+                ignored_extensions_regex.pattern, map(lambda x: x['path'], result)
             )
-            result = [x for x in result if not ignored_extentions_regex.match(x['path'])]
+            result = [x for x in result if not ignored_extensions_regex.match(x['path'])]
             log.debug('HdfsSensor.poke: after ext filter result is %s', result)
         return result
 
     def poke(self, context):
         sb = self.hook(self.hdfs_conn_id).get_conn()
-        self.log.info('Poking for file {self.filepath}'.format(**locals()))
+        self.log.info('Poking for file %s', self.filepath)
         try:
             # IMOO it's not right here, as there no raise of any kind.
             # if the filepath is let's say '/data/mydirectory',
diff --git a/airflow/sensors/hive_partition_sensor.py b/airflow/sensors/hive_partition_sensor.py
index c8eddc3443fc6..5165540bb4346 100644
--- a/airflow/sensors/hive_partition_sensor.py
+++ b/airflow/sensors/hive_partition_sensor.py
@@ -31,12 +31,12 @@ class HivePartitionSensor(BaseSensorOperator):
 
     :param table: The name of the table to wait for, supports the dot
         notation (my_database.my_table)
-    :type table: string
+    :type table: str
     :param partition: The partition clause to wait for. This is passed as
         is to the metastore Thrift client ``get_partitions_by_filter`` method,
         and apparently supports SQL like notation as in ``ds='2015-01-01'
         AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"``
-    :type partition: string
+    :type partition: str
     :param metastore_conn_id: reference to the metastore thrift service
         connection id
     :type metastore_conn_id: str
@@ -65,8 +65,8 @@ def poke(self, context):
         if '.' in self.table:
             self.schema, self.table = self.table.split('.')
         self.log.info(
-            'Poking for table {self.schema}.{self.table}, '
-            'partition {self.partition}'.format(**locals()))
+            'Poking for table %s.%s, partition %s', self.schema, self.table, self.partition
+        )
         if not hasattr(self, 'hook'):
             from airflow.hooks.hive_hooks import HiveMetastoreHook
             self.hook = HiveMetastoreHook(
diff --git a/airflow/sensors/http_sensor.py b/airflow/sensors/http_sensor.py
index 33f8531368ec1..4a1c75be30bb4 100644
--- a/airflow/sensors/http_sensor.py
+++ b/airflow/sensors/http_sensor.py
@@ -32,11 +32,11 @@ class HttpSensor(BaseSensorOperator):
         404 not found or response_check function returned False
 
     :param http_conn_id: The connection to run the sensor against
-    :type http_conn_id: string
+    :type http_conn_id: str
     :param method: The HTTP request method to use
-    :type method: string
+    :type method: str
     :param endpoint: The relative part of the full url
-    :type endpoint: string
+    :type endpoint: str
     :param request_params: The parameters to be added to the GET url
     :type request_params: a dictionary of string key/value pairs
     :param headers: The HTTP headers to be added to the GET request
diff --git a/airflow/sensors/named_hive_partition_sensor.py b/airflow/sensors/named_hive_partition_sensor.py
index 4a076a3dd6870..6f25458965854 100644
--- a/airflow/sensors/named_hive_partition_sensor.py
+++ b/airflow/sensors/named_hive_partition_sensor.py
@@ -34,7 +34,7 @@ class NamedHivePartitionSensor(BaseSensorOperator):
         Thrift client ``get_partitions_by_name`` method. Note that
         you cannot use logical or comparison operators as in
         HivePartitionSensor.
-    :type partition_names: list of strings
+    :type partition_names: list[str]
     :param metastore_conn_id: reference to the metastore thrift service
         connection id
     :type metastore_conn_id: str
@@ -61,9 +61,9 @@ def __init__(self,
         self.partition_names = partition_names
         self.hook = hook
         if self.hook and metastore_conn_id != 'metastore_default':
-            self.log.warning('A hook was passed but a non default'
-                             'metastore_conn_id='
-                             '{} was used'.format(metastore_conn_id))
+            self.log.warning(
+                'A hook was passed but a non defaul metastore_conn_id=%s was used', metastore_conn_id
+            )
 
     @staticmethod
     def parse_partition_name(partition):
@@ -89,9 +89,7 @@ def poke_partition(self, partition):
 
         schema, table, partition = self.parse_partition_name(partition)
 
-        self.log.info(
-            'Poking for {schema}.{table}/{partition}'.format(**locals())
-        )
+        self.log.info('Poking for %s.%s/%s', schema, table, partition)
         return self.hook.check_for_named_partition(
             schema, table, partition)
 
diff --git a/airflow/sensors/s3_key_sensor.py b/airflow/sensors/s3_key_sensor.py
index fa2eb786ffa03..a72b1c7a87d46 100644
--- a/airflow/sensors/s3_key_sensor.py
+++ b/airflow/sensors/s3_key_sensor.py
@@ -41,6 +41,17 @@ class S3KeySensor(BaseSensorOperator):
     :type wildcard_match: bool
     :param aws_conn_id: a reference to the s3 connection
     :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
     """
     template_fields = ('bucket_key', 'bucket_name')
 
@@ -50,6 +61,7 @@ def __init__(self,
                  bucket_name=None,
                  wildcard_match=False,
                  aws_conn_id='aws_default',
+                 verify=None,
                  *args,
                  **kwargs):
         super(S3KeySensor, self).__init__(*args, **kwargs)
@@ -68,12 +80,12 @@ def __init__(self,
         self.bucket_key = bucket_key
         self.wildcard_match = wildcard_match
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
 
     def poke(self, context):
         from airflow.hooks.S3_hook import S3Hook
-        hook = S3Hook(aws_conn_id=self.aws_conn_id)
-        full_url = "s3://" + self.bucket_name + "/" + self.bucket_key
-        self.log.info('Poking for key : {full_url}'.format(**locals()))
+        hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
+        self.log.info('Poking for key : s3://%s/%s', self.bucket_name, self.bucket_key)
         if self.wildcard_match:
             return hook.check_for_wildcard_key(self.bucket_key,
                                                self.bucket_name)
diff --git a/airflow/sensors/s3_prefix_sensor.py b/airflow/sensors/s3_prefix_sensor.py
index 917dd46e26c28..16980e7b4ac31 100644
--- a/airflow/sensors/s3_prefix_sensor.py
+++ b/airflow/sensors/s3_prefix_sensor.py
@@ -38,6 +38,19 @@ class S3PrefixSensor(BaseSensorOperator):
     :param delimiter: The delimiter intended to show hierarchy.
         Defaults to '/'.
     :type delimiter: str
+    :param aws_conn_id: a reference to the s3 connection
+    :type aws_conn_id: str
+    :param verify: Whether or not to verify SSL certificates for S3 connection.
+        By default SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :type verify: bool or str
     """
     template_fields = ('prefix', 'bucket_name')
 
@@ -47,6 +60,7 @@ def __init__(self,
                  prefix,
                  delimiter='/',
                  aws_conn_id='aws_default',
+                 verify=None,
                  *args,
                  **kwargs):
         super(S3PrefixSensor, self).__init__(*args, **kwargs)
@@ -56,12 +70,12 @@ def __init__(self,
         self.delimiter = delimiter
         self.full_url = "s3://" + bucket_name + '/' + prefix
         self.aws_conn_id = aws_conn_id
+        self.verify = verify
 
     def poke(self, context):
-        self.log.info('Poking for prefix : {self.prefix}\n'
-                      'in bucket s3://{self.bucket_name}'.format(**locals()))
+        self.log.info('Poking for prefix : %s in bucket s3://%s', self.prefix, self.bucket_name)
         from airflow.hooks.S3_hook import S3Hook
-        hook = S3Hook(aws_conn_id=self.aws_conn_id)
+        hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
         return hook.check_for_prefix(
             prefix=self.prefix,
             delimiter=self.delimiter,
diff --git a/airflow/sensors/sql_sensor.py b/airflow/sensors/sql_sensor.py
index e9208fa89042d..ba4e6bb03d426 100644
--- a/airflow/sensors/sql_sensor.py
+++ b/airflow/sensors/sql_sensor.py
@@ -18,7 +18,9 @@
 # under the License.
 
 from builtins import str
+from typing import Iterable
 
+from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
 from airflow.sensors.base_sensor_operator import BaseSensorOperator
 from airflow.utils.decorators import apply_defaults
@@ -30,25 +32,37 @@ class SqlSensor(BaseSensorOperator):
     sql returns no row, or if the first cell in (0, '0', '').
 
     :param conn_id: The connection to run the sensor against
-    :type conn_id: string
+    :type conn_id: str
     :param sql: The sql to run. To pass, it needs to return at least one cell
         that contains a non-zero / empty string value.
+    :type sql: str
+    :param parameters: The parameters to render the SQL query with (optional).
+    :type parameters: mapping or iterable
     """
-    template_fields = ('sql',)
-    template_ext = ('.hql', '.sql',)
+    template_fields = ('sql',)  # type: Iterable[str]
+    template_ext = ('.hql', '.sql',)  # type: Iterable[str]
     ui_color = '#7c7287'
 
     @apply_defaults
-    def __init__(self, conn_id, sql, *args, **kwargs):
-        self.sql = sql
+    def __init__(self, conn_id, sql, parameters=None, *args, **kwargs):
         self.conn_id = conn_id
+        self.sql = sql
+        self.parameters = parameters
         super(SqlSensor, self).__init__(*args, **kwargs)
 
     def poke(self, context):
-        hook = BaseHook.get_connection(self.conn_id).get_hook()
+        conn = BaseHook.get_connection(self.conn_id)
+
+        allowed_conn_type = {'google_cloud_platform', 'jdbc', 'mssql',
+                             'mysql', 'oracle', 'postgres',
+                             'presto', 'sqlite', 'vertica'}
+        if conn.conn_type not in allowed_conn_type:
+            raise AirflowException("The connection type is not supported by SqlSensor. " +
+                                   "Supported connection types: {}".format(list(allowed_conn_type)))
+        hook = conn.get_hook()
 
-        self.log.info('Poking: %s', self.sql)
-        records = hook.get_records(self.sql)
+        self.log.info('Poking: %s (with parameters %s)', self.sql, self.parameters)
+        records = hook.get_records(self.sql, self.parameters)
         if not records:
             return False
         else:
diff --git a/airflow/sensors/web_hdfs_sensor.py b/airflow/sensors/web_hdfs_sensor.py
index 67b1d385b40a2..f31a307a67ff9 100644
--- a/airflow/sensors/web_hdfs_sensor.py
+++ b/airflow/sensors/web_hdfs_sensor.py
@@ -40,5 +40,5 @@ def __init__(self,
     def poke(self, context):
         from airflow.hooks.webhdfs_hook import WebHDFSHook
         c = WebHDFSHook(self.webhdfs_conn_id)
-        self.log.info('Poking for file {self.filepath}'.format(**locals()))
+        self.log.info('Poking for file %s', self.filepath)
         return c.check_for_path(hdfs_path=self.filepath)
diff --git a/airflow/settings.py b/airflow/settings.py
index 7c0376d12f9b5..fcf54a869aeb0 100644
--- a/airflow/settings.py
+++ b/airflow/settings.py
@@ -26,12 +26,14 @@
 import logging
 import os
 import pendulum
+import sys
+from typing import Any
 
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, exc
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.pool import NullPool
 
-from airflow import configuration as conf
+from airflow.configuration import conf, AIRFLOW_HOME, WEBSERVER_CONFIG  # NOQA F401
 from airflow.logging_config import configure_logging
 from airflow.utils.sqlalchemy import setup_event_handlers
 
@@ -46,7 +48,7 @@
         TIMEZONE = pendulum.local_timezone()
     else:
         TIMEZONE = pendulum.timezone(tz)
-except:
+except Exception:
     pass
 log.info("Configured default timezone %s" % TIMEZONE)
 
@@ -69,7 +71,7 @@ def timing(cls, stat, dt):
         pass
 
 
-Stats = DummyStatsLogger
+Stats = DummyStatsLogger  # type: Any
 
 if conf.getboolean('scheduler', 'statsd_on'):
     from statsd import StatsClient
@@ -82,13 +84,13 @@ def timing(cls, stat, dt):
 else:
     Stats = DummyStatsLogger
 
-HEADER = """\
-  ____________       _____________
- ____    |__( )_________  __/__  /________      __
-____  /| |_  /__  ___/_  /_ __  /_  __ \_ | /| / /
-___  ___ |  / _  /   _  __/ _  / / /_/ /_ |/ |/ /
- _/_/  |_/_/  /_/    /_/    /_/  \____/____/|__/
- """
+HEADER = '\n'.join([
+    r'  ____________       _____________',
+    r' ____    |__( )_________  __/__  /________      __',
+    r'____  /| |_  /__  ___/_  /_ __  /_  __ \_ | /| / /',
+    r'___  ___ |  / _  /   _  __/ _  / / /_/ /_ |/ |/ /',
+    r' _/_/  |_/_/  /_/    /_/    /_/  \____/____/|__/',
+])
 
 LOGGING_LEVEL = logging.INFO
 
@@ -98,9 +100,10 @@ def timing(cls, stat, dt):
 LOG_FORMAT = conf.get('core', 'log_format')
 SIMPLE_LOG_FORMAT = conf.get('core', 'simple_log_format')
 
-AIRFLOW_HOME = None
 SQL_ALCHEMY_CONN = None
 DAGS_FOLDER = None
+PLUGINS_FOLDER = None
+LOGGING_CLASS_PATH = None
 
 engine = None
 Session = None
@@ -134,13 +137,18 @@ def policy(task_instance):
 
 
 def configure_vars():
-    global AIRFLOW_HOME
     global SQL_ALCHEMY_CONN
     global DAGS_FOLDER
-    AIRFLOW_HOME = os.path.expanduser(conf.get('core', 'AIRFLOW_HOME'))
+    global PLUGINS_FOLDER
     SQL_ALCHEMY_CONN = conf.get('core', 'SQL_ALCHEMY_CONN')
     DAGS_FOLDER = os.path.expanduser(conf.get('core', 'DAGS_FOLDER'))
 
+    PLUGINS_FOLDER = conf.get(
+        'core',
+        'plugins_folder',
+        fallback=os.path.join(AIRFLOW_HOME, 'plugins')
+    )
+
 
 def configure_orm(disable_connection_pool=False):
     log.debug("Setting up DB connection pool (PID %s)" % os.getpid())
@@ -153,7 +161,7 @@ def configure_orm(disable_connection_pool=False):
         engine_args['poolclass'] = NullPool
         log.debug("settings.configure_orm(): Using NullPool")
     elif 'sqlite' not in SQL_ALCHEMY_CONN:
-        # Engine args not supported by sqlite.
+        # Pool size engine args not supported by sqlite.
         # If no config value is defined for the pool size, select a reasonable value.
         # 0 means no limit, which could lead to exceeding the Database connection limit.
         try:
@@ -170,17 +178,27 @@ def configure_orm(disable_connection_pool=False):
         except conf.AirflowConfigException:
             pool_recycle = 1800
 
-        log.info("setting.configure_orm(): Using pool settings. pool_size={}, "
-                 "pool_recycle={}".format(pool_size, pool_recycle))
+        log.info("settings.configure_orm(): Using pool settings. pool_size={}, "
+                 "pool_recycle={}, pid={}".format(pool_size, pool_recycle, os.getpid()))
         engine_args['pool_size'] = pool_size
         engine_args['pool_recycle'] = pool_recycle
 
+    # Allow the user to specify an encoding for their DB otherwise default
+    # to utf-8 so jobs & users with non-latin1 characters can still use
+    # us.
+    engine_args['encoding'] = conf.get('core', 'SQL_ENGINE_ENCODING', fallback='utf-8')
+    # For Python2 we get back a newstr and need a str
+    engine_args['encoding'] = engine_args['encoding'].__str__()
+
     engine = create_engine(SQL_ALCHEMY_CONN, **engine_args)
     reconnect_timeout = conf.getint('core', 'SQL_ALCHEMY_RECONNECT_TIMEOUT')
     setup_event_handlers(engine, reconnect_timeout)
 
     Session = scoped_session(
-        sessionmaker(autocommit=False, autoflush=False, bind=engine))
+        sessionmaker(autocommit=False,
+                     autoflush=False,
+                     bind=engine,
+                     expire_on_commit=False))
 
 
 def dispose_orm():
@@ -211,30 +229,70 @@ def configure_adapters():
         pass
 
 
+def validate_session():
+    worker_precheck = conf.getboolean('core', 'worker_precheck', fallback=False)
+    if not worker_precheck:
+        return True
+    else:
+        check_session = sessionmaker(bind=engine)
+        session = check_session()
+        try:
+            session.execute("select 1")
+            conn_status = True
+        except exc.DBAPIError as err:
+            log.error(err)
+            conn_status = False
+        session.close()
+        return conn_status
+
+
 def configure_action_logging():
     """
     Any additional configuration (register callback) for airflow.utils.action_loggers
     module
-    :return: None
+    :rtype: None
     """
     pass
 
 
+def prepare_classpath():
+    """
+    Ensures that certain subfolders of AIRFLOW_HOME are on the classpath
+    """
+
+    if DAGS_FOLDER not in sys.path:
+        sys.path.append(DAGS_FOLDER)
+
+    # Add ./config/ for loading custom log parsers etc, or
+    # airflow_local_settings etc.
+    config_path = os.path.join(AIRFLOW_HOME, 'config')
+    if config_path not in sys.path:
+        sys.path.append(config_path)
+
+    if PLUGINS_FOLDER not in sys.path:
+        sys.path.append(PLUGINS_FOLDER)
+
+
 try:
-    from airflow_local_settings import *
+    from airflow_local_settings import *  # noqa F403 F401
     log.info("Loaded airflow_local_settings.")
-except:
+except Exception:
     pass
 
-configure_logging()
-configure_vars()
-configure_adapters()
-# The webservers import this file from models.py with the default settings.
-configure_orm()
-configure_action_logging()
 
-# Ensure we close DB connections at scheduler and gunicon worker terminations
-atexit.register(dispose_orm)
+def initialize():
+    configure_vars()
+    prepare_classpath()
+    global LOGGING_CLASS_PATH
+    LOGGING_CLASS_PATH = configure_logging()
+    configure_adapters()
+    # The webservers import this file from models.py with the default settings.
+    configure_orm()
+    configure_action_logging()
+
+    # Ensure we close DB connections at scheduler and gunicon worker terminations
+    atexit.register(dispose_orm)
+
 
 # Const stuff
 
diff --git a/airflow/task/task_runner/__init__.py b/airflow/task/task_runner/__init__.py
index 0edc020d41e3d..960a9e9e8e426 100644
--- a/airflow/task/task_runner/__init__.py
+++ b/airflow/task/task_runner/__init__.py
@@ -18,7 +18,7 @@
 # under the License.
 
 from airflow import configuration
-from airflow.task.task_runner.bash_task_runner import BashTaskRunner
+from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
 from airflow.exceptions import AirflowException
 
 _TASK_RUNNER = configuration.conf.get('core', 'TASK_RUNNER')
@@ -29,13 +29,13 @@ def get_task_runner(local_task_job):
     Get the task runner that can be used to run the given job.
 
     :param local_task_job: The LocalTaskJob associated with the TaskInstance
-    that needs to be executed.
+        that needs to be executed.
     :type local_task_job: airflow.jobs.LocalTaskJob
     :return: The task runner to use to run the task.
     :rtype: airflow.task.task_runner.base_task_runner.BaseTaskRunner
     """
-    if _TASK_RUNNER == "BashTaskRunner":
-        return BashTaskRunner(local_task_job)
+    if _TASK_RUNNER == "StandardTaskRunner":
+        return StandardTaskRunner(local_task_job)
     elif _TASK_RUNNER == "CgroupTaskRunner":
         from airflow.contrib.task_runner.cgroup_task_runner import CgroupTaskRunner
         return CgroupTaskRunner(local_task_job)
diff --git a/airflow/task/task_runner/base_task_runner.py b/airflow/task/task_runner/base_task_runner.py
index 337f6a63baec6..f215c91b90a29 100644
--- a/airflow/task/task_runner/base_task_runner.py
+++ b/airflow/task/task_runner/base_task_runner.py
@@ -60,12 +60,6 @@ def __init__(self, local_task_job):
 
         # Always provide a copy of the configuration file settings
         cfg_path = tmp_configuration_copy()
-        # The following command should always work since the user doing chmod is the same
-        # as the one who just created the file.
-        subprocess.call(
-            ['chmod', '600', cfg_path],
-            close_fds=True
-        )
 
         # Add sudo commands to change user if we need to. Needed to handle SubDagOperator
         # case using a SequentialExecutor.
@@ -106,21 +100,22 @@ def _read_task_logs(self, stream):
                           self._task_instance.job_id, self._task_instance.task_id,
                           line.rstrip('\n'))
 
-    def run_command(self, run_with, join_args=False):
+    def run_command(self, run_with=None, join_args=False):
         """
-        Run the task command
+        Run the task command.
 
-        :param run_with: list of tokens to run the task command with
-        E.g. ['bash', '-c']
+        :param run_with: list of tokens to run the task command with e.g. ``['bash', '-c']``
         :type run_with: list
-        :param join_args: whether to concatenate the list of command tokens
-        E.g. ['airflow', 'run'] vs ['airflow run']
+        :param join_args: whether to concatenate the list of command tokens e.g. ``['airflow', 'run']`` vs
+            ``['airflow run']``
         :param join_args: bool
         :return: the process that was run
         :rtype: subprocess.Popen
         """
+        run_with = run_with or []
         cmd = [" ".join(self._command)] if join_args else self._command
         full_cmd = run_with + cmd
+
         self.log.info('Running: %s', full_cmd)
         proc = subprocess.Popen(
             full_cmd,
@@ -150,8 +145,8 @@ def start(self):
     def return_code(self):
         """
         :return: The return code associated with running the task instance or
-        None if the task is not yet done.
-        :rtype int:
+            None if the task is not yet done.
+        :rtype: int
         """
         raise NotImplementedError()
 
@@ -166,4 +161,7 @@ def on_finish(self):
         A callback that should be called when this is done running.
         """
         if self._cfg_path and os.path.isfile(self._cfg_path):
-            subprocess.call(['sudo', 'rm', self._cfg_path], close_fds=True)
+            if self.run_as_user:
+                subprocess.call(['sudo', 'rm', self._cfg_path], close_fds=True)
+            else:
+                os.remove(self._cfg_path)
diff --git a/airflow/task/task_runner/bash_task_runner.py b/airflow/task/task_runner/standard_task_runner.py
similarity index 85%
rename from airflow/task/task_runner/bash_task_runner.py
rename to airflow/task/task_runner/standard_task_runner.py
index 4ddcac5982666..6b65fc35c555a 100644
--- a/airflow/task/task_runner/bash_task_runner.py
+++ b/airflow/task/task_runner/standard_task_runner.py
@@ -23,15 +23,15 @@
 from airflow.utils.helpers import reap_process_group
 
 
-class BashTaskRunner(BaseTaskRunner):
+class StandardTaskRunner(BaseTaskRunner):
     """
     Runs the raw Airflow task by invoking through the Bash shell.
     """
     def __init__(self, local_task_job):
-        super(BashTaskRunner, self).__init__(local_task_job)
+        super(StandardTaskRunner, self).__init__(local_task_job)
 
     def start(self):
-        self.process = self.run_command(['bash', '-c'], join_args=True)
+        self.process = self.run_command()
 
     def return_code(self):
         return self.process.poll()
@@ -41,4 +41,4 @@ def terminate(self):
             reap_process_group(self.process.pid, self.log)
 
     def on_finish(self):
-        super(BashTaskRunner, self).on_finish()
+        super(StandardTaskRunner, self).on_finish()
diff --git a/airflow/ti_deps/dep_context.py b/airflow/ti_deps/dep_context.py
index 6d39998988f83..256063f3048e9 100644
--- a/airflow/ti_deps/dep_context.py
+++ b/airflow/ti_deps/dep_context.py
@@ -33,10 +33,11 @@ class DepContext(object):
     """
     A base class for contexts that specifies which dependencies should be evaluated in
     the context for a task instance to satisfy the requirements of the context. Also
-    stores state related to the context that can be used by dependendency classes.
+    stores state related to the context that can be used by dependency classes.
 
     For example there could be a SomeRunContext that subclasses this class which has
     dependencies for:
+
     - Making sure there are slots available on the infrastructure to run the task instance
     - A task-instance's task-specific dependencies are met (e.g. the previous task
       instance completed successfully)
@@ -44,25 +45,27 @@ class DepContext(object):
 
     :param deps: The context-specific dependencies that need to be evaluated for a
         task instance to run in this execution context.
-    :type deps: set(BaseTIDep)
+    :type deps: set(airflow.ti_deps.deps.base_ti_dep.BaseTIDep)
     :param flag_upstream_failed: This is a hack to generate the upstream_failed state
         creation while checking to see whether the task instance is runnable. It was the
         shortest path to add the feature. This is bad since this class should be pure (no
         side effects).
-    :type flag_upstream_failed: boolean
+    :type flag_upstream_failed: bool
     :param ignore_all_deps: Whether or not the context should ignore all ignoreable
         dependencies. Overrides the other ignore_* parameters
-    :type ignore_all_deps: boolean
+    :type ignore_all_deps: bool
     :param ignore_depends_on_past: Ignore depends_on_past parameter of DAGs (e.g. for
         Backfills)
-    :type ignore_depends_on_past: boolean
+    :type ignore_depends_on_past: bool
     :param ignore_in_retry_period: Ignore the retry period for task instances
-    :type ignore_in_retry_period: boolean
+    :type ignore_in_retry_period: bool
+    :param ignore_in_reschedule_period: Ignore the reschedule period for task instances
+    :type ignore_in_reschedule_period: bool
     :param ignore_task_deps: Ignore task-specific dependencies such as depends_on_past and
         trigger rule
-    :type ignore_task_deps: boolean
+    :type ignore_task_deps: bool
     :param ignore_ti_state: Ignore the task instance's previous failure/success
-    :type ignore_ti_state: boolean
+    :type ignore_ti_state: bool
     """
     def __init__(
             self,
@@ -71,6 +74,7 @@ def __init__(
             ignore_all_deps=False,
             ignore_depends_on_past=False,
             ignore_in_retry_period=False,
+            ignore_in_reschedule_period=False,
             ignore_task_deps=False,
             ignore_ti_state=False):
         self.deps = deps or set()
@@ -78,6 +82,7 @@ def __init__(
         self.ignore_all_deps = ignore_all_deps
         self.ignore_depends_on_past = ignore_depends_on_past
         self.ignore_in_retry_period = ignore_in_retry_period
+        self.ignore_in_reschedule_period = ignore_in_reschedule_period
         self.ignore_task_deps = ignore_task_deps
         self.ignore_ti_state = ignore_ti_state
 
@@ -91,6 +96,7 @@ def __init__(
     State.SKIPPED,
     State.UPSTREAM_FAILED,
     State.UP_FOR_RETRY,
+    State.UP_FOR_RESCHEDULE,
 }
 
 # Context to get the dependencies that need to be met in order for a task instance to
diff --git a/airflow/ti_deps/deps/base_ti_dep.py b/airflow/ti_deps/deps/base_ti_dep.py
index 90c156cb760b9..1be9bd7d67650 100644
--- a/airflow/ti_deps/deps/base_ti_dep.py
+++ b/airflow/ti_deps/deps/base_ti_dep.py
@@ -66,9 +66,9 @@ def _get_dep_statuses(self, ti, session, dep_context=None):
         representing if each of the passed in task's upstream tasks succeeded or not.
 
         :param ti: the task instance to get the dependency status for
-        :type ti: TaskInstance
+        :type ti: airflow.models.TaskInstance
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :param dep_context: the context for which this dependency should be evaluated for
         :type dep_context: DepContext
         """
@@ -81,9 +81,9 @@ def get_dep_statuses(self, ti, session, dep_context=None):
         checks for all dependencies.
 
         :param ti: the task instance to get the dependency status for
-        :type ti: TaskInstance
+        :type ti: airflow.models.TaskInstance
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :param dep_context: the context for which this dependency should be evaluated for
         :type dep_context: DepContext
         """
@@ -114,9 +114,9 @@ def is_met(self, ti, session, dep_context=None):
         passing.
 
         :param ti: the task instance to see if this dependency is met for
-        :type ti: TaskInstance
+        :type ti: airflow.models.TaskInstance
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :param dep_context: The context this dependency is being checked under that stores
             state that can be used by this dependency.
         :type dep_context: BaseDepContext
@@ -130,9 +130,9 @@ def get_failure_reasons(self, ti, session, dep_context=None):
         Returns an iterable of strings that explain why this dependency wasn't met.
 
         :param ti: the task instance to see if this dependency is met for
-        :type ti: TaskInstance
+        :type ti: airflow.models.TaskInstance
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         :param dep_context: The context this dependency is being checked under that stores
             state that can be used by this dependency.
         :type dep_context: BaseDepContext
diff --git a/airflow/ti_deps/deps/ready_to_reschedule.py b/airflow/ti_deps/deps/ready_to_reschedule.py
new file mode 100644
index 0000000000000..348cf00f5feb6
--- /dev/null
+++ b/airflow/ti_deps/deps/ready_to_reschedule.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.models.taskreschedule import TaskReschedule
+from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
+from airflow.utils import timezone
+from airflow.utils.db import provide_session
+from airflow.utils.state import State
+
+
+class ReadyToRescheduleDep(BaseTIDep):
+    NAME = "Ready To Reschedule"
+    IGNOREABLE = True
+    IS_TASK_DEP = True
+    RESCHEDULEABLE_STATES = {State.UP_FOR_RESCHEDULE, State.NONE}
+
+    @provide_session
+    def _get_dep_statuses(self, ti, session, dep_context):
+        """
+        Determines whether a task is ready to be rescheduled. Only tasks in
+        NONE state with at least one row in task_reschedule table are
+        handled by this dependency class, otherwise this dependency is
+        considered as passed. This dependency fails if the latest reschedule
+        request's reschedule date is still in future.
+        """
+        if dep_context.ignore_in_reschedule_period:
+            yield self._passing_status(
+                reason="The context specified that being in a reschedule period was "
+                       "permitted.")
+            return
+
+        if ti.state not in self.RESCHEDULEABLE_STATES:
+            yield self._passing_status(
+                reason="The task instance is not in State_UP_FOR_RESCHEDULE or NONE state.")
+            return
+
+        task_reschedules = TaskReschedule.find_for_task_instance(task_instance=ti)
+        if not task_reschedules:
+            yield self._passing_status(
+                reason="There is no reschedule request for this task instance.")
+            return
+
+        now = timezone.utcnow()
+        next_reschedule_date = task_reschedules[-1].reschedule_date
+        if now >= next_reschedule_date:
+            yield self._passing_status(
+                reason="Task instance id ready for reschedule.")
+            return
+
+        yield self._failing_status(
+            reason="Task is not ready for reschedule yet but will be rescheduled "
+                   "automatically. Current date is {0} and task will be rescheduled "
+                   "at {1}.".format(now.isoformat(), next_reschedule_date.isoformat()))
diff --git a/airflow/ti_deps/deps/trigger_rule_dep.py b/airflow/ti_deps/deps/trigger_rule_dep.py
index 76e5e1324ffd7..833e5f1d5fc34 100644
--- a/airflow/ti_deps/deps/trigger_rule_dep.py
+++ b/airflow/ti_deps/deps/trigger_rule_dep.py
@@ -103,24 +103,24 @@ def _evaluate_trigger_rule(
         rule was met.
 
         :param ti: the task instance to evaluate the trigger rule of
-        :type ti: TaskInstance
+        :type ti: airflow.models.TaskInstance
         :param successes: Number of successful upstream tasks
-        :type successes: boolean
+        :type successes: bool
         :param skipped: Number of skipped upstream tasks
-        :type skipped: boolean
+        :type skipped: bool
         :param failed: Number of failed upstream tasks
-        :type failed: boolean
+        :type failed: bool
         :param upstream_failed: Number of upstream_failed upstream tasks
-        :type upstream_failed: boolean
+        :type upstream_failed: bool
         :param done: Number of completed upstream tasks
-        :type done: boolean
+        :type done: bool
         :param flag_upstream_failed: This is a hack to generate
             the upstream_failed state creation while checking to see
             whether the task instance is runnable. It was the shortest
             path to add the feature
-        :type flag_upstream_failed: boolean
+        :type flag_upstream_failed: bool
         :param session: database session
-        :type session: Session
+        :type session: sqlalchemy.orm.session.Session
         """
 
         TR = airflow.models.TriggerRule
@@ -152,6 +152,14 @@ def _evaluate_trigger_rule(
             elif tr == TR.ONE_FAILED:
                 if upstream_done and not (failed or upstream_failed):
                     ti.set_state(State.SKIPPED, session)
+            elif tr == TR.NONE_FAILED:
+                if upstream_failed or failed:
+                    ti.set_state(State.UPSTREAM_FAILED, session)
+                elif skipped == upstream:
+                    ti.set_state(State.SKIPPED, session)
+            elif tr == TR.NONE_SKIPPED:
+                if skipped:
+                    ti.set_state(State.SKIPPED, session)
 
         if tr == TR.ONE_SUCCESS:
             if successes <= 0:
@@ -194,6 +202,23 @@ def _evaluate_trigger_rule(
                     "upstream_task_ids={3}"
                     .format(tr, upstream_done, upstream_tasks_state,
                             task.upstream_task_ids))
+        elif tr == TR.NONE_FAILED:
+            num_failures = upstream - successes - skipped
+            if num_failures > 0:
+                yield self._failing_status(
+                    reason="Task's trigger rule '{0}' requires all upstream "
+                    "tasks to have succeeded or been skipped, but found {1} non-success(es). "
+                    "upstream_tasks_state={2}, upstream_task_ids={3}"
+                    .format(tr, num_failures, upstream_tasks_state,
+                            task.upstream_task_ids))
+        elif tr == TR.NONE_SKIPPED:
+            if skipped > 0:
+                yield self._failing_status(
+                    reason="Task's trigger rule '{0}' requires all upstream "
+                    "tasks to not have been skipped, but found {1} task(s) skipped. "
+                    "upstream_tasks_state={2}, upstream_task_ids={3}"
+                    .format(tr, skipped, upstream_tasks_state,
+                            task.upstream_task_ids))
         else:
             yield self._failing_status(
                 reason="No strategy to evaluate trigger rule '{0}'.".format(tr))
diff --git a/airflow/utils/__init__.py b/airflow/utils/__init__.py
index cc4e12be8033a..e8e889c59b7a5 100644
--- a/airflow/utils/__init__.py
+++ b/airflow/utils/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/utils/asciiart.py b/airflow/utils/asciiart.py
index faf796c18d4fd..491d79aa3e74f 100644
--- a/airflow/utils/asciiart.py
+++ b/airflow/utils/asciiart.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-bug = r"""\
+bug = r"""
       =,             .=
      =.|    ,---.    |.=
      =.| "-(:::::)-" |.=
diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py
index 4a1e57a062b4b..6c3bf42c9458c 100644
--- a/airflow/utils/cli.py
+++ b/airflow/utils/cli.py
@@ -30,7 +30,7 @@
 from argparse import Namespace
 from datetime import datetime
 
-import airflow.models
+from airflow.models.log import Log
 from airflow.utils import cli_action_loggers
 
 
@@ -46,7 +46,7 @@ def action_logging(f):
         end_datetime : end datetime instance by utc
         full_command : full command line arguments
         user : current user
-        log : airflow.models.Log ORM instance
+        log : airflow.models.log.Log ORM instance
         dag_id : dag id (optional)
         task_id : task_id (optional)
         execution_date : execution date (optional)
@@ -94,10 +94,8 @@ def _build_metrics(func_name, namespace):
     :return: dict with metrics
     """
 
-    metrics = {'sub_command': func_name}
-    metrics['start_datetime'] = datetime.utcnow()
-    metrics['full_command'] = '{}'.format(list(sys.argv))
-    metrics['user'] = getpass.getuser()
+    metrics = {'sub_command': func_name, 'start_datetime': datetime.utcnow(),
+               'full_command': '{}'.format(list(sys.argv)), 'user': getpass.getuser()}
 
     assert isinstance(namespace, Namespace)
     tmp_dic = vars(namespace)
@@ -107,7 +105,7 @@ def _build_metrics(func_name, namespace):
     metrics['host_name'] = socket.gethostname()
 
     extra = json.dumps(dict((k, metrics[k]) for k in ('host_name', 'full_command')))
-    log = airflow.models.Log(
+    log = Log(
         event='cli_{}'.format(func_name),
         task_instance=None,
         owner=metrics['user'],
diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py
index e61642d30012c..1c96e5f55c89e 100644
--- a/airflow/utils/cli_action_loggers.py
+++ b/airflow/utils/cli_action_loggers.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,8 +24,9 @@
 from __future__ import absolute_import
 
 import logging
+from typing import List, Callable
 
-import airflow.settings
+from airflow.utils.db import create_session
 
 
 def register_pre_exec_callback(action_logger):
@@ -37,7 +38,7 @@ def register_pre_exec_callback(action_logger):
     :param action_logger: An action logger function
     :return: None
     """
-    logging.debug("Adding {} to pre execution callback".format(action_logger))
+    logging.debug("Adding %s to pre execution callback", action_logger)
     __pre_exec_callbacks.append(action_logger)
 
 
@@ -50,7 +51,7 @@ def register_post_exec_callback(action_logger):
     :param action_logger: An action logger function
     :return: None
     """
-    logging.debug("Adding {} to post execution callback".format(action_logger))
+    logging.debug("Adding %s to post execution callback", action_logger)
     __post_exec_callbacks.append(action_logger)
 
 
@@ -61,12 +62,12 @@ def on_pre_execution(**kwargs):
     :param kwargs:
     :return: None
     """
-    logging.debug("Calling callbacks: {}".format(__pre_exec_callbacks))
+    logging.debug("Calling callbacks: %s", __pre_exec_callbacks)
     for cb in __pre_exec_callbacks:
         try:
             cb(**kwargs)
         except Exception:
-            logging.exception('Failed on pre-execution callback using {}'.format(cb))
+            logging.exception('Failed on pre-execution callback using %s', cb)
 
 
 def on_post_execution(**kwargs):
@@ -78,12 +79,12 @@ def on_post_execution(**kwargs):
     :param kwargs:
     :return: None
     """
-    logging.debug("Calling callbacks: {}".format(__post_exec_callbacks))
+    logging.debug("Calling callbacks: %s", __post_exec_callbacks)
     for cb in __post_exec_callbacks:
         try:
             cb(**kwargs)
         except Exception:
-            logging.exception('Failed on post-execution callback using {}'.format(cb))
+            logging.exception('Failed on post-execution callback using %s', cb)
 
 
 def default_action_log(log, **_):
@@ -94,13 +95,12 @@ def default_action_log(log, **_):
     :param **_: other keyword arguments that is not being used by this function
     :return: None
     """
-    session = airflow.settings.Session()
-    session.add(log)
-    session.commit()
+    with create_session() as session:
+        session.add(log)
 
 
-__pre_exec_callbacks = []
-__post_exec_callbacks = []
+__pre_exec_callbacks = []  # type: List[Callable]
+__post_exec_callbacks = []  # type: List[Callable]
 
 # By default, register default action log into pre-execution callback
 register_pre_exec_callback(default_action_log)
diff --git a/airflow/utils/compression.py b/airflow/utils/compression.py
index f6e923a15619f..2565299175bb0 100644
--- a/airflow/utils/compression.py
+++ b/airflow/utils/compression.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/utils/configuration.py b/airflow/utils/configuration.py
index 18a338c23f6ff..6a621d5fa9c18 100644
--- a/airflow/utils/configuration.py
+++ b/airflow/utils/configuration.py
@@ -26,16 +26,18 @@
 from airflow import configuration as conf
 
 
-def tmp_configuration_copy():
+def tmp_configuration_copy(chmod=0o600):
     """
     Returns a path for a temporary file including a full copy of the configuration
     settings.
     :return: a path to a temporary file
     """
-    cfg_dict = conf.as_dict(display_sensitive=True)
+    cfg_dict = conf.as_dict(display_sensitive=True, raw=True)
     temp_fd, cfg_path = mkstemp()
 
     with os.fdopen(temp_fd, 'w') as temp_file:
+        if chmod is not None:
+            os.fchmod(temp_fd, chmod)
         json.dump(cfg_dict, temp_file)
 
     return cfg_path
diff --git a/airflow/utils/dag_processing.py b/airflow/utils/dag_processing.py
index 543eb41692e03..b2055a5d7f6fc 100644
--- a/airflow/utils/dag_processing.py
+++ b/airflow/utils/dag_processing.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,17 +22,37 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import logging
+import multiprocessing
 import os
 import re
+import signal
+import sys
 import time
 import zipfile
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
-
+from collections import namedtuple
+from datetime import timedelta
+from importlib import import_module
+import enum
+
+import psutil
+from six.moves import range, reload_module
+from sqlalchemy import or_
+from tabulate import tabulate
+
+# To avoid circular imports
+import airflow.models
+from airflow import configuration as conf
 from airflow.dag.base_dag import BaseDag, BaseDagBag
 from airflow.exceptions import AirflowException
+from airflow.settings import Stats
+from airflow.models import errors
 from airflow.utils import timezone
+from airflow.utils.db import provide_session
 from airflow.utils.log.logging_mixin import LoggingMixin
+from airflow.utils.state import State
 
 
 class SimpleDag(BaseDag):
@@ -44,7 +64,7 @@ class SimpleDag(BaseDag):
     def __init__(self, dag, pickle_id=None):
         """
         :param dag: the DAG
-        :type dag: DAG
+        :type dag: airflow.models.DAG
         :param pickle_id: ID associated with the pickled version of this DAG.
         :type pickle_id: unicode
         """
@@ -121,6 +141,103 @@ def get_task_special_arg(self, task_id, special_arg_name):
             return None
 
 
+class SimpleTaskInstance(object):
+    def __init__(self, ti):
+        self._dag_id = ti.dag_id
+        self._task_id = ti.task_id
+        self._execution_date = ti.execution_date
+        self._start_date = ti.start_date
+        self._end_date = ti.end_date
+        self._try_number = ti.try_number
+        self._state = ti.state
+        self._executor_config = ti.executor_config
+        if hasattr(ti, 'run_as_user'):
+            self._run_as_user = ti.run_as_user
+        else:
+            self._run_as_user = None
+        if hasattr(ti, 'pool'):
+            self._pool = ti.pool
+        else:
+            self._pool = None
+        if hasattr(ti, 'priority_weight'):
+            self._priority_weight = ti.priority_weight
+        else:
+            self._priority_weight = None
+        self._queue = ti.queue
+        self._key = ti.key
+
+    @property
+    def dag_id(self):
+        return self._dag_id
+
+    @property
+    def task_id(self):
+        return self._task_id
+
+    @property
+    def execution_date(self):
+        return self._execution_date
+
+    @property
+    def start_date(self):
+        return self._start_date
+
+    @property
+    def end_date(self):
+        return self._end_date
+
+    @property
+    def try_number(self):
+        return self._try_number
+
+    @property
+    def state(self):
+        return self._state
+
+    @property
+    def pool(self):
+        return self._pool
+
+    @property
+    def priority_weight(self):
+        return self._priority_weight
+
+    @property
+    def queue(self):
+        return self._queue
+
+    @property
+    def key(self):
+        return self._key
+
+    @property
+    def executor_config(self):
+        return self._executor_config
+
+    @provide_session
+    def construct_task_instance(self, session=None, lock_for_update=False):
+        """
+        Construct a TaskInstance from the database based on the primary key
+
+        :param session: DB session.
+        :param lock_for_update: if True, indicates that the database should
+            lock the TaskInstance (issuing a FOR UPDATE clause) until the
+            session is committed.
+        """
+        TI = airflow.models.TaskInstance
+
+        qry = session.query(TI).filter(
+            TI.dag_id == self._dag_id,
+            TI.task_id == self._task_id,
+            TI.execution_date == self._execution_date)
+
+        if lock_for_update:
+            ti = qry.with_for_update().first()
+        else:
+            ti = qry.first()
+        return ti
+
+
 class SimpleDagBag(BaseDagBag):
     """
     A collection of SimpleDag objects with some convenience methods.
@@ -131,7 +248,7 @@ def __init__(self, simple_dags):
         Constructor.
 
         :param simple_dags: SimpleDag objects that should be in this
-        :type: list(SimpleDag)
+        :type list(airflow.utils.dag_processing.SimpleDagBag)
         """
         self.simple_dags = simple_dags
         self.dag_id_to_simple_dag = {}
@@ -153,37 +270,57 @@ def get_dag(self, dag_id):
         :type dag_id: unicode
         :return: if the given DAG ID exists in the bag, return the BaseDag
         corresponding to that ID. Otherwise, throw an Exception
-        :rtype: SimpleDag
+        :rtype: airflow.utils.dag_processing.SimpleDag
         """
         if dag_id not in self.dag_id_to_simple_dag:
             raise AirflowException("Unknown DAG ID {}".format(dag_id))
         return self.dag_id_to_simple_dag[dag_id]
 
 
-def list_py_file_paths(directory, safe_mode=True):
+def list_py_file_paths(directory, safe_mode=True,
+                       include_examples=None):
     """
     Traverse a directory and look for Python files.
 
     :param directory: the directory to traverse
     :type directory: unicode
     :param safe_mode: whether to use a heuristic to determine whether a file
-    contains Airflow DAG definitions
+        contains Airflow DAG definitions
     :return: a list of paths to Python files in the specified directory
     :rtype: list[unicode]
     """
+    if include_examples is None:
+        include_examples = conf.getboolean('core', 'LOAD_EXAMPLES')
     file_paths = []
     if directory is None:
         return []
     elif os.path.isfile(directory):
         return [directory]
     elif os.path.isdir(directory):
-        patterns = []
+        patterns_by_dir = {}
         for root, dirs, files in os.walk(directory, followlinks=True):
-            ignore_file = [f for f in files if f == '.airflowignore']
-            if ignore_file:
-                f = open(os.path.join(root, ignore_file[0]), 'r')
-                patterns += [p for p in f.read().split('\n') if p]
-                f.close()
+            patterns = patterns_by_dir.get(root, [])
+            ignore_file = os.path.join(root, '.airflowignore')
+            if os.path.isfile(ignore_file):
+                with open(ignore_file, 'r') as f:
+                    # If we have new patterns create a copy so we don't change
+                    # the previous list (which would affect other subdirs)
+                    patterns = patterns + [p for p in f.read().split('\n') if p]
+
+            # If we can ignore any subdirs entirely we should - fewer paths
+            # to walk is better. We have to modify the ``dirs`` array in
+            # place for this to affect os.walk
+            dirs[:] = [
+                d
+                for d in dirs
+                if not any(re.search(p, os.path.join(root, d)) for p in patterns)
+            ]
+
+            # We want patterns defined in a parent folder's .airflowignore to
+            # apply to subdirs too
+            for d in dirs:
+                patterns_by_dir[os.path.join(root, d)] = patterns
+
             for f in files:
                 try:
                     file_path = os.path.join(root, f)
@@ -212,6 +349,10 @@ def list_py_file_paths(directory, safe_mode=True):
                 except Exception:
                     log = LoggingMixin().log
                     log.exception("Error while examining %s", f)
+    if include_examples:
+        import airflow.example_dags
+        example_dag_folder = airflow.example_dags.__path__[0]
+        file_paths.extend(list_py_file_paths(example_dag_folder, safe_mode, False))
     return file_paths
 
 
@@ -268,7 +409,7 @@ def done(self):
     def result(self):
         """
         :return: result of running SchedulerJob.process_file()
-        :rtype: list[SimpleDag]
+        :rtype: list[airflow.utils.dag_processing.SimpleDag]
         """
         raise NotImplementedError()
 
@@ -291,69 +432,584 @@ def file_path(self):
         raise NotImplementedError()
 
 
+DagParsingStat = namedtuple('DagParsingStat',
+                            ['file_paths', 'all_pids', 'done',
+                             'all_files_processed', 'result_count'])
+
+
+class DagParsingSignal(enum.Enum):
+    AGENT_HEARTBEAT = 'agent_heartbeat'
+    MANAGER_DONE = 'manager_done'
+    TERMINATE_MANAGER = 'terminate_manager'
+    END_MANAGER = 'end_manager'
+
+
+class DagFileProcessorAgent(LoggingMixin):
+    """
+    Agent for DAG file processing. It is responsible for all DAG parsing
+    related jobs in scheduler process. Mainly it can spin up DagFileProcessorManager
+    in a subprocess, collect DAG parsing results from it and communicate
+    signal/DAG parsing stat with it.
+    """
+
+    def __init__(self,
+                 dag_directory,
+                 file_paths,
+                 max_runs,
+                 processor_factory,
+                 async_mode):
+        """
+        :param dag_directory: Directory where DAG definitions are kept. All
+            files in file_paths should be under this directory
+        :type dag_directory: unicode
+        :param file_paths: list of file paths that contain DAG definitions
+        :type file_paths: list[unicode]
+        :param max_runs: The number of times to parse and schedule each file. -1
+            for unlimited.
+        :type max_runs: int
+        :param processor_factory: function that creates processors for DAG
+            definition files. Arguments are (dag_definition_path, log_file_path)
+        :type processor_factory: (unicode, unicode, list) -> (AbstractDagFileProcessor)
+        :param async_mode: Whether to start agent in async mode
+        :type async_mode: bool
+        """
+        self._file_paths = file_paths
+        self._file_path_queue = []
+        self._dag_directory = dag_directory
+        self._max_runs = max_runs
+        self._processor_factory = processor_factory
+        self._async_mode = async_mode
+        # Map from file path to the processor
+        self._processors = {}
+        # Map from file path to the last runtime
+        self._last_runtime = {}
+        # Map from file path to the last finish time
+        self._last_finish_time = {}
+        # Map from file path to the number of runs
+        self._run_count = defaultdict(int)
+        # Pids of DAG parse
+        self._all_pids = []
+        # Pipe for communicating signals
+        self._parent_signal_conn, self._child_signal_conn = multiprocessing.Pipe()
+        # Pipe for communicating DagParsingStat
+        self._stat_queue = multiprocessing.Queue()
+        self._result_queue = multiprocessing.Queue()
+        self._process = None
+        self._done = False
+        # Initialized as true so we do not deactivate w/o any actual DAG parsing.
+        self._all_files_processed = True
+        self._result_count = 0
+
+    def start(self):
+        """
+        Launch DagFileProcessorManager processor and start DAG parsing loop in manager.
+        """
+        self._process = self._launch_process(self._dag_directory,
+                                             self._file_paths,
+                                             self._max_runs,
+                                             self._processor_factory,
+                                             self._child_signal_conn,
+                                             self._stat_queue,
+                                             self._result_queue,
+                                             self._async_mode)
+        self.log.info("Launched DagFileProcessorManager with pid: %s", self._process.pid)
+
+    def heartbeat(self):
+        """
+        Should only be used when launched DAG file processor manager in sync mode.
+        Send agent heartbeat signal to the manager.
+        """
+        self._parent_signal_conn.send(DagParsingSignal.AGENT_HEARTBEAT)
+
+    def wait_until_finished(self):
+        """
+        Should only be used when launched DAG file processor manager in sync mode.
+        Wait for done signal from the manager.
+        """
+        while True:
+            if self._parent_signal_conn.recv() == DagParsingSignal.MANAGER_DONE:
+                break
+
+    @staticmethod
+    def _launch_process(dag_directory,
+                        file_paths,
+                        max_runs,
+                        processor_factory,
+                        signal_conn,
+                        _stat_queue,
+                        result_queue,
+                        async_mode):
+        def helper():
+            # Reload configurations and settings to avoid collision with parent process.
+            # Because this process may need custom configurations that cannot be shared,
+            # e.g. RotatingFileHandler. And it can cause connection corruption if we
+            # do not recreate the SQLA connection pool.
+            os.environ['CONFIG_PROCESSOR_MANAGER_LOGGER'] = 'True'
+            # Replicating the behavior of how logging module was loaded
+            # in logging_config.py
+            reload_module(import_module(airflow.settings.LOGGING_CLASS_PATH.rsplit('.', 1)[0]))
+            reload_module(airflow.settings)
+            airflow.settings.initialize()
+            del os.environ['CONFIG_PROCESSOR_MANAGER_LOGGER']
+            processor_manager = DagFileProcessorManager(dag_directory,
+                                                        file_paths,
+                                                        max_runs,
+                                                        processor_factory,
+                                                        signal_conn,
+                                                        _stat_queue,
+                                                        result_queue,
+                                                        async_mode)
+
+            processor_manager.start()
+
+        p = multiprocessing.Process(target=helper,
+                                    args=(),
+                                    name="DagFileProcessorManager")
+        p.start()
+        return p
+
+    def harvest_simple_dags(self):
+        """
+        Harvest DAG parsing results from result queue and sync metadata from stat queue.
+        :return: List of parsing result in SimpleDag format.
+        """
+        # Metadata and results to be harvested can be inconsistent,
+        # but it should not be a big problem.
+        self._sync_metadata()
+        # Heartbeating after syncing metadata so we do not restart manager
+        # if it processed all files for max_run times and exit normally.
+        self._heartbeat_manager()
+        simple_dags = []
+        # multiprocessing.Queue().qsize will not work on MacOS.
+        if sys.platform == "darwin":
+            qsize = self._result_count
+        else:
+            qsize = self._result_queue.qsize()
+        for _ in range(qsize):
+            simple_dags.append(self._result_queue.get())
+
+        self._result_count = 0
+
+        return simple_dags
+
+    def _heartbeat_manager(self):
+        """
+        Heartbeat DAG file processor and start it if it is not alive.
+        :return:
+        """
+        if self._process and not self._process.is_alive() and not self.done:
+            self.start()
+
+    def _sync_metadata(self):
+        """
+        Sync metadata from stat queue and only keep the latest stat.
+        :return:
+        """
+        while not self._stat_queue.empty():
+            stat = self._stat_queue.get()
+            self._file_paths = stat.file_paths
+            self._all_pids = stat.all_pids
+            self._done = stat.done
+            self._all_files_processed = stat.all_files_processed
+            self._result_count += stat.result_count
+
+    @property
+    def file_paths(self):
+        return self._file_paths
+
+    @property
+    def done(self):
+        return self._done
+
+    @property
+    def all_files_processed(self):
+        return self._all_files_processed
+
+    def terminate(self):
+        """
+        Send termination signal to DAG parsing processor manager
+        and expect it to terminate all DAG file processors.
+        """
+        self.log.info("Sending termination message to manager.")
+        self._child_signal_conn.send(DagParsingSignal.TERMINATE_MANAGER)
+
+    def end(self):
+        """
+        Terminate (and then kill) the manager process launched.
+        :return:
+        """
+        if not self._process:
+            self.log.warn('Ending without manager process.')
+            return
+        this_process = psutil.Process(os.getpid())
+        try:
+            manager_process = psutil.Process(self._process.pid)
+        except psutil.NoSuchProcess:
+            self.log.info("Manager process not running.")
+            return
+
+        # First try SIGTERM
+        if manager_process.is_running() \
+                and manager_process.pid in [x.pid for x in this_process.children()]:
+            self.log.info("Terminating manager process: %s", manager_process.pid)
+            manager_process.terminate()
+            # TODO: Remove magic number
+            timeout = 5
+            self.log.info("Waiting up to %ss for manager process to exit...", timeout)
+            try:
+                psutil.wait_procs({manager_process}, timeout)
+            except psutil.TimeoutExpired:
+                self.log.debug("Ran out of time while waiting for "
+                               "processes to exit")
+
+        # Then SIGKILL
+        if manager_process.is_running() \
+                and manager_process.pid in [x.pid for x in this_process.children()]:
+            self.log.info("Killing manager process: %s", manager_process.pid)
+            manager_process.kill()
+            manager_process.wait()
+
+
 class DagFileProcessorManager(LoggingMixin):
     """
     Given a list of DAG definition files, this kicks off several processors
-    in parallel to process them. The parallelism is limited and as the
+    in parallel to process them and put the results to a multiprocessing.Queue
+    for DagFileProcessorAgent to harvest. The parallelism is limited and as the
     processors finish, more are launched. The files are processed over and
     over again, but no more often than the specified interval.
 
     :type _file_path_queue: list[unicode]
     :type _processors: dict[unicode, AbstractDagFileProcessor]
     :type _last_runtime: dict[unicode, float]
-    :type _last_finish_time: dict[unicode, datetime]
+    :type _last_finish_time: dict[unicode, datetime.datetime]
     """
 
     def __init__(self,
                  dag_directory,
                  file_paths,
-                 parallelism,
-                 process_file_interval,
-                 min_file_parsing_loop_time,
                  max_runs,
-                 processor_factory):
+                 processor_factory,
+                 signal_conn,
+                 stat_queue,
+                 result_queue,
+                 async_mode=True):
         """
         :param dag_directory: Directory where DAG definitions are kept. All
-        files in file_paths should be under this directory
+            files in file_paths should be under this directory
         :type dag_directory: unicode
         :param file_paths: list of file paths that contain DAG definitions
         :type file_paths: list[unicode]
-        :param parallelism: maximum number of simultaneous process to run at once
-        :type parallelism: int
-        :param process_file_interval: process a file at most once every this
-        many seconds
-        :type process_file_interval: float
-        :param min_file_parsing_loop_time: wait until at least this many seconds have
-        passed before parsing files once all files have finished parsing.
-        :type min_file_parsing_loop_time: float
         :param max_runs: The number of times to parse and schedule each file. -1
-        for unlimited.
+            for unlimited.
         :type max_runs: int
-        :type process_file_interval: float
         :param processor_factory: function that creates processors for DAG
-        definition files. Arguments are (dag_definition_path)
-        :type processor_factory: (unicode, unicode) -> (AbstractDagFileProcessor)
-
+            definition files. Arguments are (dag_definition_path)
+        :type processor_factory: (unicode, unicode, list) -> (AbstractDagFileProcessor)
+        :param signal_conn: connection to communicate signal with processor agent.
+        :type signal_conn: airflow.models.connection.Connection
+        :param stat_queue: the queue to use for passing back parsing stat to agent.
+        :type stat_queue: multiprocessing.Queue
+        :param result_queue: the queue to use for passing back the result to agent.
+        :type result_queue: multiprocessing.Queue
+        :param async_mode: whether to start the manager in async mode
+        :type async_mode: bool
         """
         self._file_paths = file_paths
         self._file_path_queue = []
-        self._parallelism = parallelism
         self._dag_directory = dag_directory
         self._max_runs = max_runs
-        self._process_file_interval = process_file_interval
-        self._min_file_parsing_loop_time = min_file_parsing_loop_time
         self._processor_factory = processor_factory
+        self._signal_conn = signal_conn
+        self._stat_queue = stat_queue
+        self._result_queue = result_queue
+        self._async_mode = async_mode
+
+        self._parallelism = conf.getint('scheduler', 'max_threads')
+        if 'sqlite' in conf.get('core', 'sql_alchemy_conn') and self._parallelism > 1:
+            self.log.error("Cannot use more than 1 thread when using sqlite. "
+                           "Setting parallelism to 1")
+            self._parallelism = 1
+
+        # Parse and schedule each file no faster than this interval.
+        self._file_process_interval = conf.getint('scheduler',
+                                                  'min_file_process_interval')
+        # How often to print out DAG file processing stats to the log. Default to
+        # 30 seconds.
+        self.print_stats_interval = conf.getint('scheduler',
+                                                'print_stats_interval')
+        # How many seconds do we wait for tasks to heartbeat before mark them as zombies.
+        self._zombie_threshold_secs = (
+            conf.getint('scheduler', 'scheduler_zombie_task_threshold'))
         # Map from file path to the processor
         self._processors = {}
         # Map from file path to the last runtime
         self._last_runtime = {}
         # Map from file path to the last finish time
         self._last_finish_time = {}
+        self._last_zombie_query_time = timezone.utcnow()
+        # Last time that the DAG dir was traversed to look for files
+        self.last_dag_dir_refresh_time = timezone.utcnow()
+        # Last time stats were printed
+        self.last_stat_print_time = timezone.datetime(2000, 1, 1)
+        # TODO: Remove magic number
+        self._zombie_query_interval = 10
         # Map from file path to the number of runs
         self._run_count = defaultdict(int)
-        # Scheduler heartbeat key.
+        # Manager heartbeat key.
         self._heart_beat_key = 'heart-beat'
 
+        # How often to scan the DAGs directory for new files. Default to 5 minutes.
+        self.dag_dir_list_interval = conf.getint('scheduler',
+                                                 'dag_dir_list_interval')
+
+        self._log = logging.getLogger('airflow.processor_manager')
+
+        signal.signal(signal.SIGINT, self._exit_gracefully)
+        signal.signal(signal.SIGTERM, self._exit_gracefully)
+
+    def _exit_gracefully(self, signum, frame):
+        """
+        Helper method to clean up DAG file processors to avoid leaving orphan processes.
+        """
+        self.log.info("Exiting gracefully upon receiving signal %s", signum)
+        self.terminate()
+        self.end()
+        self.log.debug("Finished terminating DAG processors.")
+        sys.exit(os.EX_OK)
+
+    def start(self):
+        """
+        Use multiple processes to parse and generate tasks for the
+        DAGs in parallel. By processing them in separate processes,
+        we can get parallelism and isolation from potentially harmful
+        user code.
+        """
+
+        self.log.info("Processing files using up to %s processes at a time ", self._parallelism)
+        self.log.info("Process each file at most once every %s seconds", self._file_process_interval)
+        self.log.info(
+            "Checking for new files in %s every %s seconds", self._dag_directory, self.dag_dir_list_interval
+        )
+
+        if self._async_mode:
+            self.log.debug("Starting DagFileProcessorManager in async mode")
+            self.start_in_async()
+        else:
+            self.log.debug("Starting DagFileProcessorManager in sync mode")
+            self.start_in_sync()
+
+    def start_in_async(self):
+        """
+        Parse DAG files repeatedly in a standalone loop.
+        """
+        while True:
+            loop_start_time = time.time()
+
+            if self._signal_conn.poll():
+                agent_signal = self._signal_conn.recv()
+                if agent_signal == DagParsingSignal.TERMINATE_MANAGER:
+                    self.terminate()
+                    break
+                elif agent_signal == DagParsingSignal.END_MANAGER:
+                    self.end()
+                    sys.exit(os.EX_OK)
+
+            self._refresh_dag_dir()
+
+            simple_dags = self.heartbeat()
+            for simple_dag in simple_dags:
+                self._result_queue.put(simple_dag)
+
+            self._print_stat()
+
+            all_files_processed = all(self.get_last_finish_time(x) is not None
+                                      for x in self.file_paths)
+            max_runs_reached = self.max_runs_reached()
+
+            dag_parsing_stat = DagParsingStat(self._file_paths,
+                                              self.get_all_pids(),
+                                              max_runs_reached,
+                                              all_files_processed,
+                                              len(simple_dags))
+            self._stat_queue.put(dag_parsing_stat)
+
+            if max_runs_reached:
+                self.log.info("Exiting dag parsing loop as all files "
+                              "have been processed %s times", self._max_runs)
+                break
+
+            loop_duration = time.time() - loop_start_time
+            if loop_duration < 1:
+                sleep_length = 1 - loop_duration
+                self.log.debug("Sleeping for %.2f seconds to prevent excessive logging", sleep_length)
+                time.sleep(sleep_length)
+
+    def start_in_sync(self):
+        """
+        Parse DAG files in a loop controlled by DagParsingSignal.
+        Actual DAG parsing loop will run once upon receiving one
+        agent heartbeat message and will report done when finished the loop.
+        """
+        while True:
+            agent_signal = self._signal_conn.recv()
+            if agent_signal == DagParsingSignal.TERMINATE_MANAGER:
+                self.terminate()
+                break
+            elif agent_signal == DagParsingSignal.END_MANAGER:
+                self.end()
+                sys.exit(os.EX_OK)
+            elif agent_signal == DagParsingSignal.AGENT_HEARTBEAT:
+
+                self._refresh_dag_dir()
+
+                simple_dags = self.heartbeat()
+                for simple_dag in simple_dags:
+                    self._result_queue.put(simple_dag)
+
+                self._print_stat()
+
+                all_files_processed = all(self.get_last_finish_time(x) is not None
+                                          for x in self.file_paths)
+                max_runs_reached = self.max_runs_reached()
+
+                dag_parsing_stat = DagParsingStat(self._file_paths,
+                                                  self.get_all_pids(),
+                                                  self.max_runs_reached(),
+                                                  all_files_processed,
+                                                  len(simple_dags))
+                self._stat_queue.put(dag_parsing_stat)
+
+                self.wait_until_finished()
+                self._signal_conn.send(DagParsingSignal.MANAGER_DONE)
+
+                if max_runs_reached:
+                    self.log.info("Exiting dag parsing loop as all files "
+                                  "have been processed %s times", self._max_runs)
+                    self._signal_conn.send(DagParsingSignal.MANAGER_DONE)
+                    break
+
+    def _refresh_dag_dir(self):
+        """
+        Refresh file paths from dag dir if we haven't done it for too long.
+        """
+        elapsed_time_since_refresh = (timezone.utcnow() -
+                                      self.last_dag_dir_refresh_time).total_seconds()
+        if elapsed_time_since_refresh > self.dag_dir_list_interval:
+            # Build up a list of Python files that could contain DAGs
+            self.log.info("Searching for files in %s", self._dag_directory)
+            self._file_paths = list_py_file_paths(self._dag_directory)
+            self.last_dag_dir_refresh_time = timezone.utcnow()
+            self.log.info("There are %s files in %s", len(self._file_paths), self._dag_directory)
+            self.set_file_paths(self._file_paths)
+
+            try:
+                self.log.debug("Removing old import errors")
+                self.clear_nonexistent_import_errors()
+            except Exception:
+                self.log.exception("Error removing old import errors")
+
+    def _print_stat(self):
+        """
+        Occasionally print out stats about how fast the files are getting processed
+        """
+        if ((timezone.utcnow() - self.last_stat_print_time).total_seconds() >
+                self.print_stats_interval):
+            if len(self._file_paths) > 0:
+                self._log_file_processing_stats(self._file_paths)
+            self.last_stat_print_time = timezone.utcnow()
+
+    @provide_session
+    def clear_nonexistent_import_errors(self, session):
+        """
+        Clears import errors for files that no longer exist.
+
+        :param session: session for ORM operations
+        :type session: sqlalchemy.orm.session.Session
+        """
+        query = session.query(errors.ImportError)
+        if self._file_paths:
+            query = query.filter(
+                ~errors.ImportError.filename.in_(self._file_paths)
+            )
+        query.delete(synchronize_session='fetch')
+        session.commit()
+
+    def _log_file_processing_stats(self, known_file_paths):
+        """
+        Print out stats about how files are getting processed.
+
+        :param known_file_paths: a list of file paths that may contain Airflow
+            DAG definitions
+        :type known_file_paths: list[unicode]
+        :return: None
+        """
+
+        # File Path: Path to the file containing the DAG definition
+        # PID: PID associated with the process that's processing the file. May
+        # be empty.
+        # Runtime: If the process is currently running, how long it's been
+        # running for in seconds.
+        # Last Runtime: If the process ran before, how long did it take to
+        # finish in seconds
+        # Last Run: When the file finished processing in the previous run.
+        headers = ["File Path",
+                   "PID",
+                   "Runtime",
+                   "Last Runtime",
+                   "Last Run"]
+
+        rows = []
+        for file_path in known_file_paths:
+            last_runtime = self.get_last_runtime(file_path)
+            file_name = os.path.basename(file_path)
+            file_name = os.path.splitext(file_name)[0].replace(os.sep, '.')
+            if last_runtime:
+                Stats.gauge(
+                    'dag_processing.last_runtime.{}'.format(file_name),
+                    last_runtime
+                )
+
+            processor_pid = self.get_pid(file_path)
+            processor_start_time = self.get_start_time(file_path)
+            runtime = ((timezone.utcnow() - processor_start_time).total_seconds()
+                       if processor_start_time else None)
+            last_run = self.get_last_finish_time(file_path)
+            if last_run:
+                seconds_ago = (timezone.utcnow() - last_run).total_seconds()
+                Stats.gauge(
+                    'dag_processing.last_run.seconds_ago.{}'.format(file_name),
+                    seconds_ago
+                )
+
+            rows.append((file_path,
+                         processor_pid,
+                         runtime,
+                         last_runtime,
+                         last_run))
+
+        # Sort by longest last runtime. (Can't sort None values in python3)
+        rows = sorted(rows, key=lambda x: x[3] or 0.0)
+
+        formatted_rows = []
+        for file_path, pid, runtime, last_runtime, last_run in rows:
+            formatted_rows.append((file_path,
+                                   pid,
+                                   "{:.2f}s".format(runtime)
+                                   if runtime else None,
+                                   "{:.2f}s".format(last_runtime)
+                                   if last_runtime else None,
+                                   last_run.strftime("%Y-%m-%dT%H:%M:%S")
+                                   if last_run else None))
+        log_str = ("\n" +
+                   "=" * 80 +
+                   "\n" +
+                   "DAG File Processing Stats\n\n" +
+                   tabulate(formatted_rows, headers=headers) +
+                   "\n" +
+                   "=" * 80)
+
+        self.log.info(log_str)
+
     @property
     def file_paths(self):
         return self._file_paths
@@ -363,7 +1019,7 @@ def get_pid(self, file_path):
         :param file_path: the path to the file that's being processed
         :type file_path: unicode
         :return: the PID of the process processing the given file or None if
-        the specified file is not being processed
+            the specified file is not being processed
         :rtype: int
         """
         if file_path in self._processors:
@@ -382,8 +1038,8 @@ def get_runtime(self, file_path):
         :param file_path: the path to the file that's being processed
         :type file_path: unicode
         :return: the current runtime (in seconds) of the process that's
-        processing the specified file or None if the file is not currently
-        being processed
+            processing the specified file or None if the file is not currently
+            being processed
         """
         if file_path in self._processors:
             return (timezone.utcnow() - self._processors[file_path].start_time)\
@@ -395,7 +1051,7 @@ def get_last_runtime(self, file_path):
         :param file_path: the path to the file that was processed
         :type file_path: unicode
         :return: the runtime (in seconds) of the process of the last run, or
-        None if the file was never processed.
+            None if the file was never processed.
         :rtype: float
         """
         return self._last_runtime.get(file_path)
@@ -405,7 +1061,7 @@ def get_last_finish_time(self, file_path):
         :param file_path: the path to the file that was processed
         :type file_path: unicode
         :return: the finish time of the process of the last run, or None if the
-        file was never processed.
+            file was never processed.
         :rtype: datetime
         """
         return self._last_finish_time.get(file_path)
@@ -415,7 +1071,7 @@ def get_start_time(self, file_path):
         :param file_path: the path to the file that's being processed
         :type file_path: unicode
         :return: the start time of the process that's processing the
-        specified file or None if the file is not currently being processed
+            specified file or None if the file is not currently being processed
         :rtype: datetime
         """
         if file_path in self._processors:
@@ -460,13 +1116,13 @@ def wait_until_finished(self):
 
     def heartbeat(self):
         """
-        This should be periodically called by the scheduler. This method will
+        This should be periodically called by the manager loop. This method will
         kick off new processes to process DAG definition files and read the
         results from the finished processors.
 
         :return: a list of SimpleDags that were produced by processors that
-        have finished since the last time this was called
-        :rtype: list[SimpleDag]
+            have finished since the last time this was called
+        :rtype: list[airflow.utils.dag_processing.SimpleDag]
         """
         finished_processors = {}
         """:type : dict[unicode, AbstractDagFileProcessor]"""
@@ -475,7 +1131,7 @@ def heartbeat(self):
 
         for file_path, processor in self._processors.items():
             if processor.done:
-                self.log.info("Processor for %s finished", file_path)
+                self.log.debug("Processor for %s finished", file_path)
                 now = timezone.utcnow()
                 finished_processors[file_path] = processor
                 self._last_runtime[file_path] = (now -
@@ -486,7 +1142,7 @@ def heartbeat(self):
                 running_processors[file_path] = processor
         self._processors = running_processors
 
-        self.log.debug("%s/%s scheduler processes running",
+        self.log.debug("%s/%s DAG parsing processes running",
                        len(self._processors), self._parallelism)
 
         self.log.debug("%s file paths queued for processing",
@@ -512,24 +1168,12 @@ def heartbeat(self):
             file_paths_in_progress = self._processors.keys()
             now = timezone.utcnow()
             file_paths_recently_processed = []
-
-            longest_parse_duration = 0
             for file_path in self._file_paths:
                 last_finish_time = self.get_last_finish_time(file_path)
-                if last_finish_time is not None:
-                    duration = now - last_finish_time
-                    longest_parse_duration = max(duration.total_seconds(),
-                                                 longest_parse_duration)
-                    if duration.total_seconds() < self._process_file_interval:
-                        file_paths_recently_processed.append(file_path)
-
-            sleep_length = max(self._min_file_parsing_loop_time - longest_parse_duration,
-                               0)
-            if sleep_length > 0:
-                self.log.debug("Sleeping for %.2f seconds to prevent excessive "
-                               "logging",
-                               sleep_length)
-                time.sleep(sleep_length)
+                if (last_finish_time is not None and
+                    (now - last_finish_time).total_seconds() <
+                        self._file_process_interval):
+                    file_paths_recently_processed.append(file_path)
 
             files_paths_at_run_limit = [file_path
                                         for file_path, num_runs in self._run_count.items()
@@ -553,24 +1197,61 @@ def heartbeat(self):
 
             self._file_path_queue.extend(files_paths_to_queue)
 
+        zombies = self._find_zombies()
+
         # Start more processors if we have enough slots and files to process
         while (self._parallelism - len(self._processors) > 0 and
                len(self._file_path_queue) > 0):
             file_path = self._file_path_queue.pop(0)
-            processor = self._processor_factory(file_path)
+            processor = self._processor_factory(file_path, zombies)
 
             processor.start()
-            self.log.info(
+            self.log.debug(
                 "Started a process (PID: %s) to generate tasks for %s",
                 processor.pid, file_path
             )
             self._processors[file_path] = processor
 
-        # Update scheduler heartbeat count.
+        # Update heartbeat count.
         self._run_count[self._heart_beat_key] += 1
 
         return simple_dags
 
+    @provide_session
+    def _find_zombies(self, session):
+        """
+        Find zombie task instances, which are tasks haven't heartbeated for too long.
+        :return: Zombie task instances in SimpleTaskInstance format.
+        """
+        now = timezone.utcnow()
+        zombies = []
+        if (now - self._last_zombie_query_time).total_seconds() \
+                > self._zombie_query_interval:
+            # to avoid circular imports
+            from airflow.jobs import LocalTaskJob as LJ
+            self.log.info("Finding 'running' jobs without a recent heartbeat")
+            TI = airflow.models.TaskInstance
+            limit_dttm = timezone.utcnow() - timedelta(
+                seconds=self._zombie_threshold_secs)
+            self.log.info("Failing jobs without heartbeat after %s", limit_dttm)
+
+            tis = (
+                session.query(TI)
+                .join(LJ, TI.job_id == LJ.id)
+                .filter(TI.state == State.RUNNING)
+                .filter(
+                    or_(
+                        LJ.state != State.RUNNING,
+                        LJ.latest_heartbeat < limit_dttm,
+                    )
+                ).all()
+            )
+            self._last_zombie_query_time = timezone.utcnow()
+            for ti in tis:
+                zombies.append(SimpleTaskInstance(ti))
+
+        return zombies
+
     def max_runs_reached(self):
         """
         :return: whether all file paths have been processed max_runs times
@@ -578,7 +1259,7 @@ def max_runs_reached(self):
         if self._max_runs == -1:  # Unlimited runs.
             return False
         for file_path in self._file_paths:
-            if self._run_count[file_path] != self._max_runs:
+            if self._run_count[file_path] < self._max_runs:
                 return False
         if self._run_count[self._heart_beat_key] < self._max_runs:
             return False
@@ -591,3 +1272,40 @@ def terminate(self):
         """
         for processor in self._processors.values():
             processor.terminate()
+
+    def end(self):
+        """
+        Kill all child processes on exit since we don't want to leave
+        them as orphaned.
+        """
+        pids_to_kill = self.get_all_pids()
+        if len(pids_to_kill) > 0:
+            # First try SIGTERM
+            this_process = psutil.Process(os.getpid())
+            # Only check child processes to ensure that we don't have a case
+            # where we kill the wrong process because a child process died
+            # but the PID got reused.
+            child_processes = [x for x in this_process.children(recursive=True)
+                               if x.is_running() and x.pid in pids_to_kill]
+            for child in child_processes:
+                self.log.info("Terminating child PID: %s", child.pid)
+                child.terminate()
+            # TODO: Remove magic number
+            timeout = 5
+            self.log.info("Waiting up to %s seconds for processes to exit...", timeout)
+            try:
+                psutil.wait_procs(
+                    child_processes, timeout=timeout,
+                    callback=lambda x: self.log.info('Terminated PID %s', x.pid))
+            except psutil.TimeoutExpired:
+                self.log.debug("Ran out of time while waiting for processes to exit")
+
+            # Then SIGKILL
+            child_processes = [x for x in this_process.children(recursive=True)
+                               if x.is_running() and x.pid in pids_to_kill]
+            if len(child_processes) > 0:
+                self.log.info("SIGKILL processes that did not terminate gracefully")
+                for child in child_processes:
+                    self.log.info("Killing child PID: %s", child.pid)
+                    child.kill()
+                    child.wait()
diff --git a/airflow/utils/dates.py b/airflow/utils/dates.py
index c147a65430da2..e3340f557f264 100644
--- a/airflow/utils/dates.py
+++ b/airflow/utils/dates.py
@@ -24,12 +24,11 @@
 
 from airflow.utils import timezone
 from datetime import datetime, timedelta
-from dateutil.relativedelta import relativedelta  # for doctest
+from dateutil.relativedelta import relativedelta  # noqa: F401 for doctest
 import six
 
 from croniter import croniter
 
-
 cron_presets = {
     '@hourly': '0 * * * *',
     '@daily': '0 0 * * *',
@@ -39,15 +38,24 @@
 }
 
 
-def date_range(
-        start_date,
-        end_date=None,
-        num=None,
-        delta=None):
+def date_range(start_date, end_date=None, num=None, delta=None):
     """
     Get a set of dates as a list based on a start, end and delta, delta
-    can be something that can be added to ``datetime.datetime``
-    or a cron expression as a ``str``
+    can be something that can be added to `datetime.datetime`
+    or a cron expression as a `str`
+
+    :Example::
+
+        date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1))
+            [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
+            datetime.datetime(2016, 1, 3, 0, 0)]
+        date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *')
+            [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
+            datetime.datetime(2016, 1, 3, 0, 0)]
+        date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *")
+            [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0),
+            datetime.datetime(2016, 3, 1, 0, 0)]
+
     :param start_date: anchor date to start the series from
     :type start_date: datetime.datetime
     :param end_date: right boundary for the date range
@@ -56,15 +64,6 @@ def date_range(
         number of entries you want in the range. This number can be negative,
         output will always be sorted regardless
     :type num: int
-    >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1))
-    [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
-     datetime.datetime(2016, 1, 3, 0, 0)]
-    >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *')
-    [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0),
-     datetime.datetime(2016, 1, 3, 0, 0)]
-    >>> date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *")
-    [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0),
-     datetime.datetime(2016, 3, 1, 0, 0)]
     """
     if not delta:
         return []
@@ -160,7 +159,7 @@ def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)):
     # We first search an upper limit for i for which start_date + upper * delta
     # exceeds dt.
     upper = 1
-    while start_date + upper*delta < dt:
+    while start_date + upper * delta < dt:
         # To speed up finding an upper limit we grow this exponentially by a
         # factor of 2
         upper *= 2
@@ -177,20 +176,18 @@ def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)):
         # Invariant: start + lower * delta < dt <= start + upper * delta
         # If start_date + (lower + 1)*delta exceeds dt, then either lower or
         # lower+1 has to be the solution we are searching for
-        if start_date + (lower + 1)*delta >= dt:
+        if start_date + (lower + 1) * delta >= dt:
             # Check if start_date + (lower + 1)*delta or
             # start_date + lower*delta is closer to dt and return the solution
-            if (
-                    (start_date + (lower + 1) * delta) - dt <=
-                    dt - (start_date + lower * delta)):
-                return start_date + (lower + 1)*delta
+            if (start_date + (lower + 1) * delta) - dt <= dt - (start_date + lower * delta):
+                return start_date + (lower + 1) * delta
             else:
                 return start_date + lower * delta
 
         # We intersect the interval and either replace the lower or upper
         # limit with the candidate
         candidate = lower + (upper - lower) // 2
-        if start_date + candidate*delta >= dt:
+        if start_date + candidate * delta >= dt:
             upper = candidate
         else:
             lower = candidate
@@ -209,11 +206,11 @@ def infer_time_unit(time_seconds_arr):
     if len(time_seconds_arr) == 0:
         return 'hours'
     max_time_seconds = max(time_seconds_arr)
-    if max_time_seconds <= 60*2:
+    if max_time_seconds <= 60 * 2:
         return 'seconds'
-    elif max_time_seconds <= 60*60*2:
+    elif max_time_seconds <= 60 * 60 * 2:
         return 'minutes'
-    elif max_time_seconds <= 24*60*60*2:
+    elif max_time_seconds <= 24 * 60 * 60 * 2:
         return 'hours'
     else:
         return 'days'
@@ -224,11 +221,11 @@ def scale_time_units(time_seconds_arr, unit):
     Convert an array of time durations in seconds to the specified time unit.
     """
     if unit == 'minutes':
-        return list(map(lambda x: x*1.0/60, time_seconds_arr))
+        return list(map(lambda x: x * 1.0 / 60, time_seconds_arr))
     elif unit == 'hours':
-        return list(map(lambda x: x*1.0/(60*60), time_seconds_arr))
+        return list(map(lambda x: x * 1.0 / (60 * 60), time_seconds_arr))
     elif unit == 'days':
-        return list(map(lambda x: x*1.0/(24*60*60), time_seconds_arr))
+        return list(map(lambda x: x * 1.0 / (24 * 60 * 60), time_seconds_arr))
     return time_seconds_arr
 
 
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index b5e0c49c61d33..da3cc5c36e9b4 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -41,9 +41,8 @@ def create_session():
     session = settings.Session()
     try:
         yield session
-        session.expunge_all()
         session.commit()
-    except:
+    except Exception:
         session.rollback()
         raise
     finally:
@@ -78,9 +77,8 @@ def wrapper(*args, **kwargs):
 
 @provide_session
 def merge_conn(conn, session=None):
-    from airflow import models
-    C = models.Connection
-    if not session.query(C).filter(C.conn_id == conn.conn_id).first():
+    from airflow.models.connection import Connection
+    if not session.query(Connection).filter(Connection.conn_id == conn.conn_id).first():
         session.add(conn)
         session.commit()
 
@@ -89,142 +87,142 @@ def initdb(rbac=False):
     session = settings.Session()
 
     from airflow import models
+    from airflow.models.connection import Connection
     upgradedb()
 
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='airflow_db', conn_type='mysql',
-            host='localhost', login='root', password='',
+            host='mysql', login='root', password='',
             schema='airflow'))
     merge_conn(
-        models.Connection(
-            conn_id='airflow_ci', conn_type='mysql',
-            host='localhost', login='root', extra="{\"local_infile\": true}",
-            schema='airflow_ci'))
-    merge_conn(
-        models.Connection(
-            conn_id='beeline_default', conn_type='beeline', port="10000",
+        Connection(
+            conn_id='beeline_default', conn_type='beeline', port=10000,
             host='localhost', extra="{\"use_beeline\": true, \"auth\": \"\"}",
             schema='default'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='bigquery_default', conn_type='google_cloud_platform',
             schema='default'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='local_mysql', conn_type='mysql',
             host='localhost', login='airflow', password='airflow',
             schema='airflow'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='presto_default', conn_type='presto',
             host='localhost',
             schema='hive', port=3400))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='google_cloud_default', conn_type='google_cloud_platform',
             schema='default',))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='hive_cli_default', conn_type='hive_cli',
             schema='default',))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='hiveserver2_default', conn_type='hiveserver2',
             host='localhost',
             schema='default', port=10000))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='metastore_default', conn_type='hive_metastore',
             host='localhost', extra="{\"authMechanism\": \"PLAIN\"}",
             port=9083))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='mongo_default', conn_type='mongo',
-            host='localhost', port=27017))
+            host='mongo', port=27017))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='mysql_default', conn_type='mysql',
             login='root',
-            host='localhost'))
+            schema='airflow',
+            host='mysql'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='postgres_default', conn_type='postgres',
             login='postgres',
+            password='airflow',
             schema='airflow',
-            host='localhost'))
+            host='postgres'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='sqlite_default', conn_type='sqlite',
             host='/tmp/sqlite_default.db'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='http_default', conn_type='http',
             host='https://www.google.com/'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='mssql_default', conn_type='mssql',
             host='localhost', port=1433))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='vertica_default', conn_type='vertica',
             host='localhost', port=5433))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='wasb_default', conn_type='wasb',
             extra='{"sas_token": null}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='webhdfs_default', conn_type='hdfs',
             host='localhost', port=50070))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='ssh_default', conn_type='ssh',
             host='localhost'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='sftp_default', conn_type='sftp',
-            host='localhost', port=22, login='travis',
+            host='localhost', port=22, login='airflow',
             extra='''
-                {"private_key": "~/.ssh/id_rsa", "ignore_hostkey_verification": true}
+                {"key_file": "~/.ssh/id_rsa", "no_host_key_check": true}
             '''))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='fs_default', conn_type='fs',
             extra='{"path": "/"}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='aws_default', conn_type='aws',
             extra='{"region_name": "us-east-1"}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='spark_default', conn_type='spark',
             host='yarn', extra='{"queue": "root.default"}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='druid_broker_default', conn_type='druid',
             host='druid-broker', port=8082, extra='{"endpoint": "druid/v2/sql"}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='druid_ingest_default', conn_type='druid',
             host='druid-overlord', port=8081, extra='{"endpoint": "druid/indexer/v1/task"}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='redis_default', conn_type='redis',
-            host='localhost', port=6379,
+            host='redis', port=6379,
             extra='{"db": 0}'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='sqoop_default', conn_type='sqoop',
             host='rmdbs', extra=''))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='emr_default', conn_type='emr',
             extra='''
                 {   "Name": "default_job_flow_name",
                     "LogUri": "s3://my-emr-log-bucket/default_job_flow_location",
                     "ReleaseLabel": "emr-4.6.0",
                     "Instances": {
+                        "Ec2KeyName": "mykey",
+                        "Ec2SubnetId": "somesubnet",
                         "InstanceGroups": [
                             {
                                 "Name": "Master nodes",
@@ -240,12 +238,10 @@ def initdb(rbac=False):
                                 "InstanceType": "r3.2xlarge",
                                 "InstanceCount": 1
                             }
-                        ]
+                        ],
+                        "TerminationProtected": false,
+                        "KeepJobFlowAliveWhenNoSteps": false
                     },
-                    "Ec2KeyName": "mykey",
-                    "KeepJobFlowAliveWhenNoSteps": false,
-                    "TerminationProtected": false,
-                    "Ec2SubnetId": "somesubnet",
                     "Applications":[
                         { "Name": "Spark" }
                     ],
@@ -265,25 +261,41 @@ def initdb(rbac=False):
                 }
             '''))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='databricks_default', conn_type='databricks',
             host='localhost'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='qubole_default', conn_type='qubole',
-            host= 'localhost'))
+            host='localhost'))
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='segment_default', conn_type='segment',
             extra='{"write_key": "my-segment-write-key"}')),
     merge_conn(
-        models.Connection(
+        Connection(
             conn_id='azure_data_lake_default', conn_type='azure_data_lake',
             extra='{"tenant": "<TENANT>", "account_name": "<ACCOUNTNAME>" }'))
     merge_conn(
-        models.Connection(
+        Connection(
+            conn_id='azure_cosmos_default', conn_type='azure_cosmos',
+            extra='{"database_name": "<DATABASE_NAME>", "collection_name": "<COLLECTION_NAME>" }'))
+    merge_conn(
+        Connection(
+            conn_id='azure_container_instances_default', conn_type='azure_container_instances',
+            extra='{"tenantId": "<TENANT>", "subscriptionId": "<SUBSCRIPTION ID>" }'))
+    merge_conn(
+        Connection(
             conn_id='cassandra_default', conn_type='cassandra',
-            host='localhost', port=9042))
+            host='cassandra', port=9042))
+    merge_conn(
+        Connection(
+            conn_id='dingding_default', conn_type='http',
+            host='', password=''))
+    merge_conn(
+        Connection(
+            conn_id='opsgenie_default', conn_type='http',
+            host='', password=''))
 
     # Known event types
     KET = models.KnownEventType
@@ -341,8 +353,8 @@ def upgradedb():
     package_dir = os.path.normpath(os.path.join(current_dir, '..'))
     directory = os.path.join(package_dir, 'migrations')
     config = Config(os.path.join(package_dir, 'alembic.ini'))
-    config.set_main_option('script_location', directory)
-    config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN)
+    config.set_main_option('script_location', directory.replace('%', '%%'))
+    config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN.replace('%', '%%'))
     command.upgrade(config, 'heads')
 
 
@@ -357,7 +369,7 @@ def resetdb(rbac):
 
     log.info("Dropping tables that exist")
 
-    models.Base.metadata.drop_all(settings.engine)
+    models.base.Base.metadata.drop_all(settings.engine)
     mc = MigrationContext.configure(settings.engine)
     if mc._version.exists(settings.engine):
         mc._version.drop(settings.engine)
diff --git a/airflow/utils/decorators.py b/airflow/utils/decorators.py
index 966d27e30aeae..15847cf050857 100644
--- a/airflow/utils/decorators.py
+++ b/airflow/utils/decorators.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -100,5 +100,22 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 if 'BUILDING_AIRFLOW_DOCS' in os.environ:
+    # flake8: noqa: F811
     # Monkey patch hook to get good function headers while building docs
     apply_defaults = lambda x: x
+
+
+class cached_property:
+    """
+    A decorator creating a property, the value of which is calculated only once and cached for later use.
+    """
+    def __init__(self, func):
+        self.func = func
+        self.__doc__ = getattr(func, '__doc__')
+
+    def __get__(self, instance, cls=None):
+        if instance is None:
+            return self
+        result = self.func(instance)
+        instance.__dict__[self.func.__name__] = result
+        return result
diff --git a/airflow/utils/email.py b/airflow/utils/email.py
index b37e3d471a413..532880161eaf9 100644
--- a/airflow/utils/email.py
+++ b/airflow/utils/email.py
@@ -22,7 +22,6 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from builtins import str
 from past.builtins import basestring
 
 import importlib
@@ -48,6 +47,9 @@ def send_email(to, subject, html_content,
     path, attr = configuration.conf.get('email', 'EMAIL_BACKEND').rsplit('.', 1)
     module = importlib.import_module(path)
     backend = getattr(module, attr)
+    to = get_email_address_list(to)
+    to = ", ".join(to)
+
     return backend(to, subject, html_content, files=files,
                    dryrun=dryrun, cc=cc, bcc=bcc,
                    mime_subtype=mime_subtype, mime_charset=mime_charset, **kwargs)
@@ -62,13 +64,13 @@ def send_email_smtp(to, subject, html_content, files=None,
 
     >>> send_email('test@example.com', 'foo', '<b>Foo</b> bar', ['/dev/null'], dryrun=True)
     """
-    SMTP_MAIL_FROM = configuration.conf.get('smtp', 'SMTP_MAIL_FROM')
+    smtp_mail_from = configuration.conf.get('smtp', 'SMTP_MAIL_FROM')
 
     to = get_email_address_list(to)
 
     msg = MIMEMultipart(mime_subtype)
     msg['Subject'] = subject
-    msg['From'] = SMTP_MAIL_FROM
+    msg['From'] = smtp_mail_from
     msg['To'] = ", ".join(to)
     recipients = to
     if cc:
@@ -96,7 +98,7 @@ def send_email_smtp(to, subject, html_content, files=None,
             part['Content-ID'] = '<%s>' % basename
             msg.attach(part)
 
-    send_MIME_email(SMTP_MAIL_FROM, recipients, msg, dryrun)
+    send_MIME_email(smtp_mail_from, recipients, msg, dryrun)
 
 
 def send_MIME_email(e_from, e_to, mime_msg, dryrun=False):
@@ -129,9 +131,9 @@ def send_MIME_email(e_from, e_to, mime_msg, dryrun=False):
 def get_email_address_list(address_string):
     if isinstance(address_string, basestring):
         if ',' in address_string:
-            address_string = address_string.split(',')
+            address_string = [address.strip() for address in address_string.split(',')]
         elif ';' in address_string:
-            address_string = address_string.split(';')
+            address_string = [address.strip() for address in address_string.split(';')]
         else:
             address_string = [address_string]
 
diff --git a/airflow/utils/file.py b/airflow/utils/file.py
index 88938d422b3af..918375fe5e5b5 100644
--- a/airflow/utils/file.py
+++ b/airflow/utils/file.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py
index 911890dc30ab5..06229079338b9 100644
--- a/airflow/utils/helpers.py
+++ b/airflow/utils/helpers.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,19 +22,20 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import errno
+import imp
+import sys
+import warnings
+
 import psutil
 
 from builtins import input
 from past.builtins import basestring
 from datetime import datetime
 from functools import reduce
-import imp
 import os
 import re
 import signal
-import subprocess
-import sys
-import warnings
 
 from jinja2 import Template
 
@@ -127,7 +128,7 @@ def chunks(items, chunk_size):
     """
     Yield successive chunks of a given size from a list of items
     """
-    if (chunk_size <= 0):
+    if chunk_size <= 0:
         raise ValueError('Chunk size must be a positive integer')
     for i in range(0, len(items), chunk_size):
         yield items[i:i + chunk_size]
@@ -171,6 +172,37 @@ def chain(*tasks):
         up_task.set_downstream(down_task)
 
 
+def cross_downstream(from_tasks, to_tasks):
+    r"""
+    Set downstream dependencies for all tasks in from_tasks to all tasks in to_tasks.
+    E.g.: cross_downstream(from_tasks=[t1, t2, t3], to_tasks=[t4, t5, t6])
+    Is equivalent to:
+
+    t1 --> t4
+       \ /
+    t2 -X> t5
+       / \
+    t3 --> t6
+
+    t1.set_downstream(t4)
+    t1.set_downstream(t5)
+    t1.set_downstream(t6)
+    t2.set_downstream(t4)
+    t2.set_downstream(t5)
+    t2.set_downstream(t6)
+    t3.set_downstream(t4)
+    t3.set_downstream(t5)
+    t3.set_downstream(t6)
+
+    :param from_tasks: List of tasks to start from.
+    :type from_tasks: List[airflow.models.BaseOperator]
+    :param to_tasks: List of tasks to set as downstream dependencies.
+    :type to_tasks: List[airflow.models.BaseOperator]
+    """
+    for task in from_tasks:
+        task.set_downstream(to_tasks)
+
+
 def pprinttable(rows):
     """Returns a pretty ascii table from tuples
 
@@ -226,6 +258,7 @@ def reap_process_group(pid, log, sig=signal.SIGTERM,
     :param sig: signal type
     :param timeout: how much time a process has to terminate
     """
+
     def on_terminate(p):
         log.info("Process %s (%s) terminated with exit code %s", p, p.pid, p.returncode)
 
@@ -237,7 +270,15 @@ def on_terminate(p):
     children = parent.children(recursive=True)
     children.append(parent)
 
-    log.info("Sending %s to GPID %s", sig, os.getpgid(pid))
+    try:
+        pg = os.getpgid(pid)
+    except OSError as err:
+        # Skip if not such process - we experience a race and it just terminated
+        if err.errno == errno.ESRCH:
+            return
+        raise
+
+    log.info("Sending %s to GPID %s", sig, pg)
     os.killpg(os.getpgid(pid), sig)
 
     gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate)
@@ -281,10 +322,10 @@ def __init__(self, parent_module, module_attributes):
         """
         :param parent_module: The string package name of the parent module. For
             example, 'airflow.operators'
-        :type parent_module: string
+        :type parent_module: str
         :param module_attributes: The file to class mappings for all importable
             classes.
-        :type module_attributes: string
+        :type module_attributes: str
         """
         self._parent_module = parent_module
         self._attribute_modules = self._build_attribute_modules(module_attributes)
@@ -374,3 +415,23 @@ def __getattr__(self, attribute):
             return loaded_attribute
 
         raise AttributeError
+
+
+def render_log_filename(ti, try_number, filename_template):
+    """
+    Given task instance, try_number, filename_template, return the rendered log filename
+
+    :param ti: task instance
+    :param try_number: try_number of the task
+    :param filename_template: filename template, which can be jinja template or python string template
+    """
+    filename_template, filename_jinja_template = parse_template_string(filename_template)
+    if filename_jinja_template:
+        jinja_context = ti.get_template_context()
+        jinja_context['try_number'] = try_number
+        return filename_jinja_template.render(**jinja_context)
+
+    return filename_template.format(dag_id=ti.dag_id,
+                                    task_id=ti.task_id,
+                                    execution_date=ti.execution_date.isoformat(),
+                                    try_number=try_number)
diff --git a/airflow/utils/json.py b/airflow/utils/json.py
index 434926f9d2be8..1767c523d6a3f 100644
--- a/airflow/utils/json.py
+++ b/airflow/utils/json.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/utils/log/__init__.py b/airflow/utils/log/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/airflow/utils/log/__init__.py
+++ b/airflow/utils/log/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/utils/log/es_task_handler.py b/airflow/utils/log/es_task_handler.py
index 04682318655b1..3d4273cb5d4d3 100644
--- a/airflow/utils/log/es_task_handler.py
+++ b/airflow/utils/log/es_task_handler.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Using `from elasticsearch import *` would break elasticseach mocking used in unit test.
+# Using `from elasticsearch import *` would break elasticsearch mocking used in unit test.
 import elasticsearch
 import pendulum
 from elasticsearch_dsl import Search
@@ -87,7 +87,7 @@ def _read(self, ti, try_number, metadata=None):
         :param try_number: try_number of the task instance
         :param metadata: log metadata,
                          can be used for steaming log reading and auto-tailing.
-        :return a list of log documents and metadata.
+        :return: a list of log documents and metadata.
         """
         if not metadata:
             metadata = {'offset': 0}
@@ -147,9 +147,7 @@ def es_read(self, log_id, offset):
                 logs = s[self.MAX_LINE_PER_PAGE * self.PAGE:self.MAX_LINE_PER_PAGE] \
                     .execute()
             except Exception as e:
-                msg = 'Could not read log with log_id: {}, ' \
-                      'error: {}'.format(log_id, str(e))
-                self.log.exception(msg)
+                self.log.exception('Could not read log with log_id: %s, error: %s', log_id, str(e))
 
         return logs
 
diff --git a/airflow/utils/log/file_processor_handler.py b/airflow/utils/log/file_processor_handler.py
index 5e637440a1fda..098b018d47275 100644
--- a/airflow/utils/log/file_processor_handler.py
+++ b/airflow/utils/log/file_processor_handler.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,11 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import errno
 import logging
 import os
 
-from airflow import configuration as conf
+from airflow import settings
 from airflow.utils.helpers import parse_template_string
 from datetime import datetime
 
@@ -41,7 +40,7 @@ def __init__(self, base_log_folder, filename_template):
         super(FileProcessorHandler, self).__init__()
         self.handler = None
         self.base_log_folder = base_log_folder
-        self.dag_dir = os.path.expanduser(conf.get('core', 'DAGS_FOLDER'))
+        self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER)
         self.filename_template, self.filename_jinja_template = \
             parse_template_string(filename_template)
 
@@ -49,9 +48,9 @@ def __init__(self, base_log_folder, filename_template):
         if not os.path.exists(self._get_log_directory()):
             try:
                 os.makedirs(self._get_log_directory())
-            except OSError as e:
+            except OSError:
                 # only ignore case where the directory already exist
-                if e.errno != errno.EEXIST:
+                if not os.path.isdir(self._get_log_directory()):
                     raise
 
                 logging.warning("%s already exists", self._get_log_directory())
@@ -116,7 +115,7 @@ def _symlink_latest_log_directory(self):
                         os.unlink(latest_log_directory_path)
                         os.symlink(log_directory, latest_log_directory_path)
                 elif (os.path.isdir(latest_log_directory_path) or
-                          os.path.isfile(latest_log_directory_path)):
+                      os.path.isfile(latest_log_directory_path)):
                     logging.warning(
                         "%s already exists as a dir/file. Skip creating symlink.",
                         latest_log_directory_path
@@ -131,14 +130,18 @@ def _init_file(self, filename):
         """
         Create log file and directory if required.
         :param filename: task instance object
-        :return relative log path of the given task instance
+        :return: relative log path of the given task instance
         """
         relative_path = self._render_filename(filename)
         full_path = os.path.join(self._get_log_directory(), relative_path)
         directory = os.path.dirname(full_path)
 
         if not os.path.exists(directory):
-            os.makedirs(directory)
+            try:
+                os.makedirs(directory)
+            except OSError:
+                if not os.path.isdir(directory):
+                    raise
 
         if not os.path.exists(full_path):
             open(full_path, "a").close()
diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py
index 627c8755707ef..111f26c279cf0 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -171,7 +171,7 @@ def _init_file(self, ti):
         """
         Create log directory and give it correct permissions.
         :param ti: task instance object
-        :return relative log path of the given task instance
+        :return: relative log path of the given task instance
         """
         # To handle log writing when tasks are impersonated, the log files need to
         # be writable by the user that runs the Airflow command and the user
diff --git a/airflow/utils/log/gcs_task_handler.py b/airflow/utils/log/gcs_task_handler.py
index 8c34792bb2138..ed7edf4a27010 100644
--- a/airflow/utils/log/gcs_task_handler.py
+++ b/airflow/utils/log/gcs_task_handler.py
@@ -49,8 +49,8 @@ def _build_hook(self):
         except Exception as e:
             self.log.error(
                 'Could not create a GoogleCloudStorageHook with connection id '
-                '"{}". {}\n\nPlease make sure that airflow[gcp_api] is installed '
-                'and the GCS connection exists.'.format(remote_conn_id, str(e))
+                '"%s". %s\n\nPlease make sure that airflow[gcp_api] is installed '
+                'and the GCS connection exists.', remote_conn_id, str(e)
             )
 
     @property
@@ -126,7 +126,7 @@ def gcs_read(self, remote_log_location):
         """
         Returns the log found at the remote_log_location.
         :param remote_log_location: the log's location in remote storage
-        :type remote_log_location: string (path)
+        :type remote_log_location: str (path)
         """
         bkt, blob = self.parse_gcs_url(remote_log_location)
         return self.hook.download(bkt, blob).decode()
@@ -136,9 +136,9 @@ def gcs_write(self, log, remote_log_location, append=True):
         Writes the log to the remote_log_location. Fails silently if no hook
         was created.
         :param log: the log to write to the remote_log_location
-        :type log: string
+        :type log: str
         :param remote_log_location: the log's location in remote storage
-        :type remote_log_location: string (path)
+        :type remote_log_location: str (path)
         :param append: if False, any existing log file is overwritten. If True,
             the new log is appended to any existing logs.
         :type append: bool
@@ -164,7 +164,8 @@ def gcs_write(self, log, remote_log_location, append=True):
         except Exception as e:
             self.log.error('Could not write logs to %s: %s', remote_log_location, e)
 
-    def parse_gcs_url(self, gsurl):
+    @staticmethod
+    def parse_gcs_url(gsurl):
         """
         Given a Google Cloud Storage URL (gs://<bucket>/<blob>), returns a
         tuple containing the corresponding bucket and blob.
diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py
index 3f696931c955b..442fecd0363c8 100644
--- a/airflow/utils/log/logging_mixin.py
+++ b/airflow/utils/log/logging_mixin.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -92,7 +92,7 @@ def write(self, message):
             self._buffer += message
         else:
             self._buffer += message
-            self.logger.log(self.level, self._buffer)
+            self.logger.log(self.level, self._buffer.rstrip())
             self._buffer = str()
 
     def flush(self):
diff --git a/airflow/utils/log/s3_task_handler.py b/airflow/utils/log/s3_task_handler.py
index 07b9b3ec11e8b..196aec50f1bd4 100644
--- a/airflow/utils/log/s3_task_handler.py
+++ b/airflow/utils/log/s3_task_handler.py
@@ -42,7 +42,7 @@ def _build_hook(self):
         try:
             from airflow.hooks.S3_hook import S3Hook
             return S3Hook(remote_conn_id)
-        except:
+        except Exception:
             self.log.error(
                 'Could not create an S3Hook with connection id "%s". '
                 'Please make sure that airflow[s3] is installed and '
@@ -132,14 +132,14 @@ def s3_read(self, remote_log_location, return_error=False):
         Returns the log found at the remote_log_location. Returns '' if no
         logs are found or there is an error.
         :param remote_log_location: the log's location in remote storage
-        :type remote_log_location: string (path)
+        :type remote_log_location: str (path)
         :param return_error: if True, returns a string error message if an
             error occurs. Otherwise returns '' when an error occurs.
         :type return_error: bool
         """
         try:
             return self.hook.read_key(remote_log_location)
-        except:
+        except Exception:
             msg = 'Could not read logs from {}'.format(remote_log_location)
             self.log.exception(msg)
             # return error if needed
@@ -151,9 +151,9 @@ def s3_write(self, log, remote_log_location, append=True):
         Writes the log to the remote_log_location. Fails silently if no hook
         was created.
         :param log: the log to write to the remote_log_location
-        :type log: string
+        :type log: str
         :param remote_log_location: the log's location in remote storage
-        :type remote_log_location: string (path)
+        :type remote_log_location: str (path)
         :param append: if False, any existing log file is overwritten. If True,
             the new log is appended to any existing logs.
         :type append: bool
@@ -169,5 +169,5 @@ def s3_write(self, log, remote_log_location, append=True):
                 replace=True,
                 encrypt=configuration.conf.getboolean('core', 'ENCRYPT_S3_LOGS'),
             )
-        except:
+        except Exception:
             self.log.exception('Could not write logs to %s', remote_log_location)
diff --git a/airflow/utils/log/wasb_task_handler.py b/airflow/utils/log/wasb_task_handler.py
index a2a0c0daee999..71e1ae5d16dfd 100644
--- a/airflow/utils/log/wasb_task_handler.py
+++ b/airflow/utils/log/wasb_task_handler.py
@@ -20,7 +20,6 @@
 import shutil
 
 from airflow import configuration
-from airflow.contrib.hooks.wasb_hook import WasbHook
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.log.file_task_handler import FileTaskHandler
 from azure.common import AzureHttpError
@@ -47,6 +46,7 @@ def __init__(self, base_log_folder, wasb_log_folder, wasb_container,
     def _build_hook(self):
         remote_conn_id = configuration.get('core', 'REMOTE_LOG_CONN_ID')
         try:
+            from airflow.contrib.hooks.wasb_hook import WasbHook
             return WasbHook(remote_conn_id)
         except AzureHttpError:
             self.log.error(
@@ -140,7 +140,7 @@ def wasb_read(self, remote_log_location, return_error=False):
         Returns the log found at the remote_log_location. Returns '' if no
         logs are found or there is an error.
         :param remote_log_location: the log's location in remote storage
-        :type remote_log_location: string (path)
+        :type remote_log_location: str (path)
         :param return_error: if True, returns a string error message if an
             error occurs. Otherwise returns '' when an error occurs.
         :type return_error: bool
@@ -159,9 +159,9 @@ def wasb_write(self, log, remote_log_location, append=True):
         Writes the log to the remote_log_location. Fails silently if no hook
         was created.
         :param log: the log to write to the remote_log_location
-        :type log: string
+        :type log: str
         :param remote_log_location: the log's location in remote storage
-        :type remote_log_location: string (path)
+        :type remote_log_location: str (path)
         :param append: if False, any existing log file is overwritten. If True,
             the new log is appended to any existing logs.
         :type append: bool
diff --git a/airflow/utils/module_loading.py b/airflow/utils/module_loading.py
index 6e638b00d3318..d5b4971b736dd 100644
--- a/airflow/utils/module_loading.py
+++ b/airflow/utils/module_loading.py
@@ -16,24 +16,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import os
-import sys
 
-from airflow import configuration as conf
 from importlib import import_module
 
 
-def prepare_classpath():
-    """
-    Ensures that the Airflow home directory is on the classpath
-    """
-    config_path = os.path.join(conf.get('core', 'airflow_home'), 'config')
-    config_path = os.path.expanduser(config_path)
-
-    if config_path not in sys.path:
-        sys.path.append(config_path)
-
-
 def import_string(dotted_path):
     """
     Import a dotted module path and return the attribute/class designated by the
@@ -48,7 +34,7 @@ def import_string(dotted_path):
 
     try:
         return getattr(module, class_name)
-    except AttributeError as err:
+    except AttributeError:
         raise ImportError('Module "{}" does not define a "{}" attribute/class'.format(
             module_path, class_name)
         )
diff --git a/airflow/utils/net.py b/airflow/utils/net.py
index 03e8fde9c90cb..c435496ffc76a 100644
--- a/airflow/utils/net.py
+++ b/airflow/utils/net.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/utils/operator_helpers.py b/airflow/utils/operator_helpers.py
index 356aa650a3a40..e015f1d5ef072 100644
--- a/airflow/utils/operator_helpers.py
+++ b/airflow/utils/operator_helpers.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,32 +18,49 @@
 # under the License.
 #
 
+AIRFLOW_VAR_NAME_FORMAT_MAPPING = {
+    'AIRFLOW_CONTEXT_DAG_ID': {'default': 'airflow.ctx.dag_id',
+                               'env_var_format': 'AIRFLOW_CTX_DAG_ID'},
+    'AIRFLOW_CONTEXT_TASK_ID': {'default': 'airflow.ctx.task_id',
+                                'env_var_format': 'AIRFLOW_CTX_TASK_ID'},
+    'AIRFLOW_CONTEXT_EXECUTION_DATE': {'default': 'airflow.ctx.execution_date',
+                                       'env_var_format': 'AIRFLOW_CTX_EXECUTION_DATE'},
+    'AIRFLOW_CONTEXT_DAG_RUN_ID': {'default': 'airflow.ctx.dag_run_id',
+                                   'env_var_format': 'AIRFLOW_CTX_DAG_RUN_ID'}
+}
+
 
-def context_to_airflow_vars(context):
+def context_to_airflow_vars(context, in_env_var_format=False):
     """
     Given a context, this function provides a dictionary of values that can be used to
     externally reconstruct relations between dags, dag_runs, tasks and task_instances.
+    Default to abc.def.ghi format and can be made to ABC_DEF_GHI format if
+    in_env_var_format is set to True.
 
-    :param context: The context for the task_instance of interest
+    :param context: The context for the task_instance of interest.
     :type context: dict
+    :param in_env_var_format: If returned vars should be in ABC_DEF_GHI format.
+    :type in_env_var_format: bool
+    :return: task_instance context as dict.
     """
-    params = {}
-    dag = context.get('dag')
-    if dag and dag.dag_id:
-        params['airflow.ctx.dag.dag_id'] = dag.dag_id
-
-    dag_run = context.get('dag_run')
-    if dag_run and dag_run.execution_date:
-        params['airflow.ctx.dag_run.execution_date'] = dag_run.execution_date.isoformat()
-
-    task = context.get('task')
-    if task and task.task_id:
-        params['airflow.ctx.task.task_id'] = task.task_id
-
+    params = dict()
+    if in_env_var_format:
+        name_format = 'env_var_format'
+    else:
+        name_format = 'default'
     task_instance = context.get('task_instance')
+    if task_instance and task_instance.dag_id:
+        params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_DAG_ID'][
+            name_format]] = task_instance.dag_id
+    if task_instance and task_instance.task_id:
+        params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_TASK_ID'][
+            name_format]] = task_instance.task_id
     if task_instance and task_instance.execution_date:
-        params['airflow.ctx.task_instance.execution_date'] = (
-            task_instance.execution_date.isoformat()
-        )
-
+        params[
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_EXECUTION_DATE'][
+                name_format]] = task_instance.execution_date.isoformat()
+    dag_run = context.get('dag_run')
+    if dag_run and dag_run.run_id:
+        params[AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_DAG_RUN_ID'][
+            name_format]] = dag_run.run_id
     return params
diff --git a/airflow/utils/operator_resources.py b/airflow/utils/operator_resources.py
index 47ff336991b95..44df83eafb508 100644
--- a/airflow/utils/operator_resources.py
+++ b/airflow/utils/operator_resources.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -33,10 +33,10 @@ class Resource(object):
     Represents a resource requirement in an execution environment for an operator.
 
     :param name: Name of the resource
-    :type name: string
+    :type name: str
     :param units_str: The string representing the units of a resource (e.g. MB for a CPU
         resource) to be used for display purposes
-    :type units_str: string
+    :type units_str: str
     :param qty: The number of units of the specified resource that are required for
         execution of the operator.
     :type qty: long
diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py
index a00fe17c18a48..62488ef4e2077 100644
--- a/airflow/utils/sqlalchemy.py
+++ b/airflow/utils/sqlalchemy.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,23 +22,27 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import datetime
 import os
+import json
+import pendulum
 import time
 import random
 
+from dateutil import relativedelta
 from sqlalchemy import event, exc, select
+from sqlalchemy.types import Text, DateTime, TypeDecorator
 
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 log = LoggingMixin().log
+utc = pendulum.timezone('UTC')
 
 
-def setup_event_handlers(
-        engine,
-        reconnect_timeout_seconds,
-        initial_backoff_seconds=0.2,
-        max_backoff_seconds=120):
-
+def setup_event_handlers(engine,
+                         reconnect_timeout_seconds,
+                         initial_backoff_seconds=0.2,
+                         max_backoff_seconds=120):
     @event.listens_for(engine, "engine_connect")
     def ping_connection(connection, branch):
         """
@@ -64,7 +68,7 @@ def ping_connection(connection, branch):
 
             try:
                 connection.scalar(select([1]))
-                # If we made it here then the connection appears to be healty
+                # If we made it here then the connection appears to be healthy
                 break
             except exc.DBAPIError as err:
                 if time.time() - start >= reconnect_timeout_seconds:
@@ -96,11 +100,24 @@ def ping_connection(connection, branch):
                 # restore "close with result"
                 connection.should_close_with_result = save_should_close_with_result
 
-
     @event.listens_for(engine, "connect")
     def connect(dbapi_connection, connection_record):
         connection_record.info['pid'] = os.getpid()
 
+    if engine.dialect.name == "sqlite":
+        @event.listens_for(engine, "connect")
+        def set_sqlite_pragma(dbapi_connection, connection_record):
+            cursor = dbapi_connection.cursor()
+            cursor.execute("PRAGMA foreign_keys=ON")
+            cursor.close()
+
+    # this ensures sanity in mysql when storing datetimes (not required for postgres)
+    if engine.dialect.name == "mysql":
+        @event.listens_for(engine, "connect")
+        def set_mysql_timezone(dbapi_connection, connection_record):
+            cursor = dbapi_connection.cursor()
+            cursor.execute("SET time_zone = '+00:00'")
+            cursor.close()
 
     @event.listens_for(engine, "checkout")
     def checkout(dbapi_connection, connection_record, connection_proxy):
@@ -111,3 +128,79 @@ def checkout(dbapi_connection, connection_record, connection_proxy):
                 "Connection record belongs to pid {}, "
                 "attempting to check out in pid {}".format(connection_record.info['pid'], pid)
             )
+
+
+class UtcDateTime(TypeDecorator):
+    """
+    Almost equivalent to :class:`~sqlalchemy.types.DateTime` with
+    ``timezone=True`` option, but it differs from that by:
+
+    - Never silently take naive :class:`~datetime.datetime`, instead it
+      always raise :exc:`ValueError` unless time zone aware value.
+    - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo`
+      is always converted to UTC.
+    - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.DateTime`,
+      it never return naive :class:`~datetime.datetime`, but time zone
+      aware value, even with SQLite or MySQL.
+    - Always returns DateTime in UTC
+
+    """
+
+    impl = DateTime(timezone=True)
+
+    def process_bind_param(self, value, dialect):
+        if value is not None:
+            if not isinstance(value, datetime.datetime):
+                raise TypeError('expected datetime.datetime, not ' +
+                                repr(value))
+            elif value.tzinfo is None:
+                raise ValueError('naive datetime is disallowed')
+
+            return value.astimezone(utc)
+
+    def process_result_value(self, value, dialect):
+        """
+        Processes DateTimes from the DB making sure it is always
+        returning UTC. Not using timezone.convert_to_utc as that
+        converts to configured TIMEZONE while the DB might be
+        running with some other setting. We assume UTC datetimes
+        in the database.
+        """
+        if value is not None:
+            if value.tzinfo is None:
+                value = value.replace(tzinfo=utc)
+            else:
+                value = value.astimezone(utc)
+
+        return value
+
+
+class Interval(TypeDecorator):
+
+    impl = Text
+
+    attr_keys = {
+        datetime.timedelta: ('days', 'seconds', 'microseconds'),
+        relativedelta.relativedelta: (
+            'years', 'months', 'days', 'leapdays', 'hours', 'minutes', 'seconds', 'microseconds',
+            'year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond',
+        ),
+    }
+
+    def process_bind_param(self, value, dialect):
+        if type(value) in self.attr_keys:
+            attrs = {
+                key: getattr(value, key)
+                for key in self.attr_keys[type(value)]
+            }
+            return json.dumps({'type': type(value).__name__, 'attrs': attrs})
+        return json.dumps(value)
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return value
+        data = json.loads(value)
+        if isinstance(data, dict):
+            type_map = {key.__name__: key for key in self.attr_keys}
+            return type_map[data['type']](**data['attrs'])
+        return data
diff --git a/airflow/utils/state.py b/airflow/utils/state.py
index 9da98510eb03d..320b996d5d500 100644
--- a/airflow/utils/state.py
+++ b/airflow/utils/state.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -43,6 +43,7 @@ class State(object):
     SHUTDOWN = "shutdown"  # External request to shut down
     FAILED = "failed"
     UP_FOR_RETRY = "up_for_retry"
+    UP_FOR_RESCHEDULE = "up_for_reschedule"
     UPSTREAM_FAILED = "upstream_failed"
     SKIPPED = "skipped"
 
@@ -51,7 +52,9 @@ class State(object):
         RUNNING,
         FAILED,
         UPSTREAM_FAILED,
+        SKIPPED,
         UP_FOR_RETRY,
+        UP_FOR_RESCHEDULE,
         QUEUED,
         NONE,
         SCHEDULED,
@@ -70,6 +73,7 @@ class State(object):
         SHUTDOWN: 'blue',
         FAILED: 'red',
         UP_FOR_RETRY: 'gold',
+        UP_FOR_RESCHEDULE: 'turquoise',
         UPSTREAM_FAILED: 'orange',
         SKIPPED: 'pink',
         REMOVED: 'lightgrey',
@@ -79,18 +83,14 @@ class State(object):
 
     @classmethod
     def color(cls, state):
-        if state in cls.state_color:
-            return cls.state_color[state]
-        else:
-            return 'white'
+        return cls.state_color.get(state, 'white')
 
     @classmethod
     def color_fg(cls, state):
         color = cls.color(state)
         if color in ['green', 'red']:
             return 'white'
-        else:
-            return 'black'
+        return 'black'
 
     @classmethod
     def finished(cls):
@@ -101,7 +101,6 @@ def finished(cls):
         """
         return [
             cls.SUCCESS,
-            cls.SHUTDOWN,
             cls.FAILED,
             cls.SKIPPED,
         ]
@@ -117,5 +116,7 @@ def unfinished(cls):
             cls.SCHEDULED,
             cls.QUEUED,
             cls.RUNNING,
-            cls.UP_FOR_RETRY
+            cls.SHUTDOWN,
+            cls.UP_FOR_RETRY,
+            cls.UP_FOR_RESCHEDULE
         ]
diff --git a/airflow/utils/tests.py b/airflow/utils/tests.py
index 6f29ffc25b2eb..33157f5fba9fc 100644
--- a/airflow/utils/tests.py
+++ b/airflow/utils/tests.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -35,5 +35,5 @@ def skipUnlessImported(module, obj):
 
 def assertEqualIgnoreMultipleSpaces(case, first, second, msg=None):
     def _trim(s):
-        re.sub("\s+", " ", s.strip())
+        return re.sub(r"\s+", " ", s.strip())
     return case.assertEqual(_trim(first), _trim(second), msg)
diff --git a/airflow/utils/timeout.py b/airflow/utils/timeout.py
index c5ca5e2f1724f..362baa197f33d 100644
--- a/airflow/utils/timeout.py
+++ b/airflow/utils/timeout.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,7 @@
 from __future__ import unicode_literals
 
 import signal
+import os
 
 from airflow.exceptions import AirflowTaskTimeout
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -35,10 +36,10 @@ class timeout(LoggingMixin):
 
     def __init__(self, seconds=1, error_message='Timeout'):
         self.seconds = seconds
-        self.error_message = error_message
+        self.error_message = error_message + ', PID: ' + str(os.getpid())
 
     def handle_timeout(self, signum, frame):
-        self.log.error("Process timed out")
+        self.log.error("Process timed out, PID: %s", str(os.getpid()))
         raise AirflowTaskTimeout(self.error_message)
 
     def __enter__(self):
diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py
index 6d49fbcbb3e85..5adaa2f5c4a5a 100644
--- a/airflow/utils/timezone.py
+++ b/airflow/utils/timezone.py
@@ -164,9 +164,9 @@ def datetime(*args, **kwargs):
     return dt.datetime(*args, **kwargs)
 
 
-def parse(string):
+def parse(string, timezone=None):
     """
     Parse a time string and return an aware datetime
     :param string: time string
     """
-    return pendulum.parse(string, tz=TIMEZONE)
+    return pendulum.parse(string, tz=timezone or TIMEZONE)
diff --git a/airflow/utils/trigger_rule.py b/airflow/utils/trigger_rule.py
index 7bad8ecb78ab5..a1e14c2e61568 100644
--- a/airflow/utils/trigger_rule.py
+++ b/airflow/utils/trigger_rule.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,6 +20,7 @@
 from __future__ import unicode_literals
 
 from builtins import object
+from typing import Set
 
 
 class TriggerRule(object):
@@ -28,9 +29,12 @@ class TriggerRule(object):
     ALL_DONE = 'all_done'
     ONE_SUCCESS = 'one_success'
     ONE_FAILED = 'one_failed'
+    NONE_FAILED = 'none_failed'
+    NONE_SKIPPED = 'none_skipped'
     DUMMY = 'dummy'
 
-    _ALL_TRIGGER_RULES = {}
+    _ALL_TRIGGER_RULES = set()  # type: Set[str]
+
     @classmethod
     def is_valid(cls, trigger_rule):
         return trigger_rule in cls.all_triggers()
diff --git a/airflow/utils/weight_rule.py b/airflow/utils/weight_rule.py
index 745cc76970cea..f7f85c0734ac6 100644
--- a/airflow/utils/weight_rule.py
+++ b/airflow/utils/weight_rule.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,6 +20,7 @@
 from __future__ import unicode_literals
 
 from builtins import object
+from typing import Set
 
 
 class WeightRule(object):
@@ -27,7 +28,8 @@ class WeightRule(object):
     UPSTREAM = 'upstream'
     ABSOLUTE = 'absolute'
 
-    _ALL_WEIGHT_RULES = {}
+    _ALL_WEIGHT_RULES = set()  # type: Set[str]
+
     @classmethod
     def is_valid(cls, weight_rule):
         return weight_rule in cls.all_weight_rules()
diff --git a/airflow/version.py b/airflow/version.py
index d11d76608dfdf..b7367c22a47e5 100644
--- a/airflow/version.py
+++ b/airflow/version.py
@@ -18,4 +18,4 @@
 # under the License.
 #
 
-version = '2.0.0dev0+incubating'
+version = '1.10.3'
diff --git a/airflow/www/__init__.py b/airflow/www/__init__.py
index f0f8b68337da6..114d189da14ab 100644
--- a/airflow/www/__init__.py
+++ b/airflow/www/__init__.py
@@ -7,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
diff --git a/airflow/www/api/__init__.py b/airflow/www/api/__init__.py
index db5ba598d7c23..b7f8352944d3f 100644
--- a/airflow/www/api/__init__.py
+++ b/airflow/www/api/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www/api/experimental/__init__.py b/airflow/www/api/experimental/__init__.py
index db5ba598d7c23..b7f8352944d3f 100644
--- a/airflow/www/api/experimental/__init__.py
+++ b/airflow/www/api/experimental/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www/api/experimental/endpoints.py b/airflow/www/api/experimental/endpoints.py
index f0bc319eb6665..8ba47fb114a62 100644
--- a/airflow/www/api/experimental/endpoints.py
+++ b/airflow/www/api/experimental/endpoints.py
@@ -24,12 +24,17 @@
 from airflow.api.common.experimental import delete_dag as delete
 from airflow.api.common.experimental import pool as pool_api
 from airflow.api.common.experimental import trigger_dag as trigger
+from airflow.api.common.experimental.get_dag_runs import get_dag_runs
 from airflow.api.common.experimental.get_task import get_task
 from airflow.api.common.experimental.get_task_instance import get_task_instance
+from airflow.api.common.experimental.get_code import get_code
+from airflow.api.common.experimental.get_dag_run_state import get_dag_run_state
 from airflow.exceptions import AirflowException
 from airflow.utils import timezone
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.www.app import csrf
+from airflow import models
+from airflow.utils.db import create_session
 
 _log = LoggingMixin().log
 
@@ -83,7 +88,7 @@ def trigger_dag(dag_id):
         return response
 
     if getattr(g, 'user', None):
-        _log.info("User {} created {}".format(g.user, dr))
+        _log.info("User %s created %s", g.user, dr)
 
     response = jsonify(message="Created {}".format(dr))
     return response
@@ -106,12 +111,47 @@ def delete_dag(dag_id):
     return jsonify(message="Removed {} record(s)".format(count), count=count)
 
 
+@api_experimental.route('/dags/<string:dag_id>/dag_runs', methods=['GET'])
+@requires_authentication
+def dag_runs(dag_id):
+    """
+    Returns a list of Dag Runs for a specific DAG ID.
+    :query param state: a query string parameter '?state=queued|running|success...'
+    :param dag_id: String identifier of a DAG
+    :return: List of DAG runs of a DAG with requested state,
+    or all runs if the state is not specified
+    """
+    try:
+        state = request.args.get('state')
+        dagruns = get_dag_runs(dag_id, state, run_url_route='airflow.graph')
+    except AirflowException as err:
+        _log.info(err)
+        response = jsonify(error="{}".format(err))
+        response.status_code = 400
+        return response
+
+    return jsonify(dagruns)
+
+
 @api_experimental.route('/test', methods=['GET'])
 @requires_authentication
 def test():
     return jsonify(status='OK')
 
 
+@api_experimental.route('/dags/<string:dag_id>/code', methods=['GET'])
+@requires_authentication
+def get_dag_code(dag_id):
+    """Return python code of a given dag_id."""
+    try:
+        return get_code(dag_id)
+    except AirflowException as err:
+        _log.info(err)
+        response = jsonify(error="{}".format(err))
+        response.status_code = err.status_code
+        return response
+
+
 @api_experimental.route('/dags/<string:dag_id>/tasks/<string:task_id>', methods=['GET'])
 @requires_authentication
 def task_info(dag_id, task_id):
@@ -131,6 +171,28 @@ def task_info(dag_id, task_id):
     return jsonify(fields)
 
 
+# ToDo: Shouldn't this be a PUT method?
+@api_experimental.route('/dags/<string:dag_id>/paused/<string:paused>', methods=['GET'])
+@requires_authentication
+def dag_paused(dag_id, paused):
+    """(Un)pauses a dag"""
+
+    DagModel = models.DagModel
+    with create_session() as session:
+        orm_dag = (
+            session.query(DagModel)
+                   .filter(DagModel.dag_id == dag_id).first()
+        )
+        if paused == 'true':
+            orm_dag.is_paused = True
+        else:
+            orm_dag.is_paused = False
+        session.merge(orm_dag)
+        session.commit()
+
+    return jsonify({'response': 'ok'})
+
+
 @api_experimental.route(
     '/dags/<string:dag_id>/dag_runs/<string:execution_date>/tasks/<string:task_id>',
     methods=['GET'])
@@ -172,6 +234,43 @@ def task_instance_info(dag_id, execution_date, task_id):
     return jsonify(fields)
 
 
+@api_experimental.route(
+    '/dags/<string:dag_id>/dag_runs/<string:execution_date>',
+    methods=['GET'])
+@requires_authentication
+def dag_run_status(dag_id, execution_date):
+    """
+    Returns a JSON with a dag_run's public instance variables.
+    The format for the exec_date is expected to be
+    "YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15". This will
+    of course need to have been encoded for URL in the request.
+    """
+
+    # Convert string datetime into actual datetime
+    try:
+        execution_date = timezone.parse(execution_date)
+    except ValueError:
+        error_message = (
+            'Given execution date, {}, could not be identified '
+            'as a date. Example date format: 2015-11-16T14:34:15+00:00'.format(
+                execution_date))
+        _log.info(error_message)
+        response = jsonify({'error': error_message})
+        response.status_code = 400
+
+        return response
+
+    try:
+        info = get_dag_run_state(dag_id, execution_date)
+    except AirflowException as err:
+        _log.info(err)
+        response = jsonify(error="{}".format(err))
+        response.status_code = err.status_code
+        return response
+
+    return jsonify(info)
+
+
 @api_experimental.route('/latest_runs', methods=['GET'])
 @requires_authentication
 def latest_dag_runs():
diff --git a/airflow/www/app.py b/airflow/www/app.py
index e9b101db37689..1fce12bc4e7a9 100644
--- a/airflow/www/app.py
+++ b/airflow/www/app.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,18 +17,21 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-import six
+from typing import Any
 
+import six
 from flask import Flask
 from flask_admin import Admin, base
 from flask_caching import Cache
 from flask_wtf.csrf import CSRFProtect
 from six.moves.urllib.parse import urlparse
 from werkzeug.wsgi import DispatcherMiddleware
+from werkzeug.contrib.fixers import ProxyFix
 
 import airflow
 from airflow import configuration as conf
 from airflow import models, LoggingMixin
+from airflow.models.connection import Connection
 from airflow.settings import Session
 
 from airflow.www.blueprints import routes
@@ -43,10 +46,19 @@
 
 def create_app(config=None, testing=False):
     app = Flask(__name__)
+    if configuration.conf.getboolean('webserver', 'ENABLE_PROXY_FIX'):
+        app.wsgi_app = ProxyFix(app.wsgi_app)
     app.secret_key = configuration.conf.get('webserver', 'SECRET_KEY')
     app.config['LOGIN_DISABLED'] = not configuration.conf.getboolean(
         'webserver', 'AUTHENTICATE')
 
+    app.config['SESSION_COOKIE_HTTPONLY'] = True
+    app.config['SESSION_COOKIE_SECURE'] = conf.getboolean('webserver', 'COOKIE_SECURE')
+    app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE')
+
+    if config:
+        app.config.from_mapping(config)
+
     csrf.init_app(app)
 
     app.config['TESTING'] = testing
@@ -58,8 +70,8 @@ def create_app(config=None, testing=False):
     api.load_auth()
     api.api_auth.init_app(app)
 
-    cache = Cache(
-        app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
+    # flake8: noqa: F841
+    cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
 
     app.register_blueprint(routes)
 
@@ -101,7 +113,7 @@ def create_app(config=None, testing=False):
         av(vs.UserModelView(
             models.User, Session, name="Users", category="Admin"))
         av(vs.ConnectionModelView(
-            models.Connection, Session, name="Connections", category="Admin"))
+            Connection, Session, name="Connections", category="Admin"))
         av(vs.VariableView(
             models.Variable, Session, name="Variables", category="Admin"))
         av(vs.XComView(
@@ -109,11 +121,11 @@ def create_app(config=None, testing=False):
 
         admin.add_link(base.MenuLink(
             category='Docs', name='Documentation',
-            url='https://airflow.incubator.apache.org/'))
+            url='https://airflow.apache.org/'))
         admin.add_link(
             base.MenuLink(category='Docs',
-                          name='Github',
-                          url='https://github.com/apache/incubator-airflow'))
+                          name='GitHub',
+                          url='https://github.com/apache/airflow'))
 
         av(vs.VersionView(name='Version', category="About"))
 
@@ -132,8 +144,8 @@ def integrate_plugins():
                 log.debug('Adding view %s', v.name)
                 admin.add_view(v)
             for bp in flask_blueprints:
-                log.debug('Adding blueprint %s', bp.name)
-                app.register_blueprint(bp)
+                log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name)
+                app.register_blueprint(bp["blueprint"])
             for ml in sorted(menu_links, key=lambda x: x.name):
                 log.debug('Adding menu link %s', ml.name)
                 admin.add_link(ml)
@@ -144,11 +156,7 @@ def integrate_plugins():
         # required for testing purposes otherwise the module retains
         # a link to the default_auth
         if app.config['TESTING']:
-            if six.PY2:
-                reload(e)
-            else:
-                import importlib
-                importlib.reload(e)
+            six.moves.reload_module(e)
 
         app.register_blueprint(e.api_experimental, url_prefix='/api/experimental')
 
@@ -166,11 +174,11 @@ def shutdown_session(exception=None):
         return app
 
 
-app = None
+app = None  # type: Any
 
 
 def root_app(env, resp):
-    resp(b'404 Not Found', [(b'Content-Type', b'text/plain')])
+    resp('404 Not Found', [('Content-Type', 'text/plain')])
     return [b'Apache Airflow is not at this location']
 
 
diff --git a/airflow/www/blueprints.py b/airflow/www/blueprints.py
index ec719b96d2f6a..e964081f8a1a2 100644
--- a/airflow/www/blueprints.py
+++ b/airflow/www/blueprints.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,10 +17,16 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+from datetime import timedelta
 from flask import (
-    url_for, Markup, Blueprint, redirect,
+    url_for, Blueprint, redirect,
 )
-import markdown
+from sqlalchemy import func
+
+from airflow import configuration as conf
+from airflow import jobs, settings
+from airflow.utils import timezone
+from airflow.www import utils as wwwutils
 
 routes = Blueprint('routes', __name__)
 
@@ -32,6 +38,35 @@ def index():
 
 @routes.route('/health')
 def health():
-    """ We can add an array of tests here to check the server's health """
-    content = Markup(markdown.markdown("The server is healthy!"))
-    return content
+    """
+    An endpoint helping check the health status of the Airflow instance,
+    including metadatabase and scheduler.
+    """
+    session = settings.Session()
+    BJ = jobs.BaseJob
+    payload = {}
+    scheduler_health_check_threshold = timedelta(seconds=conf.getint('scheduler',
+                                                                     'scheduler_health_check_threshold'
+                                                                     ))
+
+    latest_scheduler_heartbeat = None
+    payload['metadatabase'] = {'status': 'healthy'}
+    try:
+        latest_scheduler_heartbeat = session.query(func.max(BJ.latest_heartbeat)). \
+            filter(BJ.state == 'running', BJ.job_type == 'SchedulerJob'). \
+            scalar()
+    except Exception:
+        payload['metadatabase']['status'] = 'unhealthy'
+
+    if not latest_scheduler_heartbeat:
+        scheduler_status = 'unhealthy'
+    else:
+        if timezone.utcnow() - latest_scheduler_heartbeat <= scheduler_health_check_threshold:
+            scheduler_status = 'healthy'
+        else:
+            scheduler_status = 'unhealthy'
+
+    payload['scheduler'] = {'status': scheduler_status,
+                            'latest_scheduler_heartbeat': str(latest_scheduler_heartbeat)}
+
+    return wwwutils.json_response(payload)
diff --git a/airflow/www/gunicorn_config.py b/airflow/www/gunicorn_config.py
index 7cf12c56c778e..db76b6de0c5af 100644
--- a/airflow/www/gunicorn_config.py
+++ b/airflow/www/gunicorn_config.py
@@ -8,9 +8,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www/static/connection_form.js b/airflow/www/static/connection_form.js
index 902cf747e52b2..8517ad2582bf4 100644
--- a/airflow/www/static/connection_form.js
+++ b/airflow/www/static/connection_form.js
@@ -54,6 +54,22 @@
                 'login': 'Username',
             }
         },
+        qubole: {
+          hidden_fields: ['login', 'schema', 'port', 'extra'],
+          relabeling: {
+            'host': 'API Endpoint',
+            'password': 'Auth Token',
+          },
+          placeholders: {
+            'host': 'https://<env>.qubole.com/api'
+          }
+        },
+        ssh: {
+          hidden_fields: ['schema'],
+          relabeling: {
+            'login': 'Username',
+          }
+        },
       }
       function connTypeChange(connectionType) {
         $("div.form-group").removeClass("hide");
@@ -68,6 +84,8 @@
         $("label[orig_text]").each(function(){
             $(this).text($(this).attr("orig_text"));
         });
+        $(".form-control").each(function(){$(this).attr('placeholder', '')});
+
         if (config[connectionType] != undefined){
           $.each(config[connectionType].hidden_fields, function(i, field){
             $("#" + field).parent().parent().addClass('hide')
@@ -77,6 +95,9 @@
             lbl.attr("orig_text", lbl.text());
             $("label[for='" + k + "']").text(v);
           });
+          $.each(config[connectionType].placeholders, function(k, v){
+            $("#" + k).attr('placeholder', v);
+          });
         }
       }
       var connectionType=$("#conn_type").val();
diff --git a/airflow/www/static/gantt-chart-d3v2.js b/airflow/www/static/gantt-chart-d3v2.js
index d21311a1c541d..245a0147e9f72 100644
--- a/airflow/www/static/gantt-chart-d3v2.js
+++ b/airflow/www/static/gantt-chart-d3v2.js
@@ -129,7 +129,7 @@ d3.gantt = function() {
       call_modal(d.taskName, d.executionDate);
     })
     .attr("class", function(d){
-      if(taskStatus[d.status] == null){ return "bar";}
+      if(taskStatus[d.status] == null){ return "null";}
       return taskStatus[d.status];
     })
     .attr("y", 0)
diff --git a/airflow/www/static/graph.css b/airflow/www/static/graph.css
index f1d34800612fa..a40abf3ed7930 100644
--- a/airflow/www/static/graph.css
+++ b/airflow/www/static/graph.css
@@ -31,11 +31,12 @@ g.node.success rect {
 g.node.up_for_retry rect {
     stroke: gold;
 }
-
 g.node.queued rect {
     stroke: grey;
 }
-
+g.node.up_for_reschedule rect{
+    stroke: turquoise;
+}
 g.node.running rect{
     stroke: lime;
 }
diff --git a/airflow/www/static/main.css b/airflow/www/static/main.css
index 57164b94e5ccf..08bfc798a01ed 100644
--- a/airflow/www/static/main.css
+++ b/airflow/www/static/main.css
@@ -44,10 +44,14 @@ td>span.glyphicon{
     padding-left: 3px;
     padding-top: 3px;
 }
-button.btn {
+button.btn, label.btn {
     border: 1px solid black;
 }
 
+.btn-group label.btn {
+    background-color: #f0f0f0;
+}
+
 div.rich_doc {
     padding: 5px 10px;
     border: 1px solid #dddddd;
@@ -75,6 +79,9 @@ span.success{
 span.up_for_retry{
     background-color: gold;
 }
+span.up_for_reschedule{
+    background-color: turquoise;
+}
 span.started{
     background-color: lime;
 }
@@ -262,3 +269,4 @@ div.square {
 .sc { color: #BA2121 } /* Literal.String.Char */
 .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
 .s2 { color: #BA2121 } /* Literal.String.Double */
+.s1 { color: #BA2121 } /* Literal.String.Single */
diff --git a/airflow/www/static/tree.css b/airflow/www/static/tree.css
index 9304bb1c55f78..17ff748b59a94 100644
--- a/airflow/www/static/tree.css
+++ b/airflow/www/static/tree.css
@@ -62,6 +62,9 @@ rect.upstream_failed {
 rect.up_for_retry {
     fill: gold;
 }
+rect.up_for_reschedule {
+    fill: turquoise;
+}
 rect.skipped {
     fill: pink;
 }
diff --git a/airflow/www/templates/admin/master.html b/airflow/www/templates/admin/master.html
index 2939a099d4aa2..7837c93f73e07 100644
--- a/airflow/www/templates/admin/master.html
+++ b/airflow/www/templates/admin/master.html
@@ -37,14 +37,52 @@
         alert('{{ hostname }}');
     });
     $('span').tooltip();
+    var CSRF = {{ csrf_token() | tojson }};
 
     $.ajaxSetup({
       beforeSend: function(xhr, settings) {
         if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-          xhr.setRequestHeader("X-CSRFToken", "{{ csrf_token() }}");
+          xhr.setRequestHeader("X-CSRFToken", CSRF);
         }
       }
     });
+
+var el = document.createElement("span");
+
+function escapeHtml(text) {
+  el.textContent = text;
+  return el.innerHTML;
+}
+
+function postAsForm(url, parameters) {
+  var form = $("<form></form>");
+
+  form.attr("method", "POST");
+  form.attr("action", url);
+
+  $.each(parameters || {}, function(key, value) {
+    var field = $('<input></input>');
+
+    field.attr("type", "hidden");
+    field.attr("name", key);
+    field.attr("value", value);
+
+    form.append(field);
+  });
+
+  var field = $('<input></input>');
+
+  field.attr("type", "hidden");
+  field.attr("name", "csrf_token");
+  field.attr("value", CSRF);
+
+  form.append(field);
+
+  // The form needs to be a part of the document in order for us to be able
+  // to submit it.
+  $(document.body).append(form);
+  form.submit();
+}
 </script>
 {% endblock %}
 
@@ -89,7 +127,7 @@
         <ul class="nav navbar-nav navbar-right">
             <li><a id="clock"></a></li>
             {% if current_user.is_authenticated %}
-              <li><a href="{{ url_for('airflow.logout') }}"><span data-toggle="tooltip" data-placement="left" title="Logout" class="glyphicon glyphicon-log-out"></span></a></li>
+              <li class="never_active"><a href="{{ url_for('airflow.logout') }}"><span data-toggle="tooltip" data-placement="left" title="Logout" class="glyphicon glyphicon-log-out"></span></a></li>
             {% endif %}
         </ul>
           <ul class="nav navbar-nav navbar-right">
diff --git a/airflow/www/templates/airflow/config.html b/airflow/www/templates/airflow/config.html
index a6f011d57ca60..89ad4fcd04bfd 100644
--- a/airflow/www/templates/airflow/config.html
+++ b/airflow/www/templates/airflow/config.html
@@ -33,10 +33,6 @@ <h2>{{ title }}</h2>
         <h5>{{ subtitle }}</h5>
     {% endif %}
 
-    {% if code %}
-        <pre>{{ code }}</pre>
-    {% endif %}
-
     {% if code_html %}
         {{ code_html|safe }}
     {% endif %}
diff --git a/airflow/www/templates/airflow/confirm.html b/airflow/www/templates/airflow/confirm.html
index 8084250c95bb7..c50a6e0bf0794 100644
--- a/airflow/www/templates/airflow/confirm.html
+++ b/airflow/www/templates/airflow/confirm.html
@@ -26,10 +26,15 @@ <h2>Wait a minute.</h2>
         <pre><code>{{ details }}</code></pre>
     {% endif %}
     </div>
-    <button class="btn btn-primary" onclick="window.location += '&confirmed=true'">
-        OK!
-    </button>
-    <button class="btn" onclick="window.history.back()">bail.</button>
+    <form method="POST">
+      <input name="csrf_token" type="hidden" value="{{ csrf_token() }}">
+      <input name="confirmed" type="hidden" value="true">
+      {% for name,val in request.form.items() if name != "csrf_token" %}
+        <input type="hidden" name="{{ name }}" value="{{ val }}">
+      {% endfor %}
+      <input class="btn btn-primary" type="submit" value="OK!">
+      <button class="btn" onclick="window.history.go(-2); return false">bail.</button>
+    </form>
     <br>
     <br>
 {% endblock %}
diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html
index adb2d387da443..5daf7820c0a9c 100644
--- a/airflow/www/templates/airflow/dag.html
+++ b/airflow/www/templates/airflow/dag.html
@@ -40,7 +40,7 @@ <h3 class="pull-left">
       {% endif %}
     </h3>
     <h4 class="pull-right">
-      <a class="label label-default" href="/admin/dagrun/?flt2_dag_id_equals={{ dag.dag_id }}">
+      <a class="label label-default" href="{{ url_for('dagrun.index_view') }}?_flt_2_dag_id={{ dag.dag_id }}">
         schedule: {{ dag.schedule_interval }}
       </a>
     </h4>
@@ -49,7 +49,7 @@ <h4 class="pull-right">
   <div>
     <ul class="nav nav-pills">
       {% if dag.parent_dag %}
-          <li class="never_active"><a href="{{ url_for('airflow.' + dag.default_view, dag_id=dag.parent_dag.dag_id) }}">
+          <li class="never_active"><a href="{{ url_for('airflow.' + dag.get_default_view(), dag_id=dag.parent_dag.dag_id) }}">
               <span class="glyphicon glyphicon-arrow-left" aria-hidden="true"></span>
               Back to {{ dag.parent_dag.dag_id }}</a>
           </li>
@@ -95,11 +95,25 @@ <h4 class="pull-right">
         </a>
       </li>
       <li>
-        <a href="{{ url_for("airflow.refresh", dag_id=dag.dag_id) }}" title="Refresh">
+        <a href="{{ url_for("airflow.trigger", dag_id=dag.dag_id, origin=url_for("airflow.tree", dag_id=dag.dag_id)) }}"
+          onclick="return confirmTriggerDag(this, '{{ dag.safe_dag_id }}')">
+          <span class="glyphicon glyphicon-play-circle" aria-hidden="true"></span>
+          Trigger DAG
+        </a>
+      </li>
+      <li>
+        <a href="{{ url_for("airflow.refresh", dag_id=dag.dag_id) }}" title="Refresh" onclick="postAsForm(this.href); return false">
           <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
           Refresh
         </a>
       </li>
+      <li>
+        <a href="{{ url_for("airflow.delete", dag_id=dag.dag_id, root=root) }}"
+        onclick="return confirmDeleteDag(this, '{{ dag.safe_dag_id }}')">
+        <span class="glyphicon glyphicon-remove-circle" style="color:red" aria-hidden="true"></span>
+        Delete
+        </a>
+      </li>
     </ul>
   </div>
   <hr>
@@ -139,74 +153,119 @@ <h4 class="modal-title" id="myModalLabel">
             View Log
           </button>
           <hr/>
-          <button id="btn_run" type="button" class="btn btn-primary"
-            title="Runs a single task instance">
-            Run
-          </button>
-          <span class="btn-group">
-            <button id="btn_ignore_all_deps"
-              type="button" class="btn" data-toggle="button"
-              title="Ignores all non-critical dependencies, including task state and task_deps"
-              >Ignore All Deps</button>
-          </span>
-          <span class="btn-group">
-            <button id="btn_ignore_ti_state"
-              type="button" class="btn" data-toggle="button"
-              title="Ignore previous success/failure"
-              >Ignore Task State</button>
-          </span>
-          <button id="btn_ignore_task_deps"
-            type="button" class="btn" data-toggle="button"
-            title="Disregard the task-specific dependencies, e.g. status of upstream task instances and depends_on_past"
-            >Ignore Task Deps</button>
-          </span>
+          <div>
+            <label style="display:inline"> Download Log (by attempts): </label>
+            <ul class="nav nav-pills" role="tablist" id="try_index" style="display:inline">
+            </ul>
+          </div>
           <hr/>
-          <button id="btn_clear" type="button" class="btn btn-primary"
-              title="Clearing deletes the previous state of the task instance, allowing it to get re-triggered by the scheduler or a backfill command"
-              >Clear
-          </button>
-          <span class="btn-group">
-            <button id="btn_past"
-              type="button" class="btn" data-toggle="button"
-              title="Also include past task instances when clearing this one"
-              >Past</button>
-            <button id="btn_future"
-              type="button" class="btn" data-toggle="button"
-              title="Also include future task instances when clearing this one"
-              >Future</button>
-            <button id="btn_upstream"
-              type="button" class="btn" data-toggle="button"
-              title="Also include upstream dependencies"
-              >Upstream</button>
-            <button id="btn_downstream"
-              type="button" class="btn active" data-toggle="button"
-              title="Also include downstream dependencies"
-              >
-              Downstream
+          <form method="POST">
+            <input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
+            <input name="dag_id" type="hidden">
+            <input name="task_id" type="hidden">
+            <input name="execution_date" type="hidden">
+            <input name="origin" type="hidden" value="{{ request.base_url }}">
+            <button id="btn_run" type="button" class="btn btn-primary"
+              title="Runs a single task instance"
+              data-action="{{ url_for('airflow.run') }}">
+              Run
             </button>
-            <button id="btn_recursive"
-              type="button" class="btn active" data-toggle="button">
-               Recursive
+            <span class="btn-group" data-toggle="buttons">
+              <label
+                class="btn"
+                title="Ignores all non-critical dependencies, including task state and task_deps">
+                <input type="checkbox" value="true" name="ignore_all_deps" autocomplete="off">
+                Ignore All Deps</label>
+              <label class="btn"
+                title="Ignore previous success/failure">
+                <input type="checkbox" value="true" name="ignore_ti_state" autocomplete="off">
+                Ignore Task State
+              </label>
+              <label class="btn"
+                title="Disregard the task-specific dependencies, e.g. status of upstream task instances and depends_on_past">
+                <input type="checkbox" value="true" name="ignore_task_deps" autocomplete="off">
+                Ignore Task Deps
+              </label>
+            </span>
+            <hr/>
+            <button id="btn_clear" type="button" class="btn btn-primary"
+                title="Clearing deletes the previous state of the task instance, allowing it to get re-triggered by the scheduler or a backfill command"
+                data-action="{{ url_for('airflow.clear') }}"
+                >Clear
             </button>
-          </span>
-          <hr/>
-          <button id="btn_success" type="button" class="btn btn-primary">
-            Mark Success
-          </button>
-          <span class="btn-group">
-            <button id="btn_success_past"
-              type="button" class="btn" data-toggle="button">Past</button>
-            <button id="btn_success_future"
-              type="button" class="btn" data-toggle="button">
-              Future
+            <span class="btn-group" data-toggle="buttons">
+              <label class="btn"
+                title="Also include past task instances when clearing this one">
+                <input type="checkbox" value="true" name="past" autocomplete="off">
+                Past
+              </label>
+              <label class="btn"
+                title="Also include future task instances when clearing this one">
+                <input type="checkbox" value="true" name="future" autocomplete="off">
+                Future
+              </label>
+              <label class="btn"
+                title="Also include upstream dependencies">
+                <input type="checkbox" value="true" name="upstream" autocomplete="off">
+                Upstream
+              </label>
+              <label class="btn active"
+                title="Also include downstream dependencies">
+                <input type="checkbox" value="true" name="downstream" checked autocomplete="off">
+                Downstream
+              </label>
+              <label class="btn active">
+                <input type="checkbox" value="true" name="recursive" checked autocomplete="off">
+                 Recursive
+              </label>
+            </span>
+            <hr/>
+            <button id="btn_failed" type="button" class="btn btn-primary"
+                data-action="{{ url_for('airflow.failed') }}">
+              Mark Failed
             </button>
-            <button id="btn_success_upstream"
-              type="button" class="btn" data-toggle="button">Upstream</button>
-            <button id="btn_success_downstream"
-              type="button" class="btn" data-toggle="button">
-              Downstream
+            <span class="btn-group" data-toggle="buttons">
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_past" autocomplete="off">
+                Past
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_future" autocomplete="off">
+                Future
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_upstream" autocomplete="off">
+                Upstream
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_downstream" autocomplete="off">
+                Downstream
+              </label>
+            </span>
+            <hr/>
+            <button id="btn_success" type="button" class="btn btn-primary"
+                data-action="{{ url_for('airflow.success') }}">
+              Mark Success
             </button>
-          </span>
+            <span class="btn-group" data-toggle="buttons">
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_past" autocomplete="off">
+                Past
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_future" autocomplete="off">
+                Future
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_upstream" autocomplete="off">
+                Upstream
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_downstream" autocomplete="off">
+                Downstream
+              </label>
+            </span>
+          </form>
         </div>
         <div class="modal-footer">
           <button type="button" class="btn btn-default" data-dismiss="modal">
@@ -228,15 +287,24 @@ <h4 class="modal-title" id="dagModalLabel">
           </h4>
         </div>
         <div class="modal-body">
-          <button id="btn_edit_dagrun" type="button" class="btn btn-primary">
-            Edit
-          </button>
-          <button id="btn_dagrun_clear" type="button" class="btn btn-primary">
-            Clear
-          </button>
-          <button id="btn_dagrun_success" type="button" class="btn btn-primary">
-            Mark Success
-          </button>
+          <form method="POST">
+            <button id="btn_edit_dagrun" type="button" class="btn btn-primary">
+              Edit
+            </button>
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+            <input name="dag_id" type="hidden">
+            <input name="execution_date" type="hidden">
+            <input name="origin" type="hidden" value="{{ request.base_url }}">
+            <button id="btn_dagrun_clear" type="button" class="btn btn-primary" data-action="{{ url_for('airflow.dagrun_clear') }}">
+              Clear
+            </button>
+            <button id="btn_dagrun_failed" type="button" class="btn btn-primary" data-action="{{ url_for('airflow.dagrun_failed') }}">
+              Mark Failed
+            </button>
+            <button id="btn_dagrun_success" type="button" class="btn btn-primary" data-action="{{ url_for('airflow.dagrun_success') }}">
+              Mark Success
+            </button>
+          </form>
         </div>
         <div class="modal-footer">
           <button type="button" class="btn btn-default" data-dismiss="modal">
@@ -274,7 +342,7 @@ <h4 class="modal-title" id="dagModalLabel">
     var task_id = '';
     var exection_date = '';
     var subdag_id = '';
-    function call_modal(t, d, sd) {
+    function call_modal(t, d, try_numbers, sd) {
       task_id = t;
       loc = String(window.location);
       $("#btn_filter").on("click", function(){
@@ -286,12 +354,35 @@ <h4 class="modal-title" id="dagModalLabel">
       $('#execution_date').html(d);
       $('#myModal').modal({});
       $("#myModal").css("margin-top","0px")
-        if (subdag_id===undefined)
-            $("#div_btn_subdag").hide();
-        else {
-            $("#div_btn_subdag").show();
-            subdag_id = "{{ dag.dag_id }}."+t;
+      if (subdag_id === undefined)
+          $("#div_btn_subdag").hide();
+      else {
+          $("#div_btn_subdag").show();
+          subdag_id = "{{ dag.dag_id }}."+t;
+      }
+
+      $("#try_index > li").remove();
+      var startIndex = (try_numbers > 2 ? 0 : 1);
+      for (var index = startIndex; index <  try_numbers; index++) {
+        var url = "{{ url_for('airflow.get_logs_with_metadata') }}" +
+          "?dag_id=" + encodeURIComponent(dag_id) +
+          "&task_id=" + encodeURIComponent(task_id) +
+          "&execution_date=" + encodeURIComponent(execution_date) +
+          "&metadata=null" +
+          "&format=file";
+
+        var showLabel = index;
+        if (index != 0) {
+          url += "&try_number=" + index;
+        } else {
+          showLabel = 'All';
         }
+
+        $("#try_index").append(`<li role="presentation" style="display:inline">
+          <a href="${url}"> ${showLabel} </a>
+          </li>`
+        );
+      }
     }
 
     function call_modal_dag(dag) {
@@ -302,6 +393,25 @@ <h4 class="modal-title" id="dagModalLabel">
       $("#dagModal").css("margin-top","0px");
     }
 
+
+    function confirmTriggerDag(link, dag_id){
+      if (confirm("Are you sure you want to run '"+dag_id+"' now?")) {
+        postAsForm(link.href, {});
+      }
+      // Never follow the link
+      return false;
+    }
+
+    function confirmDeleteDag(link, dag_id){
+      if (confirm("Are you sure you want to delete '"+dag_id+"' now?\n\
+        This option will delete ALL metadata, DAG runs, etc.\n\
+        EXCEPT Log.\n\
+        This cannot be undone.")) {
+        postAsForm(link.href, {});
+      }
+      return false;
+    }
+
     $("#btn_rendered").click(function(){
       url = "{{ url_for('airflow.rendered') }}" +
         "?task_id=" + encodeURIComponent(task_id) +
@@ -311,7 +421,7 @@ <h4 class="modal-title" id="dagModalLabel">
     });
 
     $("#btn_subdag").click(function(){
-      url = "{{ url_for( 'airflow.' + dag.default_view ) }}" +
+      url = "{{ url_for( 'airflow.' + dag.get_default_view() ) }}" +
         "?dag_id=" + encodeURIComponent(subdag_id) +
         "&execution_date=" + encodeURIComponent(execution_date);
       window.location = url;
@@ -321,7 +431,8 @@ <h4 class="modal-title" id="dagModalLabel">
       url = "{{ url_for('airflow.log') }}" +
         "?task_id=" + encodeURIComponent(task_id) +
         "&dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date);
+        "&execution_date=" + encodeURIComponent(execution_date) +
+        "&format=json";
       window.location = url;
     });
 
@@ -334,83 +445,24 @@ <h4 class="modal-title" id="dagModalLabel">
     });
 
     $("#btn_ti").click(function(){
-      url = "/admin/taskinstance/" +
+      url = "{{ url_for('taskinstance.index_view') }}" +
         "?flt1_dag_id_equals=" + encodeURIComponent(dag_id) +
         "&flt2_task_id_equals=" + encodeURIComponent(task_id) +
         "&sort=3&desc=1";
       window.location = url;
     });
 
-    $("#btn_run").click(function(){
-      url = "{{ url_for('airflow.run') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&ignore_all_deps=" + $('#btn_ignore_all_deps').hasClass('active') +
-        "&ignore_task_deps=" + $('#btn_ignore_task_deps').hasClass('active') +
-        "&ignore_ti_state=" + $('#btn_ignore_ti_state').hasClass('active') +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_clear").click(function(){
-      url = "{{ url_for('airflow.clear') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&future=" + $('#btn_future').hasClass('active') +
-        "&past=" + $('#btn_past').hasClass('active') +
-        "&upstream=" + $('#btn_upstream').hasClass('active') +
-        "&downstream=" + $('#btn_downstream').hasClass('active') +
-        "&recursive=" + $('#btn_recursive').hasClass('active') +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_dagrun_clear").click(function(){
-      url = "{{ url_for('airflow.dagrun_clear') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_success").click(function(){
-      url = "{{ url_for('airflow.success') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&upstream=" + $('#btn_success_upstream').hasClass('active') +
-        "&downstream=" + $('#btn_success_downstream').hasClass('active') +
-        "&future=" + $('#btn_success_future').hasClass('active') +
-        "&past=" + $('#btn_success_past').hasClass('active') +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-
-      window.location = url;
-    });
-
-    $('#btn_dagrun_success').click(function(){
-      url = "{{ url_for('airflow.dagrun_success') }}" +
-        "?dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_gantt").click(function(){
-      url = "{{ url_for('airflow.gantt') }}" +
-        "?dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date);
-      window.location = url;
-    });
-
-    $("#btn_graph").click(function(){
-      url = "{{ url_for('airflow.graph') }}" +
-        "?dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date);
-      window.location = url;
-    });
+    $('form button[data-action]').click(function (e) {
+      var form = $(this).closest('form').get(0);
+      form.dag_id.value = dag_id;
+      form.execution_date.value = execution_date;
+      form.origin.value = window.location;
+      if (form.task_id) {
+        form.task_id.value = task_id;
+      }
+      form.action = $(this).data("action");
+      form.submit();
+    })
 
     $("#pause_resume").change(function() {
       var dag_id =  $(this).attr('dag_id');
@@ -424,7 +476,7 @@ <h4 class="modal-title" id="dagModalLabel">
     });
 
     $('#btn_edit_dagrun').click(function(){
-      window.location = '/admin/dagrun/edit/?id=' + id;
+      window.location = "{{ url_for('dagrun.edit_view') }}?id=" + encodeURIComponent(id);
     });
 
   </script>
diff --git a/airflow/www/templates/airflow/dag_code.html b/airflow/www/templates/airflow/dag_code.html
index 35f85b2f789b1..65d957356cd78 100644
--- a/airflow/www/templates/airflow/dag_code.html
+++ b/airflow/www/templates/airflow/dag_code.html
@@ -21,25 +21,7 @@
 {% block body %}
     {{ super() }}
     <h4>{{ title }}</h4>
-    {% if html_code %}
-        {{ html_code|safe }}
-    {% endif %}
-    {% if code %}
-        <pre>{{ code }}</pre>
-    {% endif %}
-
-    {% if code_dict %}
-        {% for k, v in code_dict.items() %}
-            <h5>{{ k }}</h5>
-            <pre>{{ v }}</pre>
-        {% endfor %}
-    {% endif %}
-    {% if html_dict %}
-        {% for k, v in html_dict.items() %}
-            <h5>{{ k }}</h5>
-            {{ v|safe }}
-        {% endfor %}
-    {% endif %}
+    {{ html_code|safe }}
 {% endblock %}
 
 {% block tail %}
diff --git a/airflow/www/templates/airflow/dag_details.html b/airflow/www/templates/airflow/dag_details.html
index 4f9f2eb6ad7f1..588e7ea5822de 100644
--- a/airflow/www/templates/airflow/dag_details.html
+++ b/airflow/www/templates/airflow/dag_details.html
@@ -42,7 +42,7 @@ <h2>{{ title }}</h2>
       </tr>
       <tr>
         <th>max_active_runs</td>
-        <td>{{ dag.active_runs | length }} / {{ dag.max_active_runs }}</td>
+        <td>{{ active_runs | length }} / {{ dag.max_active_runs }}</td>
       </tr>
       <tr>
         <th>concurrency</td>
diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html
index 23978905780ea..1a0ebc01f44ba 100644
--- a/airflow/www/templates/airflow/dags.html
+++ b/airflow/www/templates/airflow/dags.html
@@ -28,7 +28,7 @@
 {% block body %}
   <h2>DAGs</h2>
 
-  <div id="main_content" style="display:none;">
+  <div id="main_content">
     <div class="row">
       <div class="col-sm-2">
       </div>
@@ -63,52 +63,37 @@ <h2>DAGs</h2>
             </tr>
         </thead>
         <tbody>
-        {% for dag_id in dag_ids_in_page %}
-            {% set dag = webserver_dags[dag_id] if dag_id in webserver_dags else None %}
+        {% for dag in dags %}
             <tr>
                 <!-- Column 1: Edit dag -->
                 <td class="text-center" style="width:10px;">
-                    {% if dag_id in orm_dags %}
-                  <a href="{{ url_for('dagmodel.edit_view') }}?id={{ dag_id }}" title="Info">
+                  <a href="{{ url_for('dagmodel.edit_view') }}?id={{ dag.dag_id }}" title="Info">
                         <span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
                     </a>
-                    {% endif %}
                 </td>
 
                 <!-- Column 2: Turn dag on/off -->
                 <td>
-                  {% if dag_id in orm_dags %}
-                    <input id="toggle-{{ dag_id }}" dag_id="{{ dag_id }}" type="checkbox" {{ "checked" if not orm_dags[dag_id].is_paused else "" }} data-toggle="toggle" data-size="mini" method="post">
-                  {% endif %}
+                  <input id="toggle-{{ dag.dag_id }}" dag_id="{{ dag.dag_id }}" type="checkbox" {{ "checked" if not dag.is_paused else "" }} data-toggle="toggle" data-size="mini" method="post">
                 </td>
 
                 <!-- Column 3: Name -->
                 <td>
-                    {% if dag_id in webserver_dags %}
-                    <a href="{{ url_for('airflow.'+dag.default_view, dag_id=dag.dag_id) }}" title="{{ dag.description }}">
-                        {{ dag_id }}
-                    </a>
-                    {% else %}
-                        {{ dag_id }}
-                        <span class="glyphicon glyphicon-info-sign" class="info" aria-hidden="true" title="This DAG isn't available in the web server's DagBag object. It shows up in this list because the scheduler marked it as active in the metadata database."></span>
-                    {% endif %}
-                    {% if dag_id not in orm_dags %}
-                        <span class="glyphicon glyphicon-info-sign" class="info" aria-hidden="true" title="This DAG seems to be existing only locally. The master scheduler doesn't seem to be aware of its existence."></span>
-                    {% endif %}
+                  <a href="{{ url_for('airflow.'+ dag.get_default_view(), dag_id=dag.dag_id) }}" title="{{ dag.description }}">
+                      {{ dag.dag_id }}
+                  </a>
                 </td>
 
                 <!-- Column 4: Dag Schedule -->
                 <td>
-                    {% if dag_id in webserver_dags %}
                   <a class="label label-default schedule {{ dag.dag_id }}" href="{{ url_for('dagrun.index_view') }}?flt2_dag_id_equals={{ dag.dag_id }}">
-                        {{ dag.schedule_interval }}
+                        {{ dag.schedule_interval | string }}
                     </a>
-                    {% endif %}
                 </td>
 
                 <!-- Column 5: Dag Owners -->
                 <td>
-                  {{ dag.owner if dag else orm_dags[dag_id].owners }}
+                  {{ dag.owners }}
                 </td>
 
                 <!-- Column 6: Recent Tasks -->
@@ -118,14 +103,12 @@ <h2>DAGs</h2>
 
                 <!-- Column 7: Last Run -->
                 <td class="text-nowrap latest_dag_run {{ dag.dag_id }}">
-                  {% if dag %}
-                    {% set last_run = dag.get_last_dagrun() %}
-                    {% if last_run and last_run.execution_date %}
-                      <a href="{{ url_for('airflow.graph', dag_id=dag.dag_id, execution_date=last_run.execution_date) }}">
-                        {{ last_run.execution_date.strftime("%Y-%m-%d %H:%M") }}
-                      </a>
-                      <span aria-hidden="true" id="statuses_info" title="Start Date: {{ last_run.start_date.strftime("%Y-%m-%d %H:%M") }}" class="glyphicon glyphicon-info-sign"></span>
-                    {% endif %}
+                  {% set last_run = dag.get_last_dagrun(include_externally_triggered=True) %}
+                  {% if last_run and last_run.execution_date %}
+                    <a href="{{ url_for('airflow.graph', dag_id=dag.dag_id, execution_date=last_run.execution_date) }}">
+                      {{ last_run.execution_date.strftime("%Y-%m-%d %H:%M") }}
+                    </a>
+                    <span aria-hidden="true" id="statuses_info" title="Start Date: {{ last_run.start_date.strftime("%Y-%m-%d %H:%M") }}" class="glyphicon glyphicon-info-sign"></span>
                   {% endif %}
                 </td>
 
@@ -139,8 +122,8 @@ <h2>DAGs</h2>
                 {% if dag %}
 
                 <!-- Trigger Dag -->
-                <a href="{{ url_for('airflow.trigger', dag_id=dag.dag_id) }}"
-                   onclick="return confirmTriggerDag('{{ dag.safe_dag_id }}')">
+                <a href="{{ url_for('airflow.trigger', dag_id=dag.dag_id) }}&origin={{ request.base_url }}"
+                   onclick="return confirmTriggerDag(this, '{{ dag.safe_dag_id }}')">
                     <span class="glyphicon glyphicon-play-circle" aria-hidden="true" data-original-title="Trigger Dag"></span>
                 </a>
 
@@ -186,10 +169,16 @@ <h2>DAGs</h2>
                 {% endif %}
 
                 <!-- Refresh -->
-                <a href="{{ url_for("airflow.refresh", dag_id=dag_id) }}">
+                <a href="{{ url_for("airflow.refresh", dag_id=dag.dag_id) }}" onclick="postAsForm(this.href); return false">
                   <span class="glyphicon glyphicon-refresh" aria-hidden="true" data-original-title="Refresh"></span>
                 </a>
 
+                <!-- Delete -->
+                <!-- Use dag_id instead of dag.dag_id, because the DAG might not exist in the webserver's DagBag -->
+                <a href="{{ url_for('airflow.delete', dag_id=dag.dag_id) }}"
+                  onclick="return confirmDeleteDag(this, '{{ dag.dag_id }}')">
+                   <span class="glyphicon glyphicon-remove-circle" style="color:red" aria-hidden="true" data-original-title="Delete Dag"></span>
+                </a>
                 </td>
             </tr>
         {% endfor %}
@@ -241,8 +230,22 @@ <h2>DAGs</h2>
         window.location = DAGS_INDEX + "?page_size=" + p_size;
       });
 
-      function confirmTriggerDag(dag_id){
-          return confirm("Are you sure you want to run '"+dag_id+"' now?");
+      function confirmTriggerDag(link, dag_id){
+        if (confirm("Are you sure you want to run '"+dag_id+"' now?")) {
+          postAsForm(link.href, {});
+        }
+        // Never follow the link
+        return false;
+      }
+
+      function confirmDeleteDag(link, dag_id){
+        if (confirm("Are you sure you want to delete '"+dag_id+"' now?\n\
+          This option will delete ALL metadata, DAG runs, etc.\n\
+          EXCEPT Log.\n\
+          This cannot be undone.")) {
+          postAsForm(link.href, {});
+        }
+        return false;
       }
       all_dags = $("[id^=toggle]");
       $.each(all_dags, function(i,v) {
@@ -253,7 +256,7 @@ <h2>DAGs</h2>
           } else {
             is_paused = 'false'
           }
-          url = 'airflow/paused?is_paused=' + is_paused + '&dag_id=' + dag_id;
+          url = {{ url_for('airflow.paused') | tojson }} + '?is_paused=' + is_paused + '&dag_id=' + encodeURIComponent(dag_id);
           $.post(url);
         });
       });
@@ -309,7 +312,7 @@ <h2>DAGs</h2>
       d3.json("{{ url_for('airflow.dag_stats') }}", function(error, json) {
         for(var dag_id in json) {
             states = json[dag_id];
-            g = d3.select('svg#dag-run-' + dag_id)
+            g = d3.select('svg#dag-run-' + dag_id.replace(/\./g, '__dot__'))
               .attr('height', diameter + (stroke_width_hover * 2))
               .attr('width', '110px')
               .selectAll("g")
@@ -388,9 +391,9 @@ <h2>DAGs</h2>
       d3.json("{{ url_for('airflow.task_stats') }}", function(error, json) {
         for(var dag_id in json) {
             states = json[dag_id];
-            g = d3.select('svg#task-run-' + dag_id)
+            g = d3.select('svg#task-run-' + dag_id.replace(/\./g, '__dot__'))
               .attr('height', diameter + (stroke_width_hover * 2))
-              .attr('width', '240px')
+              .attr('width', '300px')
               .selectAll("g")
               .data(states)
               .enter()
diff --git a/airflow/www/templates/airflow/gantt.html b/airflow/www/templates/airflow/gantt.html
index 01d3cd8cdfcc5..b070409654e08 100644
--- a/airflow/www/templates/airflow/gantt.html
+++ b/airflow/www/templates/airflow/gantt.html
@@ -55,7 +55,7 @@
     var dag_id = '{{ dag.dag_id }}';
     var task_id = '';
     var exection_date = '';
-    data = {{ data |safe }};
+    data = {{ data |tojson|safe }};
     var gantt = d3.gantt()
       .taskTypes(data.taskNames)
       .taskStatus(data.taskStatus)
diff --git a/airflow/www/templates/airflow/graph.html b/airflow/www/templates/airflow/graph.html
index 33cdd9737b2fa..79fbb74394745 100644
--- a/airflow/www/templates/airflow/graph.html
+++ b/airflow/www/templates/airflow/graph.html
@@ -23,99 +23,105 @@
 
 {% block head_css %}
 {{ super() }}
-<link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='dagre.css') }}">
-<link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='graph.css') }}">
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='dagre.css') }}">
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='graph.css') }}">
 {% endblock %}
 
 {% block body %}
 {{ super() }}
-    {% if dag.doc_md %}
-        <div class="rich_doc" style="margin-bottom: 15px;">{{ doc_md|safe }}</div>
-    {% endif %}
-    <div class="form-inline">
-        <form method="get" style="float:left;">
-            {{ state_token }}
-            Base date: {{ form.base_date(class_="form-control") }}
-            Number of runs: {{ form.num_runs(class_="form-control") }}
-            Run:
-            {{ form.execution_date(class_="form-control") | safe }}
-            Layout:
-            {{ form.arrange(class_="form-control") | safe }}
-            <input type="hidden" name="root" value="{{ root }}">
-            <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
-            <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
-            <input type="submit" value="Go" class="btn btn-default"
-            action="" method="get">
-        </form>
-            <div class="input-group" style="float: right;">
-              <input type="text" id="searchbox" class="form-control" placeholder="Search for..." onenter="null">
-            </div><!-- /input-group -->
-        <div style="clear: both;"/>
-    </div>
+{% if dag.doc_md %}
+<div class="rich_doc" style="margin-bottom: 15px;">{{ dag.doc_md|safe }}</div>
+{% endif %}
+<div class="form-inline">
+  <form method="get" style="float:left;">
+    {{ state_token }}
+    Base date: {{ form.base_date(class_="form-control") }}
+    Number of runs: {{ form.num_runs(class_="form-control") }}
+    Run:
+    {{ form.execution_date(class_="form-control") | safe }}
+    Layout:
+    {{ form.arrange(class_="form-control") | safe }}
+    <input type="hidden" name="root" value="{{ root }}">
+    <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
+    <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
+    <input type="submit" value="Go" class="btn btn-default">
+  </form>
+  <div class="input-group" style="float: right;">
+    <input type="text" id="searchbox" class="form-control" placeholder="Search for...">
+  </div>
+  <div style="clear: both;"></div>
+</div>
 <hr/>
 <div>
-    {% for op in operators %}
-    <div class="legend_item" style="border-width:1px;float:left;background:{{ op.ui_color }};color:{{ op.ui_fgcolor }};">
-        {{ op.__name__ }}
-    </div>
-    {% endfor %}
-
-    <div style"background-color: blue;">
-    <div class="legend_item state" style="border-color:white;">no status</div>
+  {% for op in operators %}
+  <div class="legend_item"
+       style="border-width:1px;float:left;background:{{ op.ui_color }};color:{{ op.ui_fgcolor }};">
+    {{ op.__name__ }}
+  </div>
+  {% endfor %}
+
+  <div>
+    <div class="legend_item state" style="border-color:white;">no_status</div>
     <div class="legend_item state" style="border-color:grey;">queued</div>
-    <div class="legend_item state" style="border-color:gold;">retry</div>
+    <div class="legend_item state" style="border-color:gold;">up_for_retry</div>
+    <div class="legend_item state" style="border-color:turquoise;">up_for_reschedule</div>
     <div class="legend_item state" style="border-color:pink;">skipped</div>
     <div class="legend_item state" style="border-color:red;">failed</div>
     <div class="legend_item state" style="border-color:lime;">running</div>
     <div class="legend_item state" style="border-color:green;">success</div>
-    </div>
-    <div style="clear:both;"></div>
+  </div>
+  <div style="clear:both;"></div>
 </div>
 <div id="error" style="display: none; margin-top: 10px;" class="alert alert-danger" role="alert">
-        <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
-        <span id="error_msg">Oops.</span>
+  <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
+  <span id="error_msg">Oops.</span>
 </div>
 <hr style="margin-bottom: 0px;"/>
 <button class="btn btn-default pull-right" id="refresh_button">
-    <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
+  <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
 </button>
 <div id="svg_container">
 
-    <svg width="{{ width }}" height="{{ height }}">
-        <g id='dig' transform="translate(20,20)"/>
-        <filter id="blur-effect-1">
-        <feGaussianBlur stdDeviation="3" />
-        </filter>
-    </svg>
-    <img id="loading" alt="spinner" src="{{ url_for('static', filename='loading.gif') }}">
+  <svg width="{{ width }}" height="{{ height }}">
+    <g id='dig' transform="translate(20,20)"></g>
+    <filter id="blur-effect-1">
+      <feGaussianBlur stdDeviation="3"></feGaussianBlur>
+    </filter>
+  </svg>
+  <img id="loading" alt="spinner" src="{{ url_for('static', filename='loading.gif') }}">
 </div>
 <hr>
 {% endblock %}
 
 {% block tail %}
-    {{ super() }}
+{{ super() }}
 
-    <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-    <script src="{{ url_for('static', filename='dagre-d3.js') }}"></script>
-    <script>
+<script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
+<script src="{{ url_for('static', filename='dagre-d3.js') }}"></script>
+<script>
 
     var highlight_color = "#000000";
     var upstream_color = "#2020A0";
     var downstream_color = "#0000FF";
 
-    var nodes = {{ nodes|safe }};
-    var edges = {{ edges|safe }};
-    var tasks = {{ tasks|safe }};
-    var task_instances = {{ task_instances|safe }};
+    var nodes = {{ nodes|tojson|safe }};
+    var edges = {{ edges|tojson|safe }};
+    var tasks = {{ tasks|tojson|safe }};
+    var task_instances = {{ task_instances|tojson|safe }};
     var execution_date = "{{ execution_date }}";
     var arrange = "{{ arrange }}";
     var g = dagreD3.json.decode(nodes, edges);
     var duration = 500;
     var stateFocusMap = {
-        'no status':false, 'failed':false, 'running':false,
-        'queued': false, 'success': false};
+        'success': false,
+        'running':false,
+        'failed':false,
+        'skipped': false,
+        'up_for_reschedule': false,
+        'up_for_retry': false,
+        'queued': false,
+        'no_status': false
+    };
 
 
     var layout = dagreD3.layout().rankDir(arrange).nodeSep(15).rankSep(15);
@@ -127,9 +133,9 @@
     d3.selectAll("g.node").on("click", function(d){
         task = tasks[d];
         if (task.task_type == "SubDagOperator")
-            call_modal(d, execution_date, true);
+            call_modal(d, execution_date, task_instances[d].try_number, true);
         else
-            call_modal(d, execution_date);
+            call_modal(d, execution_date,task_instances[d].try_number);
     });
 
 
@@ -269,21 +275,24 @@
         $.each(task_instances, function(task_id, ti) {
           $('tspan').filter(function(index) { return $(this).text() === task_id; })
             .parent().parent().parent()
-            .attr("class", "node enter " + ti.state)
+            .attr("class", "node enter " + (ti.state ? ti.state : "no_status"))
             .attr("data-toggle", "tooltip")
             .attr("data-original-title", function(d) {
               // Tooltip
               task = tasks[task_id];
-              tt =  "Task_id: " + ti.task_id + "<br>";
-              tt += "Run: " + ti.execution_date + "<br>";
+              tt = "";
+              if(ti.task_id != undefined){
+                tt +=  "Task_id: " + escapeHtml(ti.task_id) + "<br>";
+              }
+              tt += "Run: " + escapeHtml(ti.execution_date) + "<br>";
               if(ti.run_id != undefined){
-                tt += "run_id: <nobr>" + ti.run_id + "</nobr><br>";
+                tt += "run_id: <nobr>" + escapeHtml(ti.run_id) + "</nobr><br>";
               }
-              tt += "Operator: " + task.task_type + "<br>";
-              tt += "Started: " + ti.start_date + "<br>";
-              tt += "Ended: " + ti.end_date + "<br>";
-              tt += "Duration: " + ti.duration + "<br>";
-              tt += "State: " + ti.state + "<br>";
+              tt += "Operator: " + escapeHtml(task.task_type) + "<br>";
+              tt += "Started: " + escapeHtml(ti.start_date) + "<br>";
+              tt += "Ended: " + escapeHtml(ti.end_date) + "<br>";
+              tt += "Duration: " + escapeHtml(ti.duration) + "<br>";
+              tt += "State: " + escapeHtml(ti.state) + "<br>";
               return tt;
             });
         });
@@ -367,7 +376,7 @@
         }
     );
 
-    </script>
+</script>
 
 
 {% endblock %}
diff --git a/airflow/www/templates/airflow/login.html b/airflow/www/templates/airflow/login.html
index cdae3178f01db..72cdff202cc11 100644
--- a/airflow/www/templates/airflow/login.html
+++ b/airflow/www/templates/airflow/login.html
@@ -1,13 +1,13 @@
-{# 
+{#
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
-  
+
     http://www.apache.org/licenses/LICENSE-2.0
-  
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,7 +25,7 @@
     <div class="row">
         <div class="col-sm-6 col-md-4 col-md-offset-4">
             <h1 class="text-center login-title">Sign in to Airflow</h1>
-            <div class="account-wall">
+            <div class="text-center account-wall">
                 <img src="{{ url_for("static", filename="pin_100.png") }}" />
                 <form class="form-signin" method="post" action="{{ url_for('airflow.login') }}">
                     <input type="text" class="form-control" placeholder="Username" name="username" required autofocus>
@@ -43,4 +43,4 @@ <h1 class="text-center login-title">Sign in to Airflow</h1>
     </div>
 </div>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/airflow/www/templates/airflow/task.html b/airflow/www/templates/airflow/task.html
index e45b745fb879c..820a4a98149c7 100644
--- a/airflow/www/templates/airflow/task.html
+++ b/airflow/www/templates/airflow/task.html
@@ -37,7 +37,9 @@ <h5>Dependencies Blocking Task From Getting Scheduled</h5>
                 </tr>
             {% endfor %}
         </table>
-        {{ html_code|safe }}
+        {% if html_code is defined %}
+            {{ html_code|safe }}
+        {% endif %}
     </div>
     <div>
         {% for attr, value in special_attrs_rendered.items() %}
@@ -70,6 +72,8 @@ <h5>Task Attributes</h5>
                 </tr>
             {% endfor %}
         </table>
-        {{ html_code|safe }}
+        {% if html_code is defined %}
+            {{ html_code|safe }}
+        {% endif %}
     </div>
 {% endblock %}
diff --git a/airflow/www/templates/airflow/ti_code.html b/airflow/www/templates/airflow/ti_code.html
index 44942ca7d7e5d..72008f04682bb 100644
--- a/airflow/www/templates/airflow/ti_code.html
+++ b/airflow/www/templates/airflow/ti_code.html
@@ -21,23 +21,8 @@
 {% block body %}
     {{ super() }}
     <h4>{{ title }}</h4>
-    {% if html_code %}
-        {{ html_code|safe }}
-    {% endif %}
-    {% if code %}
-        <pre>{{ code }}</pre>
-    {% endif %}
-
-    {% if code_dict %}
-        {% for k, v in code_dict.items() %}
-            <h5>{{ k }}</h5>
-            <pre>{{ v }}</pre>
-        {% endfor %}
-    {% endif %}
-    {% if html_dict %}
-        {% for k, v in html_dict.items() %}
-            <h5>{{ k }}</h5>
-            {{ v|safe }}
-        {% endfor %}
-    {% endif %}
+    {% for k, v in html_dict.items() %}
+        <h5>{{ k }}</h5>
+        {{ v|safe }}
+    {% endfor %}
 {% endblock %}
diff --git a/airflow/www/templates/airflow/tree.html b/airflow/www/templates/airflow/tree.html
index f20127ceb3178..f16743ce35967 100644
--- a/airflow/www/templates/airflow/tree.html
+++ b/airflow/www/templates/airflow/tree.html
@@ -20,67 +20,66 @@
 
 {% block head_css %}
 {{ super() }}
-<link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='tree.css') }}">
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='tree.css') }}">
 <link href="{{ admin_static.url(filename='vendor/bootstrap-daterangepicker/daterangepicker-bs2.css') }}" rel="stylesheet">
 {% endblock %}
 
 {% block body %}
 {{ super() }}
 <div style="float: left" class="form-inline">
-    <form method="get" style="float:left;">
-        Base date: {{ form.base_date(class_="form-control") }}
-        Number of runs: {{ form.num_runs(class_="form-control") }}
-        <input type="hidden" name="root" value="{{ root if root else '' }}">
-        <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
-        <input type="submit" value="Go" class="btn btn-default"
-        action="" method="get">
-        <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
-    </form>
+  <form method="get" style="float:left;">
+    Base date: {{ form.base_date(class_="form-control") }}
+    Number of runs: {{ form.num_runs(class_="form-control") }}
+    <input type="hidden" name="root" value="{{ root if root else '' }}">
+    <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
+    <input type="submit" value="Go" class="btn btn-default">
+    <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
+  </form>
 </div>
 <div style="clear: both;"></div>
 <hr/>
 <div>
-    <div class="legend_item" style="border: none;">no status</div>
-    <div class="square" style="background: white;"></div>
-    <div class="legend_item" style="border: none;">queued</div>
-    <div class="square" style="background: grey;"></div>
-    <div class="legend_item" style="border: none;">retry</div>
-    <div class="square" style="background: gold;"></div>
-    <div class="legend_item" style="border: none;">skipped</div>
-    <div class="square" style="background: pink;"></div>
-    <div class="legend_item" style="border: none;">failed</div>
-    <div class="square" style="background: red;"></div>
-    <div class="legend_item" style="border: none;">running</div>
-    <div class="square" style="background: lime;"></div>
-    <div class="legend_item" style="border: none;">success</div>
-    <div class="square" style="background: green;"></div>
-    {% for op in operators %}
-        <div class="legend_circle" style="background:{{ op.ui_color }};">
-        </div>
-        <div class="legend_item" style="float:left;border-color:white;">{{ op.__name__ }}</div>
-    {% endfor %}
-    <div style="clear:both;"></div>
+  <div class="legend_item" style="border: none;">no_status</div>
+  <div class="square" style="background: white;"></div>
+  <div class="legend_item" style="border: none;">queued</div>
+  <div class="square" style="background: grey;"></div>
+  <div class="legend_item" style="border: none;">up_for_retry</div>
+  <div class="square" style="background: gold;"></div>
+  <div class="legend_item" style="border: none;">up_for_reschedule</div>
+  <div class="square" style="background: turquoise;"></div>
+  <div class="legend_item" style="border: none;">skipped</div>
+  <div class="square" style="background: pink;"></div>
+  <div class="legend_item" style="border: none;">failed</div>
+  <div class="square" style="background: red;"></div>
+  <div class="legend_item" style="border: none;">running</div>
+  <div class="square" style="background: lime;"></div>
+  <div class="legend_item" style="border: none;">success</div>
+  <div class="square" style="background: green;"></div>
+  {% for op in operators %}
+  <div class="legend_circle" style="background:{{ op.ui_color }};">
+  </div>
+  <div class="legend_item" style="float:left;border-color:white;">{{ op.__name__ }}</div>
+  {% endfor %}
+  <div style="clear:both;"></div>
 </div>
 <hr/>
 <div id="svg_container">
-  <img id='loading' width="50"
-    src="{{ url_for('static', filename='loading.gif') }}">
-    <svg class='tree' width="100%">
-        <filter id="blur-effect-1">
-            <feGaussianBlur stdDeviation="3" />
-        </filter>
-    </svg>
+  <img id='loading' width="50" src="{{ url_for('static', filename='loading.gif') }}">
+  <svg class='tree' width="100%">
+    <filter id="blur-effect-1">
+      <feGaussianBlur stdDeviation="3"></feGaussianBlur>
+    </filter>
+  </svg>
 </div>
 {% endblock %}
 
 {% block tail %}
-  {{ super() }}
-  <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-  <script>
+{{ super() }}
+<script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
+<script>
 $('span.status_square').tooltip({html: true});
 
-var data = {{ data|safe }};
+var data = {{ data|tojson|safe }};
 var barHeight = 20;
 var axisHeight = 40;
 var square_x = 500;
@@ -198,13 +197,15 @@
       .attr("title", function(d){
         var tt = "";
         if (d.operator != undefined) {
-          tt += "operator: " + d.operator + "<br/>";
-          tt += "depends_on_past: " + d.depends_on_past + "<br/>";
-          tt += "upstream: " + d.num_dep + "<br/>";
-          tt += "retries: " + d.retries + "<br/>";
-          tt += "owner: " + d.owner + "<br/>";
-          tt += "start_date: " + d.start_date + "<br/>";
-          tt += "end_date: " + d.end_date + "<br/>";
+          if (d.operator != undefined) {
+            tt += "operator: " + escapeHtml(d.operator) + "<br/>";
+          }
+          tt += "depends_on_past: " + escapeHtml(d.depends_on_past) + "<br/>";
+          tt += "upstream: " + escapeHtml(d.num_dep) + "<br/>";
+          tt += "retries: " + escapeHtml(d.retries) + "<br/>";
+          tt += "owner: " + escapeHtml(d.owner) + "<br/>";
+          tt += "start_date: " + escapeHtml(d.start_date) + "<br/>";
+          tt += "end_date: " + escapeHtml(d.end_date) + "<br/>";
         }
         return tt;
       })
@@ -233,9 +234,9 @@
         if(d.task_id === undefined)
             call_modal_dag(d);
         else if(nodeobj[d.task_id].operator=='SubDagOperator')
-            call_modal(d.task_id, d.execution_date, true);
+            call_modal(d.task_id, d.execution_date, d.try_number, true);
         else
-            call_modal(d.task_id, d.execution_date);
+            call_modal(d.task_id, d.execution_date, d.try_number);
       })
       .attr("class", function(d) {return "state " + d.state})
       .attr("data-toggle", "tooltip")
@@ -245,17 +246,24 @@
       .style("stroke-width", function(d) {return (d.run_id != undefined)? "2": "1"})
       .style("stroke-opacity", function(d) {return d.external_trigger ? "0": "1"})
       .attr("title", function(d){
-        s =  "Task_id: " + d.task_id + "<br>";
-        s += "Run: " + d.execution_date + "<br>";
+        var s = "";
+        if (d.task_id != undefined ) {
+          s += "Task_id: " + escapeHtml(d.task_id) + "<br>";
+        }
+        s += "Run: " + escapeHtml(d.execution_date) + "<br>";
         if(d.run_id != undefined){
-          s += "run_id: <nobr>" + d.run_id + "</nobr><br>";
+          s += "run_id: <nobr>" + escapeHtml(d.run_id) + "</nobr><br>";
+        }
+        if (d.operator != undefined) {
+          s += "Operator: " + escapeHtml(d.operator) + "<br>"
         }
-        s += "Operator: " + d.operator + "<br>"
         if(d.start_date != undefined){
-          s += "Started: " + d.start_date + "<br>";
-          s += "Ended: " + d.end_date + "<br>";
-          s += "Duration: " + d.duration + "<br>";
-          s += "State: " + d.state + "<br>";
+          s += "Started: " + escapeHtml(d.start_date) + "<br>";
+          s += "Ended: " + escapeHtml(d.end_date) + "<br>";
+          if (d.duration != undefined) {
+            s += "Duration: " + escapeHtml(d.duration) + "<br>";
+          }
+          s += "State: " + escapeHtml(d.state) + "<br>";
         }
         return s;
       })
@@ -377,5 +385,6 @@
   }
 }
 set_tooltip();
-  </script>
+
+</script>
 {% endblock %}
diff --git a/airflow/www/templates/airflow/variables/README.md b/airflow/www/templates/airflow/variables/README.md
index 3fd539f8b54fb..e3b30a1a64b93 100644
--- a/airflow/www/templates/airflow/variables/README.md
+++ b/airflow/www/templates/airflow/variables/README.md
@@ -1,5 +1,24 @@
-## Variable Editor
-----
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Variable Editor
+
 This folder contains forms used to edit values in the "Variable" key-value
 store.  This data can be edited under the "Admin" admin tab, but sometimes
 it is preferable to use a form that can perform checking and provide a nicer
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index 7d9c8a07501c7..b15c515882bfb 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -17,24 +17,30 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+# flake8: noqa: E402
+import inspect
 from future import standard_library
-standard_library.install_aliases()
-from builtins import str
-from builtins import object
+standard_library.install_aliases()  # noqa: E402
+from builtins import str, object
 
 from cgi import escape
 from io import BytesIO as IO
 import functools
 import gzip
+import io
 import json
+import os
+import re
 import time
+import wtforms
+from wtforms.compat import text_type
+import zipfile
 
-from flask import after_this_request, request, Response
-from flask_admin.contrib.sqla.filters import FilterConverter
+from flask import after_this_request, request, Markup, Response
 from flask_admin.model import filters
+import flask_admin.contrib.sqla.filters as sqlafilters
 from flask_login import current_user
-import wtforms
-from wtforms.compat import text_type
+from six.moves.urllib.parse import urlencode
 
 from airflow import configuration, models, settings
 from airflow.utils.db import create_session
@@ -55,16 +61,21 @@
 
 
 def should_hide_value_for_key(key_name):
-    return any(s in key_name.lower() for s in DEFAULT_SENSITIVE_VARIABLE_FIELDS) \
-        and configuration.conf.getboolean('admin', 'hide_sensitive_variable_fields')
+    # It is possible via importing variables from file that a key is empty.
+    if key_name:
+        config_set = configuration.conf.getboolean('admin',
+                                                   'hide_sensitive_variable_fields')
+        field_comp = any(s in key_name.lower() for s in DEFAULT_SENSITIVE_VARIABLE_FIELDS)
+        return config_set and field_comp
+    return False
 
 
 class LoginMixin(object):
     def is_accessible(self):
         return (
             not AUTHENTICATE or (
-                not current_user.is_anonymous() and
-                current_user.is_authenticated()
+                not current_user.is_anonymous and
+                current_user.is_authenticated
             )
         )
 
@@ -73,7 +84,7 @@ class SuperUserMixin(object):
     def is_accessible(self):
         return (
             not AUTHENTICATE or
-            (not current_user.is_anonymous() and current_user.is_superuser())
+            (not current_user.is_anonymous and current_user.is_superuser())
         )
 
 
@@ -81,22 +92,16 @@ class DataProfilingMixin(object):
     def is_accessible(self):
         return (
             not AUTHENTICATE or
-            (not current_user.is_anonymous() and current_user.data_profiling())
+            (not current_user.is_anonymous and current_user.data_profiling())
         )
 
 
 def get_params(**kwargs):
-    params = []
-    for k, v in kwargs.items():
-        if k == 'showPaused':
-            # True is default or None
-            if v or v is None:
-                continue
-            params.append('{}={}'.format(k, v))
-        elif v:
-            params.append('{}={}'.format(k, v))
-    params = sorted(params, key=lambda x: x.split('=')[0])
-    return '&'.join(params)
+    if 'showPaused' in kwargs:
+        v = kwargs['showPaused']
+        if v or v is None:
+            kwargs.pop('showPaused')
+    return urlencode({d: v if v is not None else '' for d, v in kwargs.items()})
 
 
 def generate_pages(current_page, num_of_pages,
@@ -127,27 +132,27 @@ def generate_pages(current_page, num_of_pages,
     """
 
     void_link = 'javascript:void(0)'
-    first_node = """<li class="paginate_button {disabled}" id="dags_first">
+    first_node = Markup("""<li class="paginate_button {disabled}" id="dags_first">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&laquo;</a>
-</li>"""
+</li>""")
 
-    previous_node = """<li class="paginate_button previous {disabled}" id="dags_previous">
+    previous_node = Markup("""<li class="paginate_button previous {disabled}" id="dags_previous">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&lt;</a>
-</li>"""
+</li>""")
 
-    next_node = """<li class="paginate_button next {disabled}" id="dags_next">
+    next_node = Markup("""<li class="paginate_button next {disabled}" id="dags_next">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&gt;</a>
-</li>"""
+</li>""")
 
-    last_node = """<li class="paginate_button {disabled}" id="dags_last">
+    last_node = Markup("""<li class="paginate_button {disabled}" id="dags_last">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&raquo;</a>
-</li>"""
+</li>""")
 
-    page_node = """<li class="paginate_button {is_active}">
+    page_node = Markup("""<li class="paginate_button {is_active}">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="2" tabindex="0">{page_num}</a>
-</li>"""
+</li>""")
 
-    output = ['<ul class="pagination" style="margin-top:0px;">']
+    output = [Markup('<ul class="pagination" style="margin-top:0px;">')]
 
     is_disabled = 'disabled' if current_page <= 0 else ''
     output.append(first_node.format(href_link="?{}"
@@ -203,9 +208,9 @@ def is_current(current, page):
                                                       showPaused=showPaused)),
                                    disabled=is_disabled))
 
-    output.append('</ul>')
+    output.append(Markup('</ul>'))
 
-    return wtforms.widgets.core.HTMLString('\n'.join(output))
+    return Markup('\n'.join(output))
 
 
 def limit_sql(sql, limit, conn_type):
@@ -246,8 +251,8 @@ def action_logging(f):
     """
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
-        # Only AnonymousUserMixin() does not have user attribute
-        if current_user and hasattr(current_user, 'user'):
+        # AnonymousUserMixin() has user attribute but its value is None.
+        if current_user and hasattr(current_user, 'user') and current_user.user:
             user = current_user.user.username
         else:
             user = 'anonymous'
@@ -286,7 +291,7 @@ def wrapper(*args, **kwargs):
             dag = dagbag.get_dag(dag_id)
             task = dag.get_task(task_id)
 
-            if current_user and hasattr(current_user, 'username'):
+            if current_user and hasattr(current_user, 'user') and current_user.user:
                 user = current_user.username
             else:
                 user = 'anonymous'
@@ -366,6 +371,22 @@ def zipper(response):
     return view_func
 
 
+def open_maybe_zipped(f, mode='r'):
+    """
+    Opens the given file. If the path contains a folder with a .zip suffix, then
+    the folder is treated as a zip archive, opening the file inside the archive.
+
+    :return: a file object, as in `open`, or as in `ZipFile.open`.
+    """
+
+    _, archive, filename = re.search(
+        r'((.*\.zip){})?(.*)'.format(re.escape(os.sep)), f).groups()
+    if archive and zipfile.is_zipfile(archive):
+        return zipfile.ZipFile(archive, mode=mode).open(filename)
+    else:
+        return io.open(f, mode=mode)
+
+
 def make_cache_key(*args, **kwargs):
     """
     Used by cache to get a unique key per URL
@@ -375,6 +396,33 @@ def make_cache_key(*args, **kwargs):
     return (path + args).encode('ascii', 'ignore')
 
 
+def get_python_source(x):
+    """
+    Helper function to get Python source (or not), preventing exceptions
+    """
+    source_code = None
+
+    if isinstance(x, functools.partial):
+        source_code = inspect.getsource(x.func)
+
+    if source_code is None:
+        try:
+            source_code = inspect.getsource(x)
+        except TypeError:
+            pass
+
+    if source_code is None:
+        try:
+            source_code = inspect.getsource(x.__call__)
+        except (TypeError, AttributeError):
+            pass
+
+    if source_code is None:
+        source_code = 'No source code available for {}'.format(type(x))
+
+    return source_code
+
+
 class AceEditorWidget(wtforms.widgets.TextArea):
     """
     Renders an ACE code editor.
@@ -395,7 +443,43 @@ def __call__(self, field, **kwargs):
         return wtforms.widgets.core.HTMLString(html)
 
 
-class UtcFilterConverter(FilterConverter):
+class UtcDateTimeFilterMixin(object):
+    def clean(self, value):
+        dt = super(UtcDateTimeFilterMixin, self).clean(value)
+        return timezone.make_aware(dt, timezone=timezone.utc)
+
+
+class UtcDateTimeEqualFilter(UtcDateTimeFilterMixin, sqlafilters.DateTimeEqualFilter):
+    pass
+
+
+class UtcDateTimeNotEqualFilter(UtcDateTimeFilterMixin, sqlafilters.DateTimeNotEqualFilter):
+    pass
+
+
+class UtcDateTimeGreaterFilter(UtcDateTimeFilterMixin, sqlafilters.DateTimeGreaterFilter):
+    pass
+
+
+class UtcDateTimeSmallerFilter(UtcDateTimeFilterMixin, sqlafilters.DateTimeSmallerFilter):
+    pass
+
+
+class UtcDateTimeBetweenFilter(UtcDateTimeFilterMixin, sqlafilters.DateTimeBetweenFilter):
+    pass
+
+
+class UtcDateTimeNotBetweenFilter(UtcDateTimeFilterMixin, sqlafilters.DateTimeNotBetweenFilter):
+    pass
+
+
+class UtcFilterConverter(sqlafilters.FilterConverter):
+
+    utcdatetime_filters = (UtcDateTimeEqualFilter, UtcDateTimeNotEqualFilter,
+                           UtcDateTimeGreaterFilter, UtcDateTimeSmallerFilter,
+                           UtcDateTimeBetweenFilter, UtcDateTimeNotBetweenFilter,
+                           sqlafilters.FilterEmpty)
+
     @filters.convert('utcdatetime')
     def conv_utcdatetime(self, column, name, **kwargs):
-        return self.conv_datetime(column, name, **kwargs)
+        return [f(column, name, **kwargs) for f in self.utcdatetime_filters]
diff --git a/airflow/www/validators.py b/airflow/www/validators.py
index 6eee76a32a8ad..3db8b2850e0ad 100644
--- a/airflow/www/validators.py
+++ b/airflow/www/validators.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -45,8 +45,11 @@ def __call__(self, form, field):
 
         if field.data < other.data:
             d = {
-                'other_label': hasattr(other, 'label') and other.label.text
-                               or self.fieldname,
+                'other_label': (
+                    hasattr(other, 'label') and
+                    other.label.text or
+                    self.fieldname
+                ),
                 'other_name': self.fieldname,
             }
             message = self.message
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 8f6725ef59b44..49ef0eba8518e 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -18,76 +18,66 @@
 # under the License.
 #
 
-from past.builtins import basestring, unicode
-
 import ast
+import codecs
+import copy
 import datetime as dt
+from io import BytesIO
+import itertools
+import json
 import logging
-import os
-import pkg_resources
-import socket
-from functools import wraps
-from datetime import timedelta
-import copy
 import math
-import json
-import bleach
-import pendulum
-import codecs
+import os
+import traceback
 from collections import defaultdict
-
-import inspect
+from datetime import timedelta
+from functools import wraps
 from textwrap import dedent
-import traceback
 
+import markdown
+import pendulum
 import sqlalchemy as sqla
-from sqlalchemy import or_, desc, and_, union_all
-
 from flask import (
     abort, jsonify, redirect, url_for, request, Markup, Response,
-    current_app, render_template, make_response)
+    current_app, render_template, make_response, send_file)
+from flask import flash
 from flask_admin import BaseView, expose, AdminIndexView
-from flask_admin.contrib.sqla import ModelView
 from flask_admin.actions import action
 from flask_admin.babel import lazy_gettext
+from flask_admin.contrib.sqla import ModelView
+from flask_admin.form.fields import DateTimeField
 from flask_admin.tools import iterdecode
-from flask import flash
-from flask._compat import PY2
-
-from jinja2.sandbox import ImmutableSandboxedEnvironment
 from jinja2 import escape
-
-import markdown
-import nvd3
-
+from jinja2.sandbox import ImmutableSandboxedEnvironment
+from past.builtins import basestring
+from pygments import highlight, lexers
+from pygments.formatters import HtmlFormatter
+from sqlalchemy import or_, desc, and_, union_all
 from wtforms import (
     Form, SelectField, TextAreaField, PasswordField,
     StringField, validators)
-from flask_admin.form.fields import DateTimeField
-
-from pygments import highlight, lexers
-from pygments.formatters import HtmlFormatter
 
 import airflow
 from airflow import configuration as conf
 from airflow import models
 from airflow import settings
-from airflow.api.common.experimental.mark_tasks import set_dag_run_state
+from airflow.api.common.experimental.mark_tasks import (set_dag_run_state_to_running,
+                                                        set_dag_run_state_to_success,
+                                                        set_dag_run_state_to_failed)
 from airflow.exceptions import AirflowException
+from airflow.models import BaseOperator, errors
 from airflow.models import XCom, DagRun
-from airflow.ti_deps.dep_context import DepContext, QUEUE_DEPS, SCHEDULER_DEPS
-
-from airflow.models import BaseOperator
+from airflow.models.connection import Connection
 from airflow.operators.subdag_operator import SubDagOperator
-
+from airflow.ti_deps.dep_context import DepContext, QUEUE_DEPS, SCHEDULER_DEPS
 from airflow.utils import timezone
-from airflow.utils.json import json_ser
-from airflow.utils.state import State
-from airflow.utils.db import create_session, provide_session
-from airflow.utils.helpers import alchemy_to_dict
 from airflow.utils.dates import infer_time_unit, scale_time_units, parse_execution_date
-from airflow.utils.timezone import datetime
+from airflow.utils.db import create_session, provide_session
+from airflow.utils.helpers import alchemy_to_dict, render_log_filename
 from airflow.utils.net import get_hostname
+from airflow.utils.state import State
+from airflow.utils.timezone import datetime
+from airflow._vendor import nvd3
 from airflow.www import utils as wwwutils
 from airflow.www.forms import (DateTimeForm, DateTimeWithNumRunsForm,
                                DateTimeWithNumRunsWithDagRunsForm)
@@ -117,13 +107,12 @@ def dag_link(v, c, m, p):
     if m.dag_id is None:
         return Markup()
 
-    dag_id = bleach.clean(m.dag_id)
     url = url_for(
         'airflow.graph',
-        dag_id=dag_id,
+        dag_id=m.dag_id,
         execution_date=m.execution_date)
     return Markup(
-        '<a href="{}">{}</a>'.format(url, dag_id))
+        '<a href="{}">{}</a>').format(url, m.dag_id)
 
 
 def log_url_formatter(v, c, m, p):
@@ -134,45 +123,43 @@ def log_url_formatter(v, c, m, p):
 
 
 def dag_run_link(v, c, m, p):
-    dag_id = bleach.clean(m.dag_id)
     url = url_for(
         'airflow.graph',
         dag_id=m.dag_id,
         run_id=m.run_id,
         execution_date=m.execution_date)
-    return Markup('<a href="{url}">{m.run_id}</a>'.format(**locals()))
+    title = m.run_id
+    return Markup('<a href="{url}">{title}</a>').format(**locals())
 
 
 def task_instance_link(v, c, m, p):
-    dag_id = bleach.clean(m.dag_id)
-    task_id = bleach.clean(m.task_id)
     url = url_for(
         'airflow.task',
-        dag_id=dag_id,
-        task_id=task_id,
+        dag_id=m.dag_id,
+        task_id=m.task_id,
         execution_date=m.execution_date.isoformat())
     url_root = url_for(
         'airflow.graph',
-        dag_id=dag_id,
-        root=task_id,
+        dag_id=m.dag_id,
+        root=m.task_id,
         execution_date=m.execution_date.isoformat())
     return Markup(
         """
         <span style="white-space: nowrap;">
-        <a href="{url}">{task_id}</a>
+        <a href="{url}">{m.task_id}</a>
         <a href="{url_root}" title="Filter on this task and upstream">
         <span class="glyphicon glyphicon-filter" style="margin-left: 0px;"
             aria-hidden="true"></span>
         </a>
         </span>
-        """.format(**locals()))
+        """).format(**locals())
 
 
 def state_token(state):
     color = State.color(state)
     return Markup(
         '<span class="label" style="background-color:{color};">'
-        '{state}</span>'.format(**locals()))
+        '{state}</span>').format(**locals())
 
 
 def parse_datetime_f(value):
@@ -196,27 +183,30 @@ def datetime_f(v, c, m, p):
     dttm = attr.isoformat() if attr else ''
     if timezone.utcnow().isoformat()[:4] == dttm[:4]:
         dttm = dttm[5:]
-    return Markup("<nobr>{}</nobr>".format(dttm))
+    return Markup("<nobr>{}</nobr>").format(dttm)
 
 
 def nobr_f(v, c, m, p):
-    return Markup("<nobr>{}</nobr>".format(getattr(m, p)))
+    return Markup("<nobr>{}</nobr>").format(getattr(m, p))
 
 
 def label_link(v, c, m, p):
     try:
         default_params = ast.literal_eval(m.default_params)
-    except:
+    except Exception:
         default_params = {}
     url = url_for(
         'airflow.chart', chart_id=m.id, iteration_no=m.iteration_no,
         **default_params)
-    return Markup("<a href='{url}'>{m.label}</a>".format(**locals()))
+    title = m.label
+    return Markup("<a href='{url}'>{title}</a>").format(**locals())
 
 
 def pool_link(v, c, m, p):
-    url = '/admin/taskinstance/?flt1_pool_equals=' + m.pool
-    return Markup("<a href='{url}'>{m.pool}</a>".format(**locals()))
+    title = m.pool
+
+    url = url_for('airflow.task', flt1_pool_equals=m.pool)
+    return Markup("<a href='{url}'>{title}</a>").format(**locals())
 
 
 def pygment_html_render(s, lexer=lexers.TextLexer):
@@ -256,7 +246,9 @@ def wrapped_markdown(s):
     'doc_yaml': lambda x: render(x, lexers.YamlLexer),
     'doc_md': wrapped_markdown,
     'python_callable': lambda x: render(
-        inspect.getsource(x), lexers.PythonLexer),
+        wwwutils.get_python_source(x),
+        lexers.PythonLexer,
+    ),
 }
 
 
@@ -266,7 +258,7 @@ def data_profiling_required(f):
     def decorated_function(*args, **kwargs):
         if (
                 current_app.config['LOGIN_DISABLED'] or
-                (not current_user.is_anonymous() and current_user.data_profiling())
+                (not current_user.is_anonymous and current_user.data_profiling())
         ):
             return f(*args, **kwargs)
         else:
@@ -277,19 +269,23 @@ def decorated_function(*args, **kwargs):
 
 
 def fused_slots(v, c, m, p):
-    url = (
-        '/admin/taskinstance/' +
-        '?flt1_pool_equals=' + m.pool +
-        '&flt2_state_equals=running')
-    return Markup("<a href='{0}'>{1}</a>".format(url, m.used_slots()))
+    url = url_for(
+        'taskinstance.index_view',
+        flt1_pool_equals=m.pool,
+        flt2_state_equals='running',
+    )
+    return Markup("<a href='{0}'>{1}</a>").format(url, m.used_slots())
 
 
 def fqueued_slots(v, c, m, p):
-    url = (
-        '/admin/taskinstance/' +
-        '?flt1_pool_equals=' + m.pool +
-        '&flt2_state_equals=queued&sort=10&desc=1')
-    return Markup("<a href='{0}'>{1}</a>".format(url, m.queued_slots()))
+    url = url_for(
+        'taskinstance.index_view',
+        flt1_pool_equals=m.pool,
+        flt2_state_equals='queued',
+        sort='1',
+        desc='1'
+    )
+    return Markup("<a href='{0}'>{1}</a>").format(url, m.queued_slots())
 
 
 def recurse_tasks(tasks, task_ids, dag_ids, task_id_to_dag):
@@ -370,6 +366,7 @@ def get_date_time_num_runs_dag_runs_form_data(request, session, dag):
         'dr_state': dr_state,
     }
 
+
 class Airflow(BaseView):
     def is_visible(self):
         return False
@@ -382,7 +379,6 @@ def index(self):
     @expose('/chart_data')
     @data_profiling_required
     @wwwutils.gzipped
-    # @cache.cached(timeout=3600, key_prefix=wwwutils.make_cache_key)
     def chart_data(self):
         from airflow import macros
         import pandas as pd
@@ -394,7 +390,7 @@ def chart_data(self):
             csv = request.args.get('csv') == "true"
             chart = session.query(models.Chart).filter_by(id=chart_id).first()
             db = session.query(
-                models.Connection).filter_by(conn_id=chart.conn_id).first()
+                Connection).filter_by(conn_id=chart.conn_id).first()
 
         payload = {
             "state": "ERROR",
@@ -404,9 +400,9 @@ def chart_data(self):
         # Processing templated fields
         try:
             args = ast.literal_eval(chart.default_params)
-            if type(args) is not type(dict()):
+            if not isinstance(args, dict):
                 raise AirflowException('Not a dict')
-        except:
+        except Exception:
             args = {}
             payload['error'] += (
                 "Default params is not valid, string has to evaluate as "
@@ -448,15 +444,15 @@ def chart_data(self):
         if not payload['error'] and len(df) == 0:
             payload['error'] += "Empty result set. "
         elif (
-                        not payload['error'] and
-                            chart.sql_layout == 'series' and
-                        chart.chart_type != "datatable" and
-                    len(df.columns) < 3):
+                not payload['error'] and
+                chart.sql_layout == 'series' and
+                chart.chart_type != "datatable" and
+                len(df.columns) < 3):
             payload['error'] += "SQL needs to return at least 3 columns. "
         elif (
-                    not payload['error'] and
-                        chart.sql_layout == 'columns' and
-                    len(df.columns) < 2):
+                not payload['error'] and
+                chart.sql_layout == 'columns' and
+                len(df.columns) < 2):
             payload['error'] += "SQL needs to return at least 2 columns. "
         elif not payload['error']:
             import numpy as np
@@ -477,7 +473,7 @@ def chart_data(self):
                         df[df.columns[x_col]])
                     df[df.columns[x_col]] = df[df.columns[x_col]].apply(
                         lambda x: int(x.strftime("%s")) * 1000)
-                except Exception as e:
+                except Exception:
                     payload['error'] = "Time conversion failed"
 
             if chart_type == 'datatable':
@@ -486,8 +482,6 @@ def chart_data(self):
             else:
                 if chart.sql_layout == 'series':
                     # User provides columns (series, x, y)
-                    xaxis_label = df.columns[1]
-                    yaxis_label = df.columns[2]
                     df[df.columns[2]] = df[df.columns[2]].astype(np.float)
                     df = df.pivot_table(
                         index=df.columns[1],
@@ -495,10 +489,8 @@ def chart_data(self):
                         values=df.columns[2], aggfunc=np.sum)
                 else:
                     # User provides columns (x, y, metric1, metric2, ...)
-                    xaxis_label = df.columns[0]
-                    yaxis_label = 'y'
                     df.index = df[df.columns[0]]
-                    df = df.sort(df.columns[0])
+                    df = df.sort_values(by=df.columns[0])
                     del df[df.columns[0]]
                     for col in df.columns:
                         df[col] = df[col].astype(np.float)
@@ -559,37 +551,33 @@ def chart(self):
     @login_required
     @provide_session
     def dag_stats(self, session=None):
-        ds = models.DagStat
+        dr = models.DagRun
+        dm = models.DagModel
+        dag_ids = session.query(dm.dag_id)
 
-        ds.update(
-            dag_ids=[dag.dag_id for dag in dagbag.dags.values() if not dag.is_subdag]
-        )
-
-        qry = (
-            session.query(ds.dag_id, ds.state, ds.count)
+        dag_state_stats = (
+            session.query(dr.dag_id, dr.state, sqla.func.count(dr.state)).group_by(dr.dag_id, dr.state)
         )
 
         data = {}
-        for dag_id, state, count in qry:
+        for (dag_id, ) in dag_ids:
+            data[dag_id] = {}
+        for dag_id, state, count in dag_state_stats:
             if dag_id not in data:
                 data[dag_id] = {}
             data[dag_id][state] = count
 
         payload = {}
-        for dag in dagbag.dags.values():
-            payload[dag.safe_dag_id] = []
+        for dag_id, d in data.items():
+            payload[dag_id] = []
             for state in State.dag_states:
-                try:
-                    count = data[dag.dag_id][state]
-                except Exception:
-                    count = 0
-                d = {
+                count = d.get(state, 0)
+                payload[dag_id].append({
                     'state': state,
                     'count': count,
-                    'dag_id': dag.dag_id,
+                    'dag_id': dag_id,
                     'color': State.color(state)
-                }
-                payload[dag.safe_dag_id].append(d)
+                })
         return wwwutils.json_response(payload)
 
     @expose('/task_stats')
@@ -600,12 +588,14 @@ def task_stats(self, session=None):
         DagRun = models.DagRun
         Dag = models.DagModel
 
+        dag_ids = session.query(Dag.dag_id)
+
         LastDagRun = (
             session.query(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label('execution_date'))
                 .join(Dag, Dag.dag_id == DagRun.dag_id)
                 .filter(DagRun.state != State.RUNNING)
-                .filter(Dag.is_active == True)
-                .filter(Dag.is_subdag == False)
+                .filter(Dag.is_active == True)  # noqa: E712
+                .filter(Dag.is_subdag == False)  # noqa: E712
                 .group_by(DagRun.dag_id)
                 .subquery('last_dag_run')
         )
@@ -613,8 +603,8 @@ def task_stats(self, session=None):
             session.query(DagRun.dag_id, DagRun.execution_date)
                 .join(Dag, Dag.dag_id == DagRun.dag_id)
                 .filter(DagRun.state == State.RUNNING)
-                .filter(Dag.is_active == True)
-                .filter(Dag.is_subdag == False)
+                .filter(Dag.is_active == True)  # noqa: E712
+                .filter(Dag.is_subdag == False)  # noqa: E712
                 .subquery('running_dag_run')
         )
 
@@ -622,13 +612,13 @@ def task_stats(self, session=None):
         # If no dag_run is active, return task instances from most recent dag_run.
         LastTI = (
             session.query(TI.dag_id.label('dag_id'), TI.state.label('state'))
-                .join(LastDagRun, and_(
+            .join(LastDagRun, and_(
                 LastDagRun.c.dag_id == TI.dag_id,
                 LastDagRun.c.execution_date == TI.execution_date))
         )
         RunningTI = (
             session.query(TI.dag_id.label('dag_id'), TI.state.label('state'))
-                .join(RunningDagRun, and_(
+            .join(RunningDagRun, and_(
                 RunningDagRun.c.dag_id == TI.dag_id,
                 RunningDagRun.c.execution_date == TI.execution_date))
         )
@@ -636,7 +626,7 @@ def task_stats(self, session=None):
         UnionTI = union_all(LastTI, RunningTI).alias('union_ti')
         qry = (
             session.query(UnionTI.c.dag_id, UnionTI.c.state, sqla.func.count())
-                .group_by(UnionTI.c.dag_id, UnionTI.c.state)
+            .group_by(UnionTI.c.dag_id, UnionTI.c.state)
         )
 
         data = {}
@@ -647,30 +637,27 @@ def task_stats(self, session=None):
         session.commit()
 
         payload = {}
-        for dag in dagbag.dags.values():
-            payload[dag.safe_dag_id] = []
+        for (dag_id, ) in dag_ids:
+            payload[dag_id] = []
             for state in State.task_states:
-                try:
-                    count = data[dag.dag_id][state]
-                except:
-                    count = 0
-                d = {
+                count = data.get(dag_id, {}).get(state, 0)
+                payload[dag_id].append({
                     'state': state,
                     'count': count,
-                    'dag_id': dag.dag_id,
+                    'dag_id': dag_id,
                     'color': State.color(state)
-                }
-                payload[dag.safe_dag_id].append(d)
+                })
         return wwwutils.json_response(payload)
 
     @expose('/code')
     @login_required
-    def code(self):
+    @provide_session
+    def code(self, session=None):
         dag_id = request.args.get('dag_id')
-        dag = dagbag.get_dag(dag_id)
-        title = dag_id
+        dm = models.DagModel
+        dag = session.query(dm).filter(dm.dag_id == dag_id).first()
         try:
-            with open(dag.fileloc, 'r') as f:
+            with wwwutils.open_maybe_zipped(dag.fileloc, 'r') as f:
                 code = f.read()
             html_code = highlight(
                 code, lexers.PythonLexer(), HtmlFormatter(linenos=True))
@@ -678,7 +665,7 @@ def code(self):
             html_code = str(e)
 
         return self.render(
-            'airflow/dag_code.html', html_code=html_code, dag=dag, title=title,
+            'airflow/dag_code.html', html_code=html_code, dag=dag, title=dag_id,
             root=request.args.get('root'),
             demo_mode=conf.getboolean('webserver', 'demo_mode'))
 
@@ -689,17 +676,26 @@ def dag_details(self, session=None):
         dag_id = request.args.get('dag_id')
         dag = dagbag.get_dag(dag_id)
         title = "DAG details"
+        root = request.args.get('root', '')
 
         TI = models.TaskInstance
-        states = (
-            session.query(TI.state, sqla.func.count(TI.dag_id))
-                .filter(TI.dag_id == dag_id)
-                .group_by(TI.state)
-                .all()
+        states = session\
+            .query(TI.state, sqla.func.count(TI.dag_id))\
+            .filter(TI.dag_id == dag_id)\
+            .group_by(TI.state)\
+            .all()
+
+        active_runs = models.DagRun.find(
+            dag_id=dag.dag_id,
+            state=State.RUNNING,
+            external_trigger=False,
+            session=session
         )
+
         return self.render(
             'airflow/dag_details.html',
-            dag=dag, title=title, states=states, State=State)
+            dag=dag, title=title, root=root, states=states, State=State,
+            active_runs=active_runs)
 
     @current_app.errorhandler(404)
     def circles(self):
@@ -749,6 +745,7 @@ def rendered(self):
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
+        root = request.args.get('root', '')
         dag = dagbag.get_dag(dag_id)
         task = copy.copy(dag.get_task(task_id))
         ti = models.TaskInstance(task=task, execution_date=dttm)
@@ -773,7 +770,8 @@ def rendered(self):
             task_id=task_id,
             execution_date=execution_date,
             form=form,
-            title=title, )
+            root=root,
+            title=title)
 
     @expose('/get_logs_with_metadata')
     @login_required
@@ -784,7 +782,12 @@ def get_logs_with_metadata(self, session=None):
         task_id = request.args.get('task_id')
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
-        try_number = int(request.args.get('try_number'))
+        if request.args.get('try_number') is not None:
+            try_number = int(request.args.get('try_number'))
+        else:
+            try_number = None
+        response_format = request.args.get('format', 'json')
+
         metadata = request.args.get('metadata')
         metadata = json.loads(metadata)
 
@@ -823,11 +826,21 @@ def get_logs_with_metadata(self, session=None):
                 ti.task = dag.get_task(ti.task_id)
                 logs, metadatas = handler.read(ti, try_number, metadata=metadata)
                 metadata = metadatas[0]
-            for i, log in enumerate(logs):
-                if PY2 and not isinstance(log, unicode):
-                    logs[i] = log.decode('utf-8')
-            message = logs[0]
-            return jsonify(message=message, metadata=metadata)
+
+            if response_format == 'json':
+                message = logs[0] if try_number is not None else logs
+                return jsonify(message=message, metadata=metadata)
+
+            file_obj = BytesIO(b'\n'.join(
+                log.encode('utf-8') for log in logs
+            ))
+            filename_template = conf.get('core', 'LOG_FILENAME_TEMPLATE')
+            attachment_filename = render_log_filename(
+                ti=ti,
+                try_number="all" if try_number is None else try_number,
+                filename_template=filename_template)
+            return send_file(file_obj, as_attachment=True,
+                             attachment_filename=attachment_filename)
         except AttributeError as e:
             error_message = ["Task log handler {} does not support read logs.\n{}\n"
                              .format(task_log_reader, str(e))]
@@ -851,12 +864,20 @@ def log(self, session=None):
             models.TaskInstance.task_id == task_id,
             models.TaskInstance.execution_date == dttm).first()
 
-        logs = [''] * (ti.next_try_number - 1 if ti is not None else 0)
+        num_logs = 0
+        if ti is not None:
+            num_logs = ti.next_try_number - 1
+            if ti.state == State.UP_FOR_RESCHEDULE:
+                # Tasks in reschedule state decremented the try number
+                num_logs += 1
+        logs = [''] * num_logs
+        root = request.args.get('root', '')
         return self.render(
             'airflow/ti_log.html',
             logs=logs, dag=dag, title="Log by attempts",
             dag_id=dag.dag_id, task_id=task_id,
-            execution_date=execution_date, form=form)
+            execution_date=execution_date, form=form,
+            root=root)
 
     @expose('/task')
     @login_required
@@ -871,6 +892,7 @@ def task(self):
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
+        root = request.args.get('root', '')
         dag = dagbag.get_dag(dag_id)
 
         if not dag or task_id not in dag.task_ids:
@@ -888,7 +910,7 @@ def task(self):
         for attr_name in dir(ti):
             if not attr_name.startswith('_'):
                 attr = getattr(ti, attr_name)
-                if type(attr) != type(self.task):
+                if type(attr) != type(self.task):  # noqa: E721
                     ti_attrs.append((attr_name, str(attr)))
 
         task_attrs = []
@@ -896,7 +918,7 @@ def task(self):
             if not attr_name.startswith('_'):
                 attr = getattr(task, attr_name)
                 if type(attr) != type(self.task) and \
-                        attr_name not in attr_renderer:
+                        attr_name not in attr_renderer:  # noqa: E721
                     task_attrs.append((attr_name, str(attr)))
 
         # Color coding the special attributes that are code
@@ -944,6 +966,7 @@ def task(self):
             execution_date=execution_date,
             special_attrs_rendered=special_attrs_rendered,
             form=form,
+            root=root,
             dag=dag, title=title)
 
     @expose('/xcom')
@@ -958,8 +981,12 @@ def xcom(self, session=None):
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
-        dag = dagbag.get_dag(dag_id)
-        if not dag or task_id not in dag.task_ids:
+        root = request.args.get('root', '')
+        dm_db = models.DagModel
+        ti_db = models.TaskInstance
+        dag = session.query(dm_db).filter(dm_db.dag_id == dag_id).first()
+        ti = session.query(ti_db).filter(ti_db.dag_id == dag_id and ti_db.task_id == task_id).first()
+        if not ti:
             flash(
                 "Task [{}.{}] doesn't seem to exist"
                 " at the moment".format(dag_id, task_id),
@@ -982,35 +1009,46 @@ def xcom(self, session=None):
             task_id=task_id,
             execution_date=execution_date,
             form=form,
+            root=root,
             dag=dag, title=title)
 
-    @expose('/run')
+    @expose('/run', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     @wwwutils.notify_owner
     def run(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
+
         dag = dagbag.get_dag(dag_id)
         task = dag.get_task(task_id)
 
-        execution_date = request.args.get('execution_date')
+        execution_date = request.form.get('execution_date')
         execution_date = pendulum.parse(execution_date)
-        ignore_all_deps = request.args.get('ignore_all_deps') == "true"
-        ignore_task_deps = request.args.get('ignore_task_deps') == "true"
-        ignore_ti_state = request.args.get('ignore_ti_state') == "true"
+        ignore_all_deps = request.form.get('ignore_all_deps') == "true"
+        ignore_task_deps = request.form.get('ignore_task_deps') == "true"
+        ignore_ti_state = request.form.get('ignore_ti_state') == "true"
+
+        from airflow.executors import GetDefaultExecutor
+        executor = GetDefaultExecutor()
+        valid_celery_config = False
+        valid_kubernetes_config = False
 
         try:
-            from airflow.executors import GetDefaultExecutor
             from airflow.executors.celery_executor import CeleryExecutor
-            executor = GetDefaultExecutor()
-            if not isinstance(executor, CeleryExecutor):
-                flash("Only works with the CeleryExecutor, sorry", "error")
-                return redirect(origin)
+            valid_celery_config = isinstance(executor, CeleryExecutor)
+        except ImportError:
+            pass
+
+        try:
+            from airflow.contrib.executors.kubernetes_executor import KubernetesExecutor
+            valid_kubernetes_config = isinstance(executor, KubernetesExecutor)
         except ImportError:
-            # in case CeleryExecutor cannot be imported it is not active either
-            flash("Only works with the CeleryExecutor, sorry", "error")
+            pass
+
+        if not valid_celery_config and not valid_kubernetes_config:
+            flash("Only works with the Celery or Kubernetes executors, sorry", "error")
             return redirect(origin)
 
         ti = models.TaskInstance(task=task, execution_date=execution_date)
@@ -1043,15 +1081,41 @@ def run(self):
             "it should start any moment now.".format(ti))
         return redirect(origin)
 
-    @expose('/trigger')
+    @expose('/delete', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     @wwwutils.notify_owner
-    def trigger(self):
-        dag_id = request.args.get('dag_id')
-        origin = request.args.get('origin') or "/admin/"
-        dag = dagbag.get_dag(dag_id)
+    def delete(self):
+        from airflow.api.common.experimental import delete_dag
+        from airflow.exceptions import DagNotFound, DagFileExists
+
+        dag_id = request.values.get('dag_id')
+        origin = request.values.get('origin') or "/admin/"
+
+        try:
+            delete_dag.delete_dag(dag_id)
+        except DagNotFound:
+            flash("DAG with id {} not found. Cannot delete".format(dag_id))
+            return redirect(request.referrer)
+        except DagFileExists:
+            flash("Dag id {} is still in DagBag. "
+                  "Remove the DAG file first.".format(dag_id))
+            return redirect(request.referrer)
+
+        flash("Deleting DAG with id {}. May take a couple minutes to fully"
+              " disappear.".format(dag_id))
+        # Upon successful delete return to origin
+        return redirect(origin)
 
+    @expose('/trigger', methods=['POST'])
+    @login_required
+    @wwwutils.action_logging
+    @wwwutils.notify_owner
+    @provide_session
+    def trigger(self, session=None):
+        dag_id = request.values.get('dag_id')
+        origin = request.values.get('origin') or "/admin/"
+        dag = session.query(models.DagModel).filter(models.DagModel.dag_id == dag_id).first()
         if not dag:
             flash("Cannot find dag {}".format(dag_id))
             return redirect(origin)
@@ -1085,7 +1149,9 @@ def _clear_dag_tis(self, dag, start_date, end_date, origin,
             count = dag.clear(
                 start_date=start_date,
                 end_date=end_date,
-                include_subdags=recursive)
+                include_subdags=recursive,
+                include_parentdag=recursive,
+            )
 
             flash("{0} task instances have been cleared".format(count))
             return redirect(origin)
@@ -1094,7 +1160,9 @@ def _clear_dag_tis(self, dag, start_date, end_date, origin,
             start_date=start_date,
             end_date=end_date,
             include_subdags=recursive,
-            dry_run=True)
+            dry_run=True,
+            include_parentdag=recursive,
+        )
         if not tis:
             flash("No task instances to clear", 'error')
             response = redirect(origin)
@@ -1109,24 +1177,24 @@ def _clear_dag_tis(self, dag, start_date, end_date, origin,
 
         return response
 
-    @expose('/clear')
+    @expose('/clear', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     @wwwutils.notify_owner
     def clear(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
         dag = dagbag.get_dag(dag_id)
 
-        execution_date = request.args.get('execution_date')
+        execution_date = request.form.get('execution_date')
         execution_date = pendulum.parse(execution_date)
-        confirmed = request.args.get('confirmed') == "true"
-        upstream = request.args.get('upstream') == "true"
-        downstream = request.args.get('downstream') == "true"
-        future = request.args.get('future') == "true"
-        past = request.args.get('past') == "true"
-        recursive = request.args.get('recursive') == "true"
+        confirmed = request.form.get('confirmed') == "true"
+        upstream = request.form.get('upstream') == "true"
+        downstream = request.form.get('downstream') == "true"
+        future = request.form.get('future') == "true"
+        past = request.form.get('past') == "true"
+        recursive = request.form.get('recursive') == "true"
 
         dag = dag.sub_dag(
             task_regex=r"^{0}$".format(task_id),
@@ -1139,16 +1207,15 @@ def clear(self):
         return self._clear_dag_tis(dag, start_date, end_date, origin,
                                    recursive=recursive, confirmed=confirmed)
 
-    @expose('/dagrun_clear')
+    @expose('/dagrun_clear', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     @wwwutils.notify_owner
     def dagrun_clear(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
-        execution_date = request.args.get('execution_date')
-        confirmed = request.args.get('confirmed') == "true"
+        dag_id = request.form.get('dag_id')
+        origin = request.form.get('origin')
+        execution_date = request.form.get('execution_date')
+        confirmed = request.form.get('confirmed') == "true"
 
         dag = dagbag.get_dag(dag_id)
         execution_date = pendulum.parse(execution_date)
@@ -1163,12 +1230,12 @@ def dagrun_clear(self):
     @provide_session
     def blocked(self, session=None):
         DR = models.DagRun
-        dags = (
-            session.query(DR.dag_id, sqla.func.count(DR.id))
-                .filter(DR.state == State.RUNNING)
-                .group_by(DR.dag_id)
-                .all()
-        )
+        dags = session\
+            .query(DR.dag_id, sqla.func.count(DR.id))\
+            .filter(DR.state == State.RUNNING)\
+            .group_by(DR.dag_id)\
+            .all()
+
         payload = []
         for dag_id, active_dag_runs in dags:
             max_active_runs = 0
@@ -1181,16 +1248,35 @@ def blocked(self, session=None):
             })
         return wwwutils.json_response(payload)
 
-    @expose('/dagrun_success')
-    @login_required
-    @wwwutils.action_logging
-    @wwwutils.notify_owner
-    def dagrun_success(self):
-        dag_id = request.args.get('dag_id')
-        execution_date = request.args.get('execution_date')
-        confirmed = request.args.get('confirmed') == 'true'
-        origin = request.args.get('origin')
+    def _mark_dagrun_state_as_failed(self, dag_id, execution_date, confirmed, origin):
+        if not execution_date:
+            flash('Invalid execution date', 'error')
+            return redirect(origin)
+
+        execution_date = pendulum.parse(execution_date)
+        dag = dagbag.get_dag(dag_id)
+
+        if not dag:
+            flash('Cannot find DAG: {}'.format(dag_id), 'error')
+            return redirect(origin)
+
+        new_dag_state = set_dag_run_state_to_failed(dag, execution_date, commit=confirmed)
+
+        if confirmed:
+            flash('Marked failed on {} task instances'.format(len(new_dag_state)))
+            return redirect(origin)
+
+        else:
+            details = '\n'.join([str(t) for t in new_dag_state])
+
+            response = self.render('airflow/confirm.html',
+                                   message=("Here's the list of task instances you are "
+                                            "about to mark as failed"),
+                                   details=details)
+
+            return response
 
+    def _mark_dagrun_state_as_success(self, dag_id, execution_date, confirmed, origin):
         if not execution_date:
             flash('Invalid execution date', 'error')
             return redirect(origin)
@@ -1202,8 +1288,8 @@ def dagrun_success(self):
             flash('Cannot find DAG: {}'.format(dag_id), 'error')
             return redirect(origin)
 
-        new_dag_state = set_dag_run_state(dag, execution_date, state=State.SUCCESS,
-                                          commit=confirmed)
+        new_dag_state = set_dag_run_state_to_success(dag, execution_date,
+                                                     commit=confirmed)
 
         if confirmed:
             flash('Marked success on {} task instances'.format(len(new_dag_state)))
@@ -1214,30 +1300,43 @@ def dagrun_success(self):
 
             response = self.render('airflow/confirm.html',
                                    message=("Here's the list of task instances you are "
-                                            "about to mark as successful:"),
+                                            "about to mark as success"),
                                    details=details)
 
             return response
 
-    @expose('/success')
+    @expose('/dagrun_failed', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     @wwwutils.notify_owner
-    def success(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
+    def dagrun_failed(self):
+        dag_id = request.form.get('dag_id')
+        execution_date = request.form.get('execution_date')
+        confirmed = request.form.get('confirmed') == 'true'
+        origin = request.form.get('origin')
+        return self._mark_dagrun_state_as_failed(dag_id, execution_date,
+                                                 confirmed, origin)
+
+    @expose('/dagrun_success', methods=['POST'])
+    @login_required
+    @wwwutils.action_logging
+    @wwwutils.notify_owner
+    def dagrun_success(self):
+        dag_id = request.form.get('dag_id')
+        execution_date = request.form.get('execution_date')
+        confirmed = request.form.get('confirmed') == 'true'
+        origin = request.form.get('origin')
+        return self._mark_dagrun_state_as_success(dag_id, execution_date,
+                                                  confirmed, origin)
+
+    def _mark_task_instance_state(self, dag_id, task_id, origin, execution_date,
+                                  confirmed, upstream, downstream,
+                                  future, past, state):
         dag = dagbag.get_dag(dag_id)
         task = dag.get_task(task_id)
         task.dag = dag
 
-        execution_date = request.args.get('execution_date')
         execution_date = pendulum.parse(execution_date)
-        confirmed = request.args.get('confirmed') == "true"
-        upstream = request.args.get('upstream') == "true"
-        downstream = request.args.get('downstream') == "true"
-        future = request.args.get('future') == "true"
-        past = request.args.get('past') == "true"
 
         if not dag:
             flash("Cannot find DAG: {}".format(dag_id))
@@ -1252,26 +1351,66 @@ def success(self):
         if confirmed:
             altered = set_state(task=task, execution_date=execution_date,
                                 upstream=upstream, downstream=downstream,
-                                future=future, past=past, state=State.SUCCESS,
+                                future=future, past=past, state=state,
                                 commit=True)
 
-            flash("Marked success on {} task instances".format(len(altered)))
+            flash("Marked {} on {} task instances".format(state, len(altered)))
             return redirect(origin)
 
         to_be_altered = set_state(task=task, execution_date=execution_date,
                                   upstream=upstream, downstream=downstream,
-                                  future=future, past=past, state=State.SUCCESS,
+                                  future=future, past=past, state=state,
                                   commit=False)
 
         details = "\n".join([str(t) for t in to_be_altered])
 
         response = self.render("airflow/confirm.html",
                                message=("Here's the list of task instances you are "
-                                        "about to mark as successful:"),
+                                        "about to mark as {}:".format(state)),
                                details=details)
 
         return response
 
+    @expose('/failed', methods=['POST'])
+    @login_required
+    @wwwutils.action_logging
+    @wwwutils.notify_owner
+    def failed(self):
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
+        execution_date = request.form.get('execution_date')
+
+        confirmed = request.form.get('confirmed') == "true"
+        upstream = request.form.get('upstream') == "true"
+        downstream = request.form.get('downstream') == "true"
+        future = request.form.get('future') == "true"
+        past = request.form.get('past') == "true"
+
+        return self._mark_task_instance_state(dag_id, task_id, origin, execution_date,
+                                              confirmed, upstream, downstream,
+                                              future, past, State.FAILED)
+
+    @expose('/success', methods=['POST'])
+    @login_required
+    @wwwutils.action_logging
+    @wwwutils.notify_owner
+    def success(self):
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
+        execution_date = request.form.get('execution_date')
+
+        confirmed = request.form.get('confirmed') == "true"
+        upstream = request.form.get('upstream') == "true"
+        downstream = request.form.get('downstream') == "true"
+        future = request.form.get('future') == "true"
+        past = request.form.get('past') == "true"
+
+        return self._mark_task_instance_state(dag_id, task_id, origin, execution_date,
+                                              confirmed, upstream, downstream,
+                                              future, past, State.SUCCESS)
+
     @expose('/tree')
     @login_required
     @wwwutils.gzipped
@@ -1282,6 +1421,10 @@ def tree(self, session=None):
         dag_id = request.args.get('dag_id')
         blur = conf.getboolean('webserver', 'demo_mode')
         dag = dagbag.get_dag(dag_id)
+        if dag_id not in dagbag.dags:
+            flash('DAG "{0}" seems to be missing.'.format(dag_id), "error")
+            return redirect('/admin/')
+
         root = request.args.get('root')
         if root:
             dag = dag.sub_dag(
@@ -1349,8 +1492,8 @@ def recurse_nodes(task, visited):
                 children_key = "_children"
 
             def set_duration(tid):
-                if (isinstance(tid, dict) and tid.get("state") == State.RUNNING and
-                            tid["start_date"] is not None):
+                if isinstance(tid, dict) and tid.get("state") == State.RUNNING \
+                        and tid["start_date"] is not None:
                     d = timezone.utcnow() - pendulum.parse(tid["start_date"])
                     tid["duration"] = d.total_seconds()
                 return tid
@@ -1377,12 +1520,9 @@ def set_duration(tid):
         data = {
             'name': '[DAG]',
             'children': [recurse_nodes(t, set()) for t in dag.roots],
-            'instances': [
-                dag_runs.get(d) or {'execution_date': d.isoformat()}
-                for d in dates],
+            'instances': [dag_runs.get(d) or {'execution_date': d.isoformat()} for d in dates],
         }
 
-        data = json.dumps(data, indent=4, default=json_ser)
         session.commit()
 
         form = DateTimeWithNumRunsForm(data={'base_date': max_date,
@@ -1489,10 +1629,10 @@ class GraphForm(DateTimeWithNumRunsWithDagRunsForm):
             ),
             blur=blur,
             root=root or '',
-            task_instances=json.dumps(task_instances, indent=2),
-            tasks=json.dumps(tasks, indent=2),
-            nodes=json.dumps(nodes, indent=2),
-            edges=json.dumps(edges, indent=2), )
+            task_instances=task_instances,
+            tasks=tasks,
+            nodes=nodes,
+            edges=edges)
 
     @expose('/duration')
     @login_required
@@ -1506,6 +1646,10 @@ def duration(self, session=None):
         num_runs = request.args.get('num_runs')
         num_runs = int(num_runs) if num_runs else default_dag_run
 
+        if dag is None:
+            flash('DAG "{0}" seems to be missing.'.format(dag_id), "error")
+            return redirect('/admin/')
+
         if base_date:
             base_date = pendulum.parse(base_date)
         else:
@@ -1536,13 +1680,13 @@ def duration(self, session=None):
         TF = models.TaskFail
         ti_fails = (
             session
-                .query(TF)
-                .filter(
+            .query(TF)
+            .filter(
                 TF.dag_id == dag.dag_id,
                 TF.execution_date >= min_date,
                 TF.execution_date <= base_date,
                 TF.task_id.in_([t.task_id for t in dag.tasks]))
-                .all()
+            .all()
         )
 
         fails_totals = defaultdict(int)
@@ -1750,10 +1894,10 @@ def landing_times(self, session=None):
     @provide_session
     def paused(self, session=None):
         DagModel = models.DagModel
-        dag_id = request.args.get('dag_id')
+        dag_id = request.values.get('dag_id')
         orm_dag = session.query(
             DagModel).filter(DagModel.dag_id == dag_id).first()
-        if request.args.get('is_paused') == 'false':
+        if request.values.get('is_paused') == 'false':
             orm_dag.is_paused = True
         else:
             orm_dag.is_paused = False
@@ -1763,30 +1907,29 @@ def paused(self, session=None):
         dagbag.get_dag(dag_id)
         return "OK"
 
-    @expose('/refresh')
+    @expose('/refresh', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     @provide_session
     def refresh(self, session=None):
-        DagModel = models.DagModel
-        dag_id = request.args.get('dag_id')
-        orm_dag = session.query(
-            DagModel).filter(DagModel.dag_id == dag_id).first()
+        # TODO: Is this method still needed after AIRFLOW-3561?
+        dm = models.DagModel
+        dag_id = request.values.get('dag_id')
+        orm_dag = session.query(dm).filter(dm.dag_id == dag_id).first()
 
         if orm_dag:
             orm_dag.last_expired = timezone.utcnow()
             session.merge(orm_dag)
         session.commit()
 
-        dagbag.get_dag(dag_id)
         flash("DAG [{}] is now fresh as a daisy".format(dag_id))
         return redirect(request.referrer)
 
-    @expose('/refresh_all')
+    @expose('/refresh_all', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     def refresh_all(self):
-        dagbag.collect_dags(only_if_updated=False)
+        # TODO: Is this method still needed after AIRFLOW-3561?
         flash("All DAGs are now up to date")
         return redirect('/')
 
@@ -1816,19 +1959,40 @@ def gantt(self, session=None):
             ti for ti in dag.get_task_instances(session, dttm, dttm)
             if ti.start_date]
         tis = sorted(tis, key=lambda ti: ti.start_date)
+        TF = models.TaskFail
+        ti_fails = list(itertools.chain(*[(
+            session
+            .query(TF)
+            .filter(TF.dag_id == ti.dag_id,
+                    TF.task_id == ti.task_id,
+                    TF.execution_date == ti.execution_date)
+            .all()
+        ) for ti in tis]))
 
-        tasks = []
+        # determine bars to show in the gantt chart
+        gantt_bar_items = []
         for ti in tis:
-            end_date = ti.end_date if ti.end_date else timezone.utcnow()
+            end_date = ti.end_date or timezone.utcnow()
+            gantt_bar_items.append((ti.task_id, ti.start_date, end_date, ti.state))
+        for tf in ti_fails:
+            end_date = tf.end_date or timezone.utcnow()
+            gantt_bar_items.append((tf.task_id, tf.start_date, end_date, State.FAILED))
+
+        tasks = []
+        for gantt_bar_item in gantt_bar_items:
+            task_id = gantt_bar_item[0]
+            start_date = gantt_bar_item[1]
+            end_date = gantt_bar_item[2]
+            state = gantt_bar_item[3]
             tasks.append({
-                'startDate': wwwutils.epoch(ti.start_date),
+                'startDate': wwwutils.epoch(start_date),
                 'endDate': wwwutils.epoch(end_date),
-                'isoStart': ti.start_date.isoformat()[:-4],
+                'isoStart': start_date.isoformat()[:-4],
                 'isoEnd': end_date.isoformat()[:-4],
-                'taskName': ti.task_id,
-                'duration': "{}".format(end_date - ti.start_date)[:-4],
-                'status': ti.state,
-                'executionDate': ti.execution_date.isoformat(),
+                'taskName': task_id,
+                'duration': "{}".format(end_date - start_date)[:-4],
+                'status': state,
+                'executionDate': dttm.isoformat(),
             })
         states = {ti.state: ti.state for ti in tis}
         data = {
@@ -1845,7 +2009,7 @@ def gantt(self, session=None):
             dag=dag,
             execution_date=dttm.isoformat(),
             form=form,
-            data=json.dumps(data, indent=2),
+            data=data,
             base_date='',
             demo_mode=demo_mode,
             root=root,
@@ -1863,7 +2027,7 @@ def task_instances(self, session=None):
         if dttm:
             dttm = pendulum.parse(dttm)
         else:
-            return ("Error: Invalid execution_date")
+            return "Error: Invalid execution_date"
 
         task_instances = {
             ti.task_id: alchemy_to_dict(ti)
@@ -1888,13 +2052,13 @@ def variables(self, form):
                 return self.render(
                     'airflow/variables/{}.html'.format(form)
                 )
-        except:
+        except Exception:
             # prevent XSS
             form = escape(form)
             return ("Error: form airflow/variables/{}.html "
                     "not found.").format(form), 404
 
-    @expose('/varimport', methods=["GET", "POST"])
+    @expose('/varimport', methods=['POST'])
     @login_required
     @wwwutils.action_logging
     def varimport(self):
@@ -1903,9 +2067,20 @@ def varimport(self):
         except Exception as e:
             flash("Missing file or syntax error: {}.".format(e))
         else:
+            suc_count = fail_count = 0
             for k, v in d.items():
-                models.Variable.set(k, v, serialize_json=isinstance(v, dict))
-            flash("{} variable(s) successfully updated.".format(len(d)))
+                try:
+                    models.Variable.set(k, v, serialize_json=isinstance(v, dict))
+                except Exception as e:
+                    logging.info('Variable import failed: {}'.format(repr(e)))
+                    fail_count += 1
+                else:
+                    suc_count += 1
+            flash("{} variable(s) successfully updated.".format(suc_count), 'info')
+            if fail_count:
+                flash(
+                    "{} variables(s) failed to be updated.".format(fail_count), 'error')
+
         return redirect('/admin/variable')
 
 
@@ -1944,108 +2119,63 @@ def get_int_arg(value, default=0):
             hide_paused = hide_paused_dags_by_default
 
         # read orm_dags from the db
-        sql_query = session.query(DM)
+        query = session.query(DM)
 
         if do_filter and owner_mode == 'ldapgroup':
-            sql_query = sql_query.filter(
+            query = query.filter(
                 ~DM.is_subdag,
                 DM.is_active,
                 DM.owners.in_(current_user.ldap_groups)
             )
         elif do_filter and owner_mode == 'user':
-            sql_query = sql_query.filter(
+            query = query.filter(
                 ~DM.is_subdag, DM.is_active,
                 DM.owners == current_user.user.username
             )
         else:
-            sql_query = sql_query.filter(
+            query = query.filter(
                 ~DM.is_subdag, DM.is_active
             )
 
         # optionally filter out "paused" dags
         if hide_paused:
-            sql_query = sql_query.filter(~DM.is_paused)
+            query = query.filter(~DM.is_paused)
+
+        if arg_search_query:
+            query = query.filter(sqla.func.lower(DM.dag_id) == arg_search_query.lower())
+
+        query = query.order_by(DM.dag_id)
+
+        start = current_page * dags_per_page
+        end = start + dags_per_page
 
-        orm_dags = {dag.dag_id: dag for dag
-                    in sql_query
-                    .all()}
+        dags = query.offset(start).limit(dags_per_page).all()
 
-        import_errors = session.query(models.ImportError).all()
+        import_errors = session.query(errors.ImportError).all()
         for ie in import_errors:
             flash(
                 "Broken DAG: [{ie.filename}] {ie.stacktrace}".format(ie=ie),
                 "error")
 
-        # get a list of all non-subdag dags visible to everyone
-        # optionally filter out "paused" dags
-        if hide_paused:
-            unfiltered_webserver_dags = [dag for dag in dagbag.dags.values() if
-                                         not dag.parent_dag and not dag.is_paused]
-
-        else:
-            unfiltered_webserver_dags = [dag for dag in dagbag.dags.values() if
-                                         not dag.parent_dag]
-
-        # optionally filter to get only dags that the user should see
-        if do_filter and owner_mode == 'ldapgroup':
-            # only show dags owned by someone in @current_user.ldap_groups
-            webserver_dags = {
-                dag.dag_id: dag
-                for dag in unfiltered_webserver_dags
-                if dag.owner in current_user.ldap_groups
-            }
-        elif do_filter and owner_mode == 'user':
-            # only show dags owned by @current_user.user.username
-            webserver_dags = {
-                dag.dag_id: dag
-                for dag in unfiltered_webserver_dags
-                if dag.owner == current_user.user.username
-            }
-        else:
-            webserver_dags = {
-                dag.dag_id: dag
-                for dag in unfiltered_webserver_dags
-            }
-
-        if arg_search_query:
-            lower_search_query = arg_search_query.lower()
-            # filter by dag_id
-            webserver_dags_filtered = {
-                dag_id: dag
-                for dag_id, dag in webserver_dags.items()
-                if (lower_search_query in dag_id.lower() or
-                    lower_search_query in dag.owner.lower())
-            }
-
-            all_dag_ids = (set([dag.dag_id for dag in orm_dags.values()
-                                if lower_search_query in dag.dag_id.lower() or
-                                lower_search_query in dag.owners.lower()]) |
-                           set(webserver_dags_filtered.keys()))
-
-            sorted_dag_ids = sorted(all_dag_ids)
-        else:
-            webserver_dags_filtered = webserver_dags
-            sorted_dag_ids = sorted(set(orm_dags.keys()) | set(webserver_dags.keys()))
-
-        start = current_page * dags_per_page
-        end = start + dags_per_page
+        from airflow.plugins_manager import import_errors as plugin_import_errors
+        for filename, stacktrace in plugin_import_errors.items():
+            flash(
+                "Broken plugin: [{filename}] {stacktrace}".format(
+                    stacktrace=stacktrace,
+                    filename=filename),
+                "error")
 
-        num_of_all_dags = len(sorted_dag_ids)
-        page_dag_ids = sorted_dag_ids[start:end]
+        num_of_all_dags = query.count()
         num_of_pages = int(math.ceil(num_of_all_dags / float(dags_per_page)))
 
         auto_complete_data = set()
-        for dag in webserver_dags_filtered.values():
-            auto_complete_data.add(dag.dag_id)
-            auto_complete_data.add(dag.owner)
-        for dag in orm_dags.values():
-            auto_complete_data.add(dag.dag_id)
-            auto_complete_data.add(dag.owners)
+        for row in query.with_entities(DM.dag_id, DM.owners):
+            auto_complete_data.add(row.dag_id)
+            auto_complete_data.add(row.owners)
 
         return self.render(
             'airflow/dags.html',
-            webserver_dags=webserver_dags_filtered,
-            orm_dags=orm_dags,
+            dags=dags,
             hide_paused=hide_paused,
             current_page=current_page,
             search_query=arg_search_query if arg_search_query else '',
@@ -2057,7 +2187,6 @@ def get_int_arg(value, default=0):
             paging=wwwutils.generate_pages(current_page, num_of_pages,
                                            search=arg_search_query,
                                            showPaused=not hide_paused),
-            dag_ids_in_page=page_dag_ids,
             auto_complete_data=auto_complete_data)
 
 
@@ -2066,8 +2195,7 @@ class QueryView(wwwutils.DataProfilingMixin, BaseView):
     @wwwutils.gzipped
     @provide_session
     def query(self, session=None):
-        dbs = session.query(models.Connection).order_by(
-            models.Connection.conn_id).all()
+        dbs = session.query(Connection).order_by(Connection.conn_id).all()
         session.expunge_all()
         db_choices = list(
             ((db.conn_id, db.conn_id) for db in dbs if db.get_hook()))
@@ -2086,7 +2214,7 @@ class QueryForm(Form):
         results = None
         has_data = False
         error = False
-        if conn_id_str:
+        if conn_id_str and request.method == 'POST':
             db = [db for db in dbs if db.conn_id == conn_id_str][0]
             hook = db.get_hook()
             try:
@@ -2183,13 +2311,10 @@ class SlaMissModelView(wwwutils.SuperUserMixin, ModelViewOnly):
 
 @provide_session
 def _connection_ids(session=None):
-    return [
-            (c.conn_id, c.conn_id)
-            for c in (
-                session.query(models.Connection.conn_id)
-                    .group_by(models.Connection.conn_id)
-            )
-    ]
+    return [(c.conn_id, c.conn_id) for c in (
+        session
+            .query(Connection.conn_id)
+            .group_by(Connection.conn_id))]
 
 
 class ChartModelView(wwwutils.DataProfilingMixin, AirflowModelView):
@@ -2296,7 +2421,7 @@ def on_model_change(self, form, model, is_created=True):
         model.last_modified = timezone.utcnow()
 
 
-chart_mapping = (
+chart_mapping = dict((
     ('line', 'lineChart'),
     ('spline', 'lineChart'),
     ('bar', 'multiBarChart'),
@@ -2305,8 +2430,7 @@ def on_model_change(self, form, model, is_created=True):
     ('stacked_area', 'stackedAreaChart'),
     ('percent_area', 'stackedAreaChart'),
     ('datatable', 'datatable'),
-)
-chart_mapping = dict(chart_mapping)
+))
 
 
 class KnownEventView(wwwutils.DataProfilingMixin, AirflowModelView):
@@ -2411,7 +2535,7 @@ def hidden_field_formatter(view, context, model, name):
     )
     column_list = ('key', 'val', 'is_encrypted',)
     column_filters = ('key', 'val')
-    column_searchable_list = ('key', 'val')
+    column_searchable_list = ('key', 'val', 'is_encrypted',)
     column_default_sort = ('key', False)
     form_widget_args = {
         'is_encrypted': {'disabled': True},
@@ -2448,7 +2572,7 @@ def action_varexport(self, ids, session=None):
             val = None
             try:
                 val = d.decode(var.val)
-            except:
+            except Exception:
                 val = var.val
             var_dict[var.key] = val
 
@@ -2522,9 +2646,18 @@ class DagRunModelView(ModelViewOnly):
             ('failed', 'failed'),
         ],
     }
-    form_args = dict(
-        dag_id=dict(validators=[validators.DataRequired()])
-    )
+    form_args = {
+        'dag_id': {
+            'validators': [
+                validators.DataRequired(),
+            ]
+        },
+        'execution_date': {
+            'filters': [
+                parse_datetime_f,
+            ]
+        }
+    }
     column_list = (
         'state', 'dag_id', 'execution_date', 'run_id', 'external_trigger')
     column_filters = column_list
@@ -2537,6 +2670,7 @@ class DagRunModelView(ModelViewOnly):
         dag_id=dag_link,
         run_id=dag_run_link
     )
+    form_overrides = dict(execution_date=DateTimeField)
 
     @action('new_delete', "Delete", "Are you sure you want to delete selected records?")
     @provide_session
@@ -2551,43 +2685,106 @@ def action_new_delete(self, ids, session=None):
         dirty_ids = []
         for row in deleted:
             dirty_ids.append(row.dag_id)
-        models.DagStat.update(dirty_ids, dirty_only=False, session=session)
 
     @action('set_running', "Set state to 'running'", None)
-    def action_set_running(self, ids):
-        self.set_dagrun_state(ids, State.RUNNING)
-
-    @action('set_failed', "Set state to 'failed'", None)
-    def action_set_failed(self, ids):
-        self.set_dagrun_state(ids, State.FAILED)
+    @provide_session
+    def action_set_running(self, ids, session=None):
+        try:
+            DR = models.DagRun
+            count = 0
+            dirty_ids = []
+            for dr in session.query(DR).filter(DR.id.in_(ids)).all():
+                dirty_ids.append(dr.dag_id)
+                count += 1
+                dr.state = State.RUNNING
+                dr.start_date = timezone.utcnow()
+            flash(
+                "{count} dag runs were set to running".format(**locals()))
+        except Exception as ex:
+            if not self.handle_view_exception(ex):
+                raise Exception("Ooops")
+            flash('Failed to set state', 'error')
 
-    @action('set_success', "Set state to 'success'", None)
-    def action_set_success(self, ids):
-        self.set_dagrun_state(ids, State.SUCCESS)
+    @action('set_failed', "Set state to 'failed'",
+            "All running task instances would also be marked as failed, are you sure?")
+    @provide_session
+    def action_set_failed(self, ids, session=None):
+        try:
+            DR = models.DagRun
+            count = 0
+            dirty_ids = []
+            altered_tis = []
+            for dr in session.query(DR).filter(DR.id.in_(ids)).all():
+                dirty_ids.append(dr.dag_id)
+                count += 1
+                altered_tis += \
+                    set_dag_run_state_to_failed(dagbag.get_dag(dr.dag_id),
+                                                dr.execution_date,
+                                                commit=True,
+                                                session=session)
+            altered_ti_count = len(altered_tis)
+            flash(
+                "{count} dag runs and {altered_ti_count} task instances "
+                "were set to failed".format(**locals()))
+        except Exception as ex:
+            if not self.handle_view_exception(ex):
+                raise Exception("Ooops")
+            flash('Failed to set state', 'error')
 
+    @action('set_success', "Set state to 'success'",
+            "All task instances would also be marked as success, are you sure?")
     @provide_session
-    def set_dagrun_state(self, ids, target_state, session=None):
+    def action_set_success(self, ids, session=None):
         try:
             DR = models.DagRun
             count = 0
             dirty_ids = []
+            altered_tis = []
             for dr in session.query(DR).filter(DR.id.in_(ids)).all():
                 dirty_ids.append(dr.dag_id)
                 count += 1
-                dr.state = target_state
-                if target_state == State.RUNNING:
-                    dr.start_date = timezone.utcnow()
-                else:
-                    dr.end_date = timezone.utcnow()
-            session.commit()
-            models.DagStat.update(dirty_ids, session=session)
+                altered_tis += \
+                    set_dag_run_state_to_success(dagbag.get_dag(dr.dag_id),
+                                                 dr.execution_date,
+                                                 commit=True,
+                                                 session=session)
+            altered_ti_count = len(altered_tis)
             flash(
-                "{count} dag runs were set to '{target_state}'".format(**locals()))
+                "{count} dag runs and {altered_ti_count} task instances "
+                "were set to success".format(**locals()))
         except Exception as ex:
             if not self.handle_view_exception(ex):
                 raise Exception("Ooops")
             flash('Failed to set state', 'error')
 
+    # Called after editing DagRun model in the UI.
+    @provide_session
+    def after_model_change(self, form, dagrun, is_created, session=None):
+        altered_tis = []
+        if dagrun.state == State.SUCCESS:
+            altered_tis = set_dag_run_state_to_success(
+                dagbag.get_dag(dagrun.dag_id),
+                dagrun.execution_date,
+                commit=True,
+                session=session)
+        elif dagrun.state == State.FAILED:
+            altered_tis = set_dag_run_state_to_failed(
+                dagbag.get_dag(dagrun.dag_id),
+                dagrun.execution_date,
+                commit=True,
+                session=session)
+        elif dagrun.state == State.RUNNING:
+            altered_tis = set_dag_run_state_to_running(
+                dagbag.get_dag(dagrun.dag_id),
+                dagrun.execution_date,
+                commit=True,
+                session=session)
+
+        altered_ti_count = len(altered_tis)
+        flash(
+            "1 dag run and {altered_ti_count} task instances "
+            "were set to '{dagrun.state}'".format(**locals()))
+
 
 class LogModelView(ModelViewOnly):
     verbose_name_plural = "logs"
@@ -2771,7 +2968,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
         'extra__google_cloud_platform__scope': StringField('Scopes (comma separated)'),
     }
     form_choices = {
-        'conn_type': models.Connection._types
+        'conn_type': Connection._types
     }
 
     def on_model_change(self, form, model, is_created):
@@ -2787,7 +2984,7 @@ def alert_fernet_key(cls):
         fk = None
         try:
             fk = conf.get('core', 'fernet_key')
-        except:
+        except Exception:
             pass
         return fk is None
 
@@ -2799,10 +2996,10 @@ def is_secure(cls):
         """
         is_secure = False
         try:
-            import cryptography
+            import cryptography  # noqa F401
             conf.get('core', 'fernet_key')
             is_secure = True
-        except:
+        except Exception:
             pass
         return is_secure
 
@@ -2830,7 +3027,7 @@ class VersionView(wwwutils.SuperUserMixin, BaseView):
     def version(self):
         # Look at the version from setup.py
         try:
-            airflow_version = pkg_resources.require("apache-airflow")[0].version
+            airflow_version = airflow.__version__
         except Exception as e:
             airflow_version = None
             logging.error(e)
@@ -2921,20 +3118,16 @@ def get_query(self):
         """
         Default filters for model
         """
-        return (
-            super(DagModelView, self)
-                .get_query()
-                .filter(or_(models.DagModel.is_active, models.DagModel.is_paused))
-                .filter(~models.DagModel.is_subdag)
-        )
+        return super(DagModelView, self)\
+            .get_query()\
+            .filter(or_(models.DagModel.is_active, models.DagModel.is_paused))\
+            .filter(~models.DagModel.is_subdag)
 
     def get_count_query(self):
         """
         Default filters for model
         """
-        return (
-            super(DagModelView, self)
-                .get_count_query()
-                .filter(models.DagModel.is_active)
-                .filter(~models.DagModel.is_subdag)
-        )
+        return super(DagModelView, self)\
+            .get_count_query()\
+            .filter(models.DagModel.is_active)\
+            .filter(~models.DagModel.is_subdag)
diff --git a/airflow/www_rbac/.eslintignore b/airflow/www_rbac/.eslintignore
new file mode 100644
index 0000000000000..924c9fc72b57d
--- /dev/null
+++ b/airflow/www_rbac/.eslintignore
@@ -0,0 +1,8 @@
+**/*{.,-}min.js
+**/*.sh
+**/*.py
+gantt-chart-d3v2.js
+jqClock.min.js
+coverage/**
+static/dist/*
+static/docs/*
diff --git a/airflow/www_rbac/.eslintrc b/airflow/www_rbac/.eslintrc
new file mode 100644
index 0000000000000..e566b6e0fb971
--- /dev/null
+++ b/airflow/www_rbac/.eslintrc
@@ -0,0 +1,5 @@
+{
+  "extends": "airbnb-base",
+  "parser": "babel-eslint",
+  "plugins": [ "html" ]
+}
diff --git a/airflow/www_rbac/__init__.py b/airflow/www_rbac/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/airflow/www_rbac/__init__.py
+++ b/airflow/www_rbac/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www_rbac/api/__init__.py b/airflow/www_rbac/api/__init__.py
index db5ba598d7c23..b7f8352944d3f 100644
--- a/airflow/www_rbac/api/__init__.py
+++ b/airflow/www_rbac/api/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www_rbac/api/experimental/__init__.py b/airflow/www_rbac/api/experimental/__init__.py
index db5ba598d7c23..b7f8352944d3f 100644
--- a/airflow/www_rbac/api/experimental/__init__.py
+++ b/airflow/www_rbac/api/experimental/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www_rbac/api/experimental/endpoints.py b/airflow/www_rbac/api/experimental/endpoints.py
index 8663bad55d631..637d3ef53d55a 100644
--- a/airflow/www_rbac/api/experimental/endpoints.py
+++ b/airflow/www_rbac/api/experimental/endpoints.py
@@ -23,6 +23,7 @@
 from airflow.api.common.experimental.get_dag_runs import get_dag_runs
 from airflow.api.common.experimental.get_task import get_task
 from airflow.api.common.experimental.get_task_instance import get_task_instance
+from airflow.api.common.experimental.get_code import get_code
 from airflow.api.common.experimental.get_dag_run_state import get_dag_run_state
 from airflow.exceptions import AirflowException
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -119,6 +120,19 @@ def test():
     return jsonify(status='OK')
 
 
+@api_experimental.route('/dags/<string:dag_id>/code', methods=['GET'])
+@requires_authentication
+def get_dag_code(dag_id):
+    """Return python code of a given dag_id."""
+    try:
+        return get_code(dag_id)
+    except AirflowException as err:
+        _log.info(err)
+        response = jsonify(error="{}".format(err))
+        response.status_code = err.status_code
+        return response
+
+
 @api_experimental.route('/dags/<string:dag_id>/tasks/<string:task_id>', methods=['GET'])
 @requires_authentication
 def task_info(dag_id, task_id):
@@ -237,6 +251,7 @@ def dag_run_status(dag_id, execution_date):
 
     return jsonify(info)
 
+
 @api_experimental.route('/latest_runs', methods=['GET'])
 @requires_authentication
 def latest_dag_runs():
diff --git a/airflow/www_rbac/app.py b/airflow/www_rbac/app.py
index 85125c6eb06b8..2af868682920d 100644
--- a/airflow/www_rbac/app.py
+++ b/airflow/www_rbac/app.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,37 +17,49 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+import logging
 import socket
-import six
+from typing import Any
 
+import six
 from flask import Flask
 from flask_appbuilder import AppBuilder, SQLA
 from flask_caching import Cache
 from flask_wtf.csrf import CSRFProtect
 from six.moves.urllib.parse import urlparse
 from werkzeug.wsgi import DispatcherMiddleware
+from werkzeug.contrib.fixers import ProxyFix
 
 from airflow import settings
 from airflow import configuration as conf
 from airflow.logging_config import configure_logging
+from airflow.www_rbac.static_config import configure_manifest_files
 
-
-app = None
+app = None  # type: Any
 appbuilder = None
 csrf = CSRFProtect()
 
+log = logging.getLogger(__name__)
 
-def create_app(config=None, testing=False, app_name="Airflow"):
+def create_app(config=None, session=None, testing=False, app_name="Airflow"):
     global app, appbuilder
     app = Flask(__name__)
+    if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'):
+        app.wsgi_app = ProxyFix(app.wsgi_app)
     app.secret_key = conf.get('webserver', 'SECRET_KEY')
 
-    airflow_home_path = conf.get('core', 'AIRFLOW_HOME')
-    webserver_config_path = airflow_home_path + '/webserver_config.py'
-    app.config.from_pyfile(webserver_config_path, silent=True)
+    app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True)
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
     app.config['APP_NAME'] = app_name
     app.config['TESTING'] = testing
 
+    app.config['SESSION_COOKIE_HTTPONLY'] = True
+    app.config['SESSION_COOKIE_SECURE'] = conf.getboolean('webserver', 'COOKIE_SECURE')
+    app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver', 'COOKIE_SAMESITE')
+
+    if config:
+        app.config.from_mapping(config)
+
     csrf.init_app(app)
 
     db = SQLA(app)
@@ -56,18 +68,30 @@ def create_app(config=None, testing=False, app_name="Airflow"):
     api.load_auth()
     api.api_auth.init_app(app)
 
-    cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})  # noqa
+    # flake8: noqa: F841
+    cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
 
     from airflow.www_rbac.blueprints import routes
     app.register_blueprint(routes)
 
     configure_logging()
+    configure_manifest_files(app)
 
     with app.app_context():
+
+        from airflow.www_rbac.security import AirflowSecurityManager
+        security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \
+            AirflowSecurityManager
+
+        if not issubclass(security_manager_class, AirflowSecurityManager):
+            raise Exception(
+                """Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager,
+                 not FAB's security manager.""")
+
         appbuilder = AppBuilder(
             app,
-            db.session,
-            security_manager_class=app.config.get('SECURITY_MANAGER_CLASS'),
+            db.session if not session else session,
+            security_manager_class=security_manager_class,
             base_template='appbuilder/baselayout.html')
 
         def init_views(appbuilder):
@@ -112,24 +136,50 @@ def init_views(appbuilder):
                                 href='https://airflow.apache.org/',
                                 category="Docs",
                                 category_icon="fa-cube")
-            appbuilder.add_link("Github",
-                                href='https://github.com/apache/incubator-airflow',
+            appbuilder.add_link("GitHub",
+                                href='https://github.com/apache/airflow',
                                 category="Docs")
             appbuilder.add_link('Version',
                                 href='/version',
                                 category='About',
                                 category_icon='fa-th')
 
+            def integrate_plugins():
+                """Integrate plugins to the context"""
+                from airflow.plugins_manager import (
+                    flask_appbuilder_views, flask_appbuilder_menu_links
+                )
+
+                for v in flask_appbuilder_views:
+                    log.debug("Adding view %s", v["name"])
+                    appbuilder.add_view(v["view"],
+                                        v["name"],
+                                        category=v["category"])
+                for ml in sorted(flask_appbuilder_menu_links, key=lambda x: x["name"]):
+                    log.debug("Adding menu link %s", ml["name"])
+                    appbuilder.add_link(ml["name"],
+                                        href=ml["href"],
+                                        category=ml["category"],
+                                        category_icon=ml["category_icon"])
+
+            integrate_plugins()
             # Garbage collect old permissions/views after they have been modified.
             # Otherwise, when the name of a view or menu is changed, the framework
             # will add the new Views and Menus names to the backend, but will not
             # delete the old ones.
-            appbuilder.security_cleanup()
+
+        def init_plugin_blueprints(app):
+            from airflow.plugins_manager import flask_blueprints
+
+            for bp in flask_blueprints:
+                log.debug("Adding blueprint %s:%s", bp["name"], bp["blueprint"].import_name)
+                app.register_blueprint(bp["blueprint"])
 
         init_views(appbuilder)
+        init_plugin_blueprints(app)
 
-        from airflow.www_rbac.security import init_roles
-        init_roles(appbuilder)
+        security_manager = appbuilder.sm
+        security_manager.sync_roles()
 
         from airflow.www_rbac.api.experimental import endpoints as e
         # required for testing purposes otherwise the module retains
@@ -158,23 +208,23 @@ def shutdown_session(exception=None):
 
 
 def root_app(env, resp):
-    resp(b'404 Not Found', [(b'Content-Type', b'text/plain')])
+    resp('404 Not Found', [('Content-Type', 'text/plain')])
     return [b'Apache Airflow is not at this location']
 
 
-def cached_app(config=None, testing=False):
+def cached_app(config=None, session=None, testing=False):
     global app, appbuilder
     if not app or not appbuilder:
         base_url = urlparse(conf.get('webserver', 'base_url'))[2]
         if not base_url or base_url == '/':
             base_url = ""
 
-        app, _ = create_app(config, testing)
+        app, _ = create_app(config, session, testing)
         app = DispatcherMiddleware(root_app, {base_url: app})
     return app
 
 
 def cached_appbuilder(config=None, testing=False):
     global appbuilder
-    cached_app(config, testing)
+    cached_app(config=config, testing=testing)
     return appbuilder
diff --git a/airflow/www_rbac/blueprints.py b/airflow/www_rbac/blueprints.py
index 35a5badb6ad1e..36c365b185ad6 100644
--- a/airflow/www_rbac/blueprints.py
+++ b/airflow/www_rbac/blueprints.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,19 +17,11 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-from flask import Markup, Blueprint, redirect
-import markdown
+from flask import Blueprint, redirect, url_for
 
 routes = Blueprint('routes', __name__)
 
 
 @routes.route('/')
 def index():
-    return redirect('/home')
-
-
-@routes.route('/health')
-def health():
-    """ We can add an array of tests here to check the server's health """
-    content = Markup(markdown.markdown("The server is healthy!"))
-    return content
+    return redirect(url_for('Airflow.index'))
diff --git a/airflow/www_rbac/compile_assets.sh b/airflow/www_rbac/compile_assets.sh
new file mode 100755
index 0000000000000..a3a4ea2349692
--- /dev/null
+++ b/airflow/www_rbac/compile_assets.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+# first bump up package.json manually, commit and tag
+if [ -d airflow/www_rbac/static/dist ]; then
+  rm -f airflow/www_rbac/static/dist/*
+fi
+
+cd airflow/www_rbac/
+npm install
+npm run build
+cd ../..
diff --git a/airflow/www_rbac/decorators.py b/airflow/www_rbac/decorators.py
index 2dd1af45df09d..1db2dfeb363b6 100644
--- a/airflow/www_rbac/decorators.py
+++ b/airflow/www_rbac/decorators.py
@@ -21,8 +21,9 @@
 import functools
 import pendulum
 from io import BytesIO as IO
-from flask import after_this_request, request, g
-from airflow import models, settings
+from flask import after_this_request, flash, redirect, request, url_for, g
+from airflow.models.log import Log
+from airflow.utils.db import create_session
 
 
 def action_logging(f):
@@ -31,26 +32,26 @@ def action_logging(f):
     """
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
-        session = settings.Session()
-        if g.user.is_anonymous():
-            user = 'anonymous'
-        else:
-            user = g.user.username
-
-        log = models.Log(
-            event=f.__name__,
-            task_instance=None,
-            owner=user,
-            extra=str(list(request.args.items())),
-            task_id=request.args.get('task_id'),
-            dag_id=request.args.get('dag_id'))
-
-        if 'execution_date' in request.args:
-            log.execution_date = pendulum.parse(
-                request.args.get('execution_date'))
-
-        session.add(log)
-        session.commit()
+
+        with create_session() as session:
+            if g.user.is_anonymous:
+                user = 'anonymous'
+            else:
+                user = g.user.username
+
+            log = Log(
+                event=f.__name__,
+                task_instance=None,
+                owner=user,
+                extra=str(list(request.args.items())),
+                task_id=request.args.get('task_id'),
+                dag_id=request.args.get('dag_id'))
+
+            if 'execution_date' in request.args:
+                log.execution_date = pendulum.parse(
+                    request.args.get('execution_date'))
+
+            session.add(log)
 
         return f(*args, **kwargs)
 
@@ -91,3 +92,36 @@ def zipper(response):
         return f(*args, **kwargs)
 
     return view_func
+
+
+def has_dag_access(**dag_kwargs):
+    """
+    Decorator to check whether the user has read / write permission on the dag.
+    """
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(self, *args, **kwargs):
+            has_access = self.appbuilder.sm.has_access
+            dag_id = request.args.get('dag_id')
+            # if it is false, we need to check whether user has write access on the dag
+            can_dag_edit = dag_kwargs.get('can_dag_edit', False)
+
+            # 1. check whether the user has can_dag_edit permissions on all_dags
+            # 2. if 1 false, check whether the user
+            #    has can_dag_edit permissions on the dag
+            # 3. if 2 false, check whether it is can_dag_read view,
+            #    and whether user has the permissions
+            if (
+                has_access('can_dag_edit', 'all_dags') or
+                has_access('can_dag_edit', dag_id) or (not can_dag_edit and
+                                                       (has_access('can_dag_read',
+                                                                   'all_dags') or
+                                                        has_access('can_dag_read',
+                                                                   dag_id)))):
+                return f(self, *args, **kwargs)
+            else:
+                flash("Access is Denied", "danger")
+                return redirect(url_for(self.appbuilder.sm.auth_view.
+                                        __class__.__name__ + ".login"))
+        return wrapper
+    return decorator
diff --git a/airflow/www_rbac/forms.py b/airflow/www_rbac/forms.py
index da9d12c7adfd9..0a36a90ef3ced 100644
--- a/airflow/www_rbac/forms.py
+++ b/airflow/www_rbac/forms.py
@@ -22,7 +22,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from airflow import models
+from airflow.models.connection import Connection
 from airflow.utils import timezone
 
 from flask_appbuilder.forms import DynamicForm
@@ -93,7 +93,7 @@ class ConnectionForm(DynamicForm):
         widget=BS3TextFieldWidget())
     conn_type = SelectField(
         lazy_gettext('Conn Type'),
-        choices=(models.Connection._types),
+        choices=Connection._types,
         widget=Select2Widget())
     host = StringField(
         lazy_gettext('Host'),
diff --git a/airflow/www_rbac/package-lock.json b/airflow/www_rbac/package-lock.json
new file mode 100644
index 0000000000000..c42050e276be5
--- /dev/null
+++ b/airflow/www_rbac/package-lock.json
@@ -0,0 +1,11434 @@
+{
+  "requires": true,
+  "lockfileVersion": 1,
+  "dependencies": {
+    "@babel/code-frame": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0-beta.44.tgz",
+      "integrity": "sha512-cuAuTTIQ9RqcFRJ/Y8PvTh+paepNcaGxwQwjIDRWPXmzzyAeCO4KqS9ikMvq0MCbRk6GlYKwfzStrcP3/jSL8g==",
+      "dev": true,
+      "requires": {
+        "@babel/highlight": "7.0.0-beta.44"
+      }
+    },
+    "@babel/generator": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.0.0-beta.44.tgz",
+      "integrity": "sha512-5xVb7hlhjGcdkKpMXgicAVgx8syK5VJz193k0i/0sLP6DzE6lRrU1K3B/rFefgdo9LPGMAOOOAWW4jycj07ShQ==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "7.0.0-beta.44",
+        "jsesc": "^2.5.1",
+        "lodash": "^4.2.0",
+        "source-map": "^0.5.0",
+        "trim-right": "^1.0.1"
+      },
+      "dependencies": {
+        "jsesc": {
+          "version": "2.5.1",
+          "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.1.tgz",
+          "integrity": "sha1-5CGiqOINawgZ3yiQj3glJrlt0f4=",
+          "dev": true
+        }
+      }
+    },
+    "@babel/helper-function-name": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.0.0-beta.44.tgz",
+      "integrity": "sha512-MHRG2qZMKMFaBavX0LWpfZ2e+hLloT++N7rfM3DYOMUOGCD8cVjqZpwiL8a0bOX3IYcQev1ruciT0gdFFRTxzg==",
+      "dev": true,
+      "requires": {
+        "@babel/helper-get-function-arity": "7.0.0-beta.44",
+        "@babel/template": "7.0.0-beta.44",
+        "@babel/types": "7.0.0-beta.44"
+      }
+    },
+    "@babel/helper-get-function-arity": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.0.0-beta.44.tgz",
+      "integrity": "sha512-w0YjWVwrM2HwP6/H3sEgrSQdkCaxppqFeJtAnB23pRiJB5E/O9Yp7JAAeWBl+gGEgmBFinnTyOv2RN7rcSmMiw==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "7.0.0-beta.44"
+      }
+    },
+    "@babel/helper-split-export-declaration": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.0.0-beta.44.tgz",
+      "integrity": "sha512-aQ7QowtkgKKzPGf0j6u77kBMdUFVBKNHw2p/3HX/POt5/oz8ec5cs0GwlgM8Hz7ui5EwJnzyfRmkNF1Nx1N7aA==",
+      "dev": true,
+      "requires": {
+        "@babel/types": "7.0.0-beta.44"
+      }
+    },
+    "@babel/highlight": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0-beta.44.tgz",
+      "integrity": "sha512-Il19yJvy7vMFm8AVAh6OZzaFoAd0hbkeMZiX3P5HGD+z7dyI7RzndHB0dg6Urh/VAFfHtpOIzDUSxmY6coyZWQ==",
+      "dev": true,
+      "requires": {
+        "chalk": "^2.0.0",
+        "esutils": "^2.0.2",
+        "js-tokens": "^3.0.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "@babel/template": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.0.0-beta.44.tgz",
+      "integrity": "sha512-w750Sloq0UNifLx1rUqwfbnC6uSUk0mfwwgGRfdLiaUzfAOiH0tHJE6ILQIUi3KYkjiCDTskoIsnfqZvWLBDng==",
+      "dev": true,
+      "requires": {
+        "@babel/code-frame": "7.0.0-beta.44",
+        "@babel/types": "7.0.0-beta.44",
+        "babylon": "7.0.0-beta.44",
+        "lodash": "^4.2.0"
+      },
+      "dependencies": {
+        "babylon": {
+          "version": "7.0.0-beta.44",
+          "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.44.tgz",
+          "integrity": "sha512-5Hlm13BJVAioCHpImtFqNOF2H3ieTOHd0fmFGMxOJ9jgeFqeAwsv3u5P5cR7CSeFrkgHsT19DgFJkHV0/Mcd8g==",
+          "dev": true
+        }
+      }
+    },
+    "@babel/traverse": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.0.0-beta.44.tgz",
+      "integrity": "sha512-UHuDz8ukQkJCDASKHf+oDt3FVUzFd+QYfuBIsiNu/4+/ix6pP/C+uQZJ6K1oEfbCMv/IKWbgDEh7fcsnIE5AtA==",
+      "dev": true,
+      "requires": {
+        "@babel/code-frame": "7.0.0-beta.44",
+        "@babel/generator": "7.0.0-beta.44",
+        "@babel/helper-function-name": "7.0.0-beta.44",
+        "@babel/helper-split-export-declaration": "7.0.0-beta.44",
+        "@babel/types": "7.0.0-beta.44",
+        "babylon": "7.0.0-beta.44",
+        "debug": "^3.1.0",
+        "globals": "^11.1.0",
+        "invariant": "^2.2.0",
+        "lodash": "^4.2.0"
+      },
+      "dependencies": {
+        "babylon": {
+          "version": "7.0.0-beta.44",
+          "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.44.tgz",
+          "integrity": "sha512-5Hlm13BJVAioCHpImtFqNOF2H3ieTOHd0fmFGMxOJ9jgeFqeAwsv3u5P5cR7CSeFrkgHsT19DgFJkHV0/Mcd8g==",
+          "dev": true
+        },
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "globals": {
+          "version": "11.7.0",
+          "resolved": "https://registry.npmjs.org/globals/-/globals-11.7.0.tgz",
+          "integrity": "sha512-K8BNSPySfeShBQXsahYB/AbbWruVOTyVpgoIDnl8odPpeSfP2J5QO2oLFFdl2j7GfDCtZj2bMKar2T49itTPCg==",
+          "dev": true
+        }
+      }
+    },
+    "@babel/types": {
+      "version": "7.0.0-beta.44",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.0.0-beta.44.tgz",
+      "integrity": "sha512-5eTV4WRmqbaFM3v9gHAIljEQJU4Ssc6fxL61JN+Oe2ga/BwyjzjamwkCVVAQjHGuAX8i0BWo42dshL8eO5KfLQ==",
+      "dev": true,
+      "requires": {
+        "esutils": "^2.0.2",
+        "lodash": "^4.2.0",
+        "to-fast-properties": "^2.0.0"
+      },
+      "dependencies": {
+        "to-fast-properties": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
+          "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=",
+          "dev": true
+        }
+      }
+    },
+    "@webassemblyjs/ast": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.5.13.tgz",
+      "integrity": "sha512-49nwvW/Hx9i+OYHg+mRhKZfAlqThr11Dqz8TsrvqGKMhdI2ijy3KBJOun2Z4770TPjrIJhR6KxChQIDaz8clDA==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/helper-module-context": "1.5.13",
+        "@webassemblyjs/helper-wasm-bytecode": "1.5.13",
+        "@webassemblyjs/wast-parser": "1.5.13",
+        "debug": "^3.1.0",
+        "mamacro": "^0.0.3"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
+    "@webassemblyjs/floating-point-hex-parser": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.5.13.tgz",
+      "integrity": "sha512-vrvvB18Kh4uyghSKb0NTv+2WZx871WL2NzwMj61jcq2bXkyhRC+8Q0oD7JGVf0+5i/fKQYQSBCNMMsDMRVAMqA==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-api-error": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.5.13.tgz",
+      "integrity": "sha512-dBh2CWYqjaDlvMmRP/kudxpdh30uXjIbpkLj9HQe+qtYlwvYjPRjdQXrq1cTAAOUSMTtzqbXIxEdEZmyKfcwsg==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-buffer": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.5.13.tgz",
+      "integrity": "sha512-v7igWf1mHcpJNbn4m7e77XOAWXCDT76Xe7Is1VQFXc4K5jRcFrl9D0NrqM4XifQ0bXiuTSkTKMYqDxu5MhNljA==",
+      "dev": true,
+      "requires": {
+        "debug": "^3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
+    "@webassemblyjs/helper-code-frame": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.5.13.tgz",
+      "integrity": "sha512-yN6ScQQDFCiAXnVctdVO/J5NQRbwyTbQzsGzEgXsAnrxhjp0xihh+nNHQTMrq5UhOqTb5LykpJAvEv9AT0jnAQ==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/wast-printer": "1.5.13"
+      }
+    },
+    "@webassemblyjs/helper-fsm": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.5.13.tgz",
+      "integrity": "sha512-hSIKzbXjVMRvy3Jzhgu+vDd/aswJ+UMEnLRCkZDdknZO3Z9e6rp1DAs0tdLItjCFqkz9+0BeOPK/mk3eYvVzZg==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-module-context": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.5.13.tgz",
+      "integrity": "sha512-zxJXULGPLB7r+k+wIlvGlXpT4CYppRz8fLUM/xobGHc9Z3T6qlmJD9ySJ2jknuktuuiR9AjnNpKYDECyaiX+QQ==",
+      "dev": true,
+      "requires": {
+        "debug": "^3.1.0",
+        "mamacro": "^0.0.3"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
+    "@webassemblyjs/helper-wasm-bytecode": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.5.13.tgz",
+      "integrity": "sha512-0n3SoNGLvbJIZPhtMFq0XmmnA/YmQBXaZKQZcW8maGKwLpVcgjNrxpFZHEOLKjXJYVN5Il8vSfG7nRX50Zn+aw==",
+      "dev": true
+    },
+    "@webassemblyjs/helper-wasm-section": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.5.13.tgz",
+      "integrity": "sha512-IJ/goicOZ5TT1axZFSnlAtz4m8KEjYr12BNOANAwGFPKXM4byEDaMNXYowHMG0yKV9a397eU/NlibFaLwr1fbw==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/helper-buffer": "1.5.13",
+        "@webassemblyjs/helper-wasm-bytecode": "1.5.13",
+        "@webassemblyjs/wasm-gen": "1.5.13",
+        "debug": "^3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
+    "@webassemblyjs/ieee754": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.5.13.tgz",
+      "integrity": "sha512-TseswvXEPpG5TCBKoLx9tT7+/GMACjC1ruo09j46ULRZWYm8XHpDWaosOjTnI7kr4SRJFzA6MWoUkAB+YCGKKg==",
+      "dev": true,
+      "requires": {
+        "ieee754": "^1.1.11"
+      }
+    },
+    "@webassemblyjs/leb128": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.5.13.tgz",
+      "integrity": "sha512-0NRMxrL+GG3eISGZBmLBLAVjphbN8Si15s7jzThaw1UE9e5BY1oH49/+MA1xBzxpf1OW5sf9OrPDOclk9wj2yg==",
+      "dev": true,
+      "requires": {
+        "long": "4.0.0"
+      },
+      "dependencies": {
+        "long": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+          "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==",
+          "dev": true
+        }
+      }
+    },
+    "@webassemblyjs/utf8": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.5.13.tgz",
+      "integrity": "sha512-Ve1ilU2N48Ew0lVGB8FqY7V7hXjaC4+PeZM+vDYxEd+R2iQ0q+Wb3Rw8v0Ri0+rxhoz6gVGsnQNb4FjRiEH/Ng==",
+      "dev": true
+    },
+    "@webassemblyjs/wasm-edit": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.5.13.tgz",
+      "integrity": "sha512-X7ZNW4+Hga4f2NmqENnHke2V/mGYK/xnybJSIXImt1ulxbCOEs/A+ZK/Km2jgihjyVxp/0z0hwIcxC6PrkWtgw==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/helper-buffer": "1.5.13",
+        "@webassemblyjs/helper-wasm-bytecode": "1.5.13",
+        "@webassemblyjs/helper-wasm-section": "1.5.13",
+        "@webassemblyjs/wasm-gen": "1.5.13",
+        "@webassemblyjs/wasm-opt": "1.5.13",
+        "@webassemblyjs/wasm-parser": "1.5.13",
+        "@webassemblyjs/wast-printer": "1.5.13",
+        "debug": "^3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
+    "@webassemblyjs/wasm-gen": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.5.13.tgz",
+      "integrity": "sha512-yfv94Se8R73zmr8GAYzezFHc3lDwE/lBXQddSiIZEKZFuqy7yWtm3KMwA1uGbv5G1WphimJxboXHR80IgX1hQA==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/helper-wasm-bytecode": "1.5.13",
+        "@webassemblyjs/ieee754": "1.5.13",
+        "@webassemblyjs/leb128": "1.5.13",
+        "@webassemblyjs/utf8": "1.5.13"
+      }
+    },
+    "@webassemblyjs/wasm-opt": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.5.13.tgz",
+      "integrity": "sha512-IkXSkgzVhQ0QYAdIayuCWMmXSYx0dHGU8Ah/AxJf1gBvstMWVnzJnBwLsXLyD87VSBIcsqkmZ28dVb0mOC3oBg==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/helper-buffer": "1.5.13",
+        "@webassemblyjs/wasm-gen": "1.5.13",
+        "@webassemblyjs/wasm-parser": "1.5.13",
+        "debug": "^3.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
+    "@webassemblyjs/wasm-parser": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.5.13.tgz",
+      "integrity": "sha512-XnYoIcu2iqq8/LrtmdnN3T+bRjqYFjRHqWbqK3osD/0r/Fcv4d9ecRzjVtC29ENEuNTK4mQ9yyxCBCbK8S/cpg==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/helper-api-error": "1.5.13",
+        "@webassemblyjs/helper-wasm-bytecode": "1.5.13",
+        "@webassemblyjs/ieee754": "1.5.13",
+        "@webassemblyjs/leb128": "1.5.13",
+        "@webassemblyjs/utf8": "1.5.13"
+      }
+    },
+    "@webassemblyjs/wast-parser": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.5.13.tgz",
+      "integrity": "sha512-Lbz65T0LQ1LgzKiUytl34CwuhMNhaCLgrh0JW4rJBN6INnBB8NMwUfQM+FxTnLY9qJ+lHJL/gCM5xYhB9oWi4A==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/floating-point-hex-parser": "1.5.13",
+        "@webassemblyjs/helper-api-error": "1.5.13",
+        "@webassemblyjs/helper-code-frame": "1.5.13",
+        "@webassemblyjs/helper-fsm": "1.5.13",
+        "long": "^3.2.0",
+        "mamacro": "^0.0.3"
+      }
+    },
+    "@webassemblyjs/wast-printer": {
+      "version": "1.5.13",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.5.13.tgz",
+      "integrity": "sha512-QcwogrdqcBh8Z+eUF8SG+ag5iwQSXxQJELBEHmLkk790wgQgnIMmntT2sMAMw53GiFNckArf5X0bsCA44j3lWQ==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/wast-parser": "1.5.13",
+        "long": "^3.2.0"
+      }
+    },
+    "@webpack-contrib/schema-utils": {
+      "version": "1.0.0-beta.0",
+      "resolved": "https://registry.npmjs.org/@webpack-contrib/schema-utils/-/schema-utils-1.0.0-beta.0.tgz",
+      "integrity": "sha512-LonryJP+FxQQHsjGBi6W786TQB1Oym+agTpY0c+Kj8alnIw+DLUJb6SI8Y1GHGhLCH1yPRrucjObUmxNICQ1pg==",
+      "dev": true,
+      "requires": {
+        "ajv": "^6.1.0",
+        "ajv-keywords": "^3.1.0",
+        "chalk": "^2.3.2",
+        "strip-ansi": "^4.0.0",
+        "text-table": "^0.2.0",
+        "webpack-log": "^1.1.2"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "abbrev": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz",
+      "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=",
+      "dev": true
+    },
+    "acorn": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.1.tgz",
+      "integrity": "sha512-d+nbxBUGKg7Arpsvbnlq61mc12ek3EY8EQldM3GPAhWJ1UVxC6TDGbIvUMNU6obBX3i1+ptCIzV4vq0gFPEGVQ==",
+      "dev": true
+    },
+    "acorn-dynamic-import": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-3.0.0.tgz",
+      "integrity": "sha512-zVWV8Z8lislJoOKKqdNMOB+s6+XV5WERty8MnKBeFgwA+19XJjJHs2RP5dzM57FftIs+jQnRToLiWazKr6sSWg==",
+      "dev": true,
+      "requires": {
+        "acorn": "^5.0.0"
+      }
+    },
+    "acorn-jsx": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz",
+      "integrity": "sha1-r9+UiPsezvyDSPb7IvRk4ypYs2s=",
+      "dev": true,
+      "requires": {
+        "acorn": "^3.0.4"
+      },
+      "dependencies": {
+        "acorn": {
+          "version": "3.3.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz",
+          "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=",
+          "dev": true
+        }
+      }
+    },
+    "ajv": {
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.2.tgz",
+      "integrity": "sha512-hOs7GfvI6tUI1LfZddH82ky6mOMyTuY0mk7kE2pWpmhhUSkumzaTO5vbVwij39MdwPQWCV4Zv57Eo06NtL/GVA==",
+      "dev": true,
+      "requires": {
+        "fast-deep-equal": "^2.0.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.1"
+      }
+    },
+    "ajv-keywords": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.2.0.tgz",
+      "integrity": "sha1-6GuBnGAs+IIa1jdBNpjx3sAhhHo=",
+      "dev": true
+    },
+    "align-text": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz",
+      "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "kind-of": "^3.0.2",
+        "longest": "^1.0.1",
+        "repeat-string": "^1.5.2"
+      }
+    },
+    "alphanum-sort": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz",
+      "integrity": "sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM=",
+      "dev": true
+    },
+    "amdefine": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz",
+      "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=",
+      "dev": true
+    },
+    "ansi-escapes": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz",
+      "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==",
+      "dev": true
+    },
+    "ansi-regex": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+      "dev": true
+    },
+    "ansi-styles": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+      "dev": true
+    },
+    "anymatch": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz",
+      "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==",
+      "dev": true,
+      "requires": {
+        "micromatch": "^3.1.4",
+        "normalize-path": "^2.1.1"
+      }
+    },
+    "aproba": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
+      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
+      "dev": true
+    },
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "dev": true,
+      "requires": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "arr-diff": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
+      "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=",
+      "dev": true
+    },
+    "arr-flatten": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
+      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==",
+      "dev": true
+    },
+    "arr-union": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
+      "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=",
+      "dev": true
+    },
+    "array-union": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz",
+      "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=",
+      "dev": true,
+      "requires": {
+        "array-uniq": "^1.0.1"
+      }
+    },
+    "array-uniq": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz",
+      "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=",
+      "dev": true
+    },
+    "array-unique": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
+      "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
+      "dev": true
+    },
+    "arrify": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz",
+      "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=",
+      "dev": true
+    },
+    "asn1.js": {
+      "version": "4.10.1",
+      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",
+      "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.0.0",
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0"
+      }
+    },
+    "assert": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/assert/-/assert-1.4.1.tgz",
+      "integrity": "sha1-mZEtWRg2tab1s0XA8H7vwI/GXZE=",
+      "dev": true,
+      "requires": {
+        "util": "0.10.3"
+      },
+      "dependencies": {
+        "inherits": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
+          "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=",
+          "dev": true
+        },
+        "util": {
+          "version": "0.10.3",
+          "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz",
+          "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=",
+          "dev": true,
+          "requires": {
+            "inherits": "2.0.1"
+          }
+        }
+      }
+    },
+    "assign-symbols": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
+      "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=",
+      "dev": true
+    },
+    "async": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+      "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
+      "dev": true
+    },
+    "async-each": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz",
+      "integrity": "sha1-GdOGodntxufByF04iu28xW0zYC0=",
+      "dev": true
+    },
+    "atob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.1.tgz",
+      "integrity": "sha1-ri1acpR38onWDdf5amMUoi3Wwio=",
+      "dev": true
+    },
+    "autoprefixer": {
+      "version": "6.7.7",
+      "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-6.7.7.tgz",
+      "integrity": "sha1-Hb0cg1ZY41zj+ZhAmdsAWFx4IBQ=",
+      "dev": true,
+      "requires": {
+        "browserslist": "^1.7.6",
+        "caniuse-db": "^1.0.30000634",
+        "normalize-range": "^0.1.2",
+        "num2fraction": "^1.2.2",
+        "postcss": "^5.2.16",
+        "postcss-value-parser": "^3.2.3"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "babel": {
+      "version": "6.23.0",
+      "resolved": "https://registry.npmjs.org/babel/-/babel-6.23.0.tgz",
+      "integrity": "sha1-0NHn2APpdHZb7qMjLU4VPA77kPQ=",
+      "dev": true
+    },
+    "babel-code-frame": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz",
+      "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=",
+      "dev": true,
+      "requires": {
+        "chalk": "^1.1.3",
+        "esutils": "^2.0.2",
+        "js-tokens": "^3.0.2"
+      }
+    },
+    "babel-core": {
+      "version": "6.26.3",
+      "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-6.26.3.tgz",
+      "integrity": "sha512-6jyFLuDmeidKmUEb3NM+/yawG0M2bDZ9Z1qbZP59cyHLz8kYGKYwpJP0UwUKKUiTRNvxfLesJnTedqczP7cTDA==",
+      "dev": true,
+      "requires": {
+        "babel-code-frame": "^6.26.0",
+        "babel-generator": "^6.26.0",
+        "babel-helpers": "^6.24.1",
+        "babel-messages": "^6.23.0",
+        "babel-register": "^6.26.0",
+        "babel-runtime": "^6.26.0",
+        "babel-template": "^6.26.0",
+        "babel-traverse": "^6.26.0",
+        "babel-types": "^6.26.0",
+        "babylon": "^6.18.0",
+        "convert-source-map": "^1.5.1",
+        "debug": "^2.6.9",
+        "json5": "^0.5.1",
+        "lodash": "^4.17.4",
+        "minimatch": "^3.0.4",
+        "path-is-absolute": "^1.0.1",
+        "private": "^0.1.8",
+        "slash": "^1.0.0",
+        "source-map": "^0.5.7"
+      }
+    },
+    "babel-eslint": {
+      "version": "8.2.6",
+      "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-8.2.6.tgz",
+      "integrity": "sha512-aCdHjhzcILdP8c9lej7hvXKvQieyRt20SF102SIGyY4cUIiw6UaAtK4j2o3dXX74jEmy0TJ0CEhv4fTIM3SzcA==",
+      "dev": true,
+      "requires": {
+        "@babel/code-frame": "7.0.0-beta.44",
+        "@babel/traverse": "7.0.0-beta.44",
+        "@babel/types": "7.0.0-beta.44",
+        "babylon": "7.0.0-beta.44",
+        "eslint-scope": "3.7.1",
+        "eslint-visitor-keys": "^1.0.0"
+      },
+      "dependencies": {
+        "babylon": {
+          "version": "7.0.0-beta.44",
+          "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.44.tgz",
+          "integrity": "sha512-5Hlm13BJVAioCHpImtFqNOF2H3ieTOHd0fmFGMxOJ9jgeFqeAwsv3u5P5cR7CSeFrkgHsT19DgFJkHV0/Mcd8g==",
+          "dev": true
+        }
+      }
+    },
+    "babel-generator": {
+      "version": "6.26.1",
+      "resolved": "https://registry.npmjs.org/babel-generator/-/babel-generator-6.26.1.tgz",
+      "integrity": "sha512-HyfwY6ApZj7BYTcJURpM5tznulaBvyio7/0d4zFOeMPUmfxkCjHocCuoLa2SAGzBI8AREcH3eP3758F672DppA==",
+      "dev": true,
+      "requires": {
+        "babel-messages": "^6.23.0",
+        "babel-runtime": "^6.26.0",
+        "babel-types": "^6.26.0",
+        "detect-indent": "^4.0.0",
+        "jsesc": "^1.3.0",
+        "lodash": "^4.17.4",
+        "source-map": "^0.5.7",
+        "trim-right": "^1.0.1"
+      }
+    },
+    "babel-helpers": {
+      "version": "6.24.1",
+      "resolved": "https://registry.npmjs.org/babel-helpers/-/babel-helpers-6.24.1.tgz",
+      "integrity": "sha1-NHHenK7DiOXIUOWX5Yom3fN2ArI=",
+      "dev": true,
+      "requires": {
+        "babel-runtime": "^6.22.0",
+        "babel-template": "^6.24.1"
+      }
+    },
+    "babel-istanbul": {
+      "version": "0.12.2",
+      "resolved": "https://registry.npmjs.org/babel-istanbul/-/babel-istanbul-0.12.2.tgz",
+      "integrity": "sha1-5yPwfJokMtiAVVILwi519cI5Fhw=",
+      "dev": true,
+      "requires": {
+        "abbrev": "1.0.x",
+        "async": "1.x",
+        "escodegen": "1.8.x",
+        "esprima": "2.7.x",
+        "handlebars": "^4.0.1",
+        "js-yaml": "3.x",
+        "mkdirp": "0.5.x",
+        "multi-glob": "^1.0.1",
+        "nopt": "3.x",
+        "object-assign": "^4.0.1",
+        "once": "1.x",
+        "resolve": "^1.1.0",
+        "source-map": "0.4.x",
+        "supports-color": "3.1.x",
+        "which": "1.2.x",
+        "wordwrap": "1.0.x"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.4.4",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
+          "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=",
+          "dev": true,
+          "requires": {
+            "amdefine": ">=0.0.4"
+          }
+        },
+        "supports-color": {
+          "version": "3.1.2",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz",
+          "integrity": "sha1-cqJiiU2dQIuVbKBf83su2KbiotU=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "babel-loader": {
+      "version": "7.1.5",
+      "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-7.1.5.tgz",
+      "integrity": "sha512-iCHfbieL5d1LfOQeeVJEUyD9rTwBcP/fcEbRCfempxTDuqrKpu0AZjLAQHEQa3Yqyj9ORKe2iHfoj4rHLf7xpw==",
+      "dev": true,
+      "requires": {
+        "find-cache-dir": "^1.0.0",
+        "loader-utils": "^1.0.2",
+        "mkdirp": "^0.5.1"
+      }
+    },
+    "babel-messages": {
+      "version": "6.23.0",
+      "resolved": "https://registry.npmjs.org/babel-messages/-/babel-messages-6.23.0.tgz",
+      "integrity": "sha1-8830cDhYA1sqKVHG7F7fbGLyYw4=",
+      "dev": true,
+      "requires": {
+        "babel-runtime": "^6.22.0"
+      }
+    },
+    "babel-plugin-css-modules-transform": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/babel-plugin-css-modules-transform/-/babel-plugin-css-modules-transform-1.6.1.tgz",
+      "integrity": "sha512-Iv98dKRgQnhuHFcn2heHH1OpHo9LGyoKjlsAkj6/Q3wkwpVyHrNfVua/WHnrwe2f7EHy1KANnOSg+q4AJ6ZzaQ==",
+      "dev": true,
+      "requires": {
+        "css-modules-require-hook": "^4.0.6",
+        "mkdirp": "^0.5.1"
+      }
+    },
+    "babel-polyfill": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.26.0.tgz",
+      "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=",
+      "dev": true,
+      "requires": {
+        "babel-runtime": "^6.26.0",
+        "core-js": "^2.5.0",
+        "regenerator-runtime": "^0.10.5"
+      },
+      "dependencies": {
+        "regenerator-runtime": {
+          "version": "0.10.5",
+          "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz",
+          "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=",
+          "dev": true
+        }
+      }
+    },
+    "babel-register": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-register/-/babel-register-6.26.0.tgz",
+      "integrity": "sha1-btAhFz4vy0htestFxgCahW9kcHE=",
+      "dev": true,
+      "requires": {
+        "babel-core": "^6.26.0",
+        "babel-runtime": "^6.26.0",
+        "core-js": "^2.5.0",
+        "home-or-tmp": "^2.0.0",
+        "lodash": "^4.17.4",
+        "mkdirp": "^0.5.1",
+        "source-map-support": "^0.4.15"
+      }
+    },
+    "babel-runtime": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
+      "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
+      "dev": true,
+      "requires": {
+        "core-js": "^2.4.0",
+        "regenerator-runtime": "^0.11.0"
+      }
+    },
+    "babel-template": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-template/-/babel-template-6.26.0.tgz",
+      "integrity": "sha1-3gPi0WOWsGn0bdn/+FIfsaDjXgI=",
+      "dev": true,
+      "requires": {
+        "babel-runtime": "^6.26.0",
+        "babel-traverse": "^6.26.0",
+        "babel-types": "^6.26.0",
+        "babylon": "^6.18.0",
+        "lodash": "^4.17.4"
+      }
+    },
+    "babel-traverse": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-traverse/-/babel-traverse-6.26.0.tgz",
+      "integrity": "sha1-RqnL1+3MYsjlwGTi0tjQ9ANXZu4=",
+      "dev": true,
+      "requires": {
+        "babel-code-frame": "^6.26.0",
+        "babel-messages": "^6.23.0",
+        "babel-runtime": "^6.26.0",
+        "babel-types": "^6.26.0",
+        "babylon": "^6.18.0",
+        "debug": "^2.6.8",
+        "globals": "^9.18.0",
+        "invariant": "^2.2.2",
+        "lodash": "^4.17.4"
+      }
+    },
+    "babel-types": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz",
+      "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=",
+      "dev": true,
+      "requires": {
+        "babel-runtime": "^6.26.0",
+        "esutils": "^2.0.2",
+        "lodash": "^4.17.4",
+        "to-fast-properties": "^1.0.3"
+      }
+    },
+    "babylon": {
+      "version": "6.18.0",
+      "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz",
+      "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==",
+      "dev": true
+    },
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "dev": true
+    },
+    "base": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
+      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
+      "dev": true,
+      "requires": {
+        "cache-base": "^1.0.1",
+        "class-utils": "^0.3.5",
+        "component-emitter": "^1.2.1",
+        "define-property": "^1.0.0",
+        "isobject": "^3.0.1",
+        "mixin-deep": "^1.2.0",
+        "pascalcase": "^0.1.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        },
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+          "dev": true
+        }
+      }
+    },
+    "base64-js": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz",
+      "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==",
+      "dev": true
+    },
+    "big.js": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.2.0.tgz",
+      "integrity": "sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==",
+      "dev": true
+    },
+    "binary-extensions": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.11.0.tgz",
+      "integrity": "sha1-RqoXUftqL5PuXmibsQh9SxTGwgU=",
+      "dev": true
+    },
+    "bluebird": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz",
+      "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==",
+      "dev": true
+    },
+    "bn.js": {
+      "version": "4.11.8",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+      "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA==",
+      "dev": true
+    },
+    "bootstrap-3-typeahead": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/bootstrap-3-typeahead/-/bootstrap-3-typeahead-4.0.2.tgz",
+      "integrity": "sha1-yxyWkESFaGIJb8jHHMIbOsu1BBI="
+    },
+    "bootstrap-toggle": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/bootstrap-toggle/-/bootstrap-toggle-2.2.2.tgz",
+      "integrity": "sha1-K4hTT8G5mGdPh3+Yug2LW3Q+lv4="
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "braces": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+      "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
+      "dev": true,
+      "requires": {
+        "arr-flatten": "^1.1.0",
+        "array-unique": "^0.3.2",
+        "extend-shallow": "^2.0.1",
+        "fill-range": "^4.0.0",
+        "isobject": "^3.0.1",
+        "repeat-element": "^1.1.2",
+        "snapdragon": "^0.8.1",
+        "snapdragon-node": "^2.0.1",
+        "split-string": "^3.0.2",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "brorand": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
+      "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=",
+      "dev": true
+    },
+    "browserify-aes": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
+      "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
+      "dev": true,
+      "requires": {
+        "buffer-xor": "^1.0.3",
+        "cipher-base": "^1.0.0",
+        "create-hash": "^1.1.0",
+        "evp_bytestokey": "^1.0.3",
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "browserify-cipher": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
+      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
+      "dev": true,
+      "requires": {
+        "browserify-aes": "^1.0.4",
+        "browserify-des": "^1.0.0",
+        "evp_bytestokey": "^1.0.0"
+      }
+    },
+    "browserify-des": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz",
+      "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==",
+      "dev": true,
+      "requires": {
+        "cipher-base": "^1.0.1",
+        "des.js": "^1.0.0",
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.1.2"
+      }
+    },
+    "browserify-rsa": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
+      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.1.0",
+        "randombytes": "^2.0.1"
+      }
+    },
+    "browserify-sign": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.0.4.tgz",
+      "integrity": "sha1-qk62jl17ZYuqa/alfmMMvXqT0pg=",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.1.1",
+        "browserify-rsa": "^4.0.0",
+        "create-hash": "^1.1.0",
+        "create-hmac": "^1.1.2",
+        "elliptic": "^6.0.0",
+        "inherits": "^2.0.1",
+        "parse-asn1": "^5.0.0"
+      }
+    },
+    "browserify-zlib": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz",
+      "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==",
+      "dev": true,
+      "requires": {
+        "pako": "~1.0.5"
+      }
+    },
+    "browserslist": {
+      "version": "1.7.7",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-1.7.7.tgz",
+      "integrity": "sha1-C9dnBCWL6CmyOYu1Dkti0aFmsLk=",
+      "dev": true,
+      "requires": {
+        "caniuse-db": "^1.0.30000639",
+        "electron-to-chromium": "^1.2.7"
+      }
+    },
+    "buffer": {
+      "version": "4.9.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
+      "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
+      "dev": true,
+      "requires": {
+        "base64-js": "^1.0.2",
+        "ieee754": "^1.1.4",
+        "isarray": "^1.0.0"
+      }
+    },
+    "buffer-from": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.0.tgz",
+      "integrity": "sha512-c5mRlguI/Pe2dSZmpER62rSCu0ryKmWddzRYsuXc50U2/g8jMOulc31VZMa4mYx31U5xsmSOpDCgH88Vl9cDGQ==",
+      "dev": true
+    },
+    "buffer-xor": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
+      "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=",
+      "dev": true
+    },
+    "builtin-modules": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
+      "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=",
+      "dev": true
+    },
+    "builtin-status-codes": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
+      "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=",
+      "dev": true
+    },
+    "cacache": {
+      "version": "10.0.4",
+      "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz",
+      "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==",
+      "dev": true,
+      "requires": {
+        "bluebird": "^3.5.1",
+        "chownr": "^1.0.1",
+        "glob": "^7.1.2",
+        "graceful-fs": "^4.1.11",
+        "lru-cache": "^4.1.1",
+        "mississippi": "^2.0.0",
+        "mkdirp": "^0.5.1",
+        "move-concurrently": "^1.0.1",
+        "promise-inflight": "^1.0.1",
+        "rimraf": "^2.6.2",
+        "ssri": "^5.2.4",
+        "unique-filename": "^1.1.0",
+        "y18n": "^4.0.0"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "dev": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        }
+      }
+    },
+    "cache-base": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
+      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
+      "dev": true,
+      "requires": {
+        "collection-visit": "^1.0.0",
+        "component-emitter": "^1.2.1",
+        "get-value": "^2.0.6",
+        "has-value": "^1.0.0",
+        "isobject": "^3.0.1",
+        "set-value": "^2.0.0",
+        "to-object-path": "^0.3.0",
+        "union-value": "^1.0.0",
+        "unset-value": "^1.0.0"
+      }
+    },
+    "caller-path": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz",
+      "integrity": "sha1-lAhe9jWB7NPaqSREqP6U6CV3dR8=",
+      "dev": true,
+      "requires": {
+        "callsites": "^0.2.0"
+      }
+    },
+    "callsites": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz",
+      "integrity": "sha1-r6uWJikQp/M8GaV3WCXGnzTjUMo=",
+      "dev": true
+    },
+    "camelcase": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz",
+      "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=",
+      "dev": true,
+      "optional": true
+    },
+    "caniuse-api": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-1.6.1.tgz",
+      "integrity": "sha1-tTTnxzTE+B7F++isoq0kNUuWLGw=",
+      "dev": true,
+      "requires": {
+        "browserslist": "^1.3.6",
+        "caniuse-db": "^1.0.30000529",
+        "lodash.memoize": "^4.1.2",
+        "lodash.uniq": "^4.5.0"
+      }
+    },
+    "caniuse-db": {
+      "version": "1.0.30000865",
+      "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000865.tgz",
+      "integrity": "sha1-gv+2TUD3VnYgqsAtOmMgeWiavGs=",
+      "dev": true
+    },
+    "center-align": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz",
+      "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "align-text": "^0.1.3",
+        "lazy-cache": "^1.0.3"
+      }
+    },
+    "chalk": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^2.2.1",
+        "escape-string-regexp": "^1.0.2",
+        "has-ansi": "^2.0.0",
+        "strip-ansi": "^3.0.0",
+        "supports-color": "^2.0.0"
+      }
+    },
+    "chardet": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.5.0.tgz",
+      "integrity": "sha512-9ZTaoBaePSCFvNlNGrsyI8ZVACP2svUtq0DkM7t4K2ClAa96sqOIRjAzDTc8zXzFt1cZR46rRzLTiHFSJ+Qw0g==",
+      "dev": true
+    },
+    "chokidar": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.4.tgz",
+      "integrity": "sha512-z9n7yt9rOvIJrMhvDtDictKrkFHeihkNl6uWMmZlmL6tJtX9Cs+87oK+teBx+JIgzvbX3yZHT3eF8vpbDxHJXQ==",
+      "dev": true,
+      "requires": {
+        "anymatch": "^2.0.0",
+        "async-each": "^1.0.0",
+        "braces": "^2.3.0",
+        "fsevents": "^1.2.2",
+        "glob-parent": "^3.1.0",
+        "inherits": "^2.0.1",
+        "is-binary-path": "^1.0.0",
+        "is-glob": "^4.0.0",
+        "lodash.debounce": "^4.0.8",
+        "normalize-path": "^2.1.1",
+        "path-is-absolute": "^1.0.0",
+        "readdirp": "^2.0.0",
+        "upath": "^1.0.5"
+      }
+    },
+    "chownr": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.0.1.tgz",
+      "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE=",
+      "dev": true
+    },
+    "chrome-trace-event": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.0.tgz",
+      "integrity": "sha512-xDbVgyfDTT2piup/h8dK/y4QZfJRSa73bw1WZ8b4XM1o7fsFubUVGYcE+1ANtOzJJELGpYoG2961z0Z6OAld9A==",
+      "dev": true,
+      "requires": {
+        "tslib": "^1.9.0"
+      }
+    },
+    "cipher-base": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
+      "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "circular-json": {
+      "version": "0.3.3",
+      "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz",
+      "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==",
+      "dev": true
+    },
+    "clap": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/clap/-/clap-1.2.3.tgz",
+      "integrity": "sha512-4CoL/A3hf90V3VIEjeuhSvlGFEHKzOz+Wfc2IVZc+FaUgU0ZQafJTP49fvnULipOPcAfqhyI2duwQyns6xqjYA==",
+      "dev": true,
+      "requires": {
+        "chalk": "^1.1.3"
+      }
+    },
+    "class-utils": {
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
+      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
+      "dev": true,
+      "requires": {
+        "arr-union": "^3.1.0",
+        "define-property": "^0.2.5",
+        "isobject": "^3.0.0",
+        "static-extend": "^0.1.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        }
+      }
+    },
+    "clean-webpack-plugin": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/clean-webpack-plugin/-/clean-webpack-plugin-0.1.19.tgz",
+      "integrity": "sha512-M1Li5yLHECcN2MahoreuODul5LkjohJGFxLPTjl3j1ttKrF5rgjZET1SJduuqxLAuT1gAPOdkhg03qcaaU1KeA==",
+      "dev": true,
+      "requires": {
+        "rimraf": "^2.6.1"
+      }
+    },
+    "cli-cursor": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz",
+      "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=",
+      "dev": true,
+      "requires": {
+        "restore-cursor": "^2.0.0"
+      }
+    },
+    "cli-width": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz",
+      "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=",
+      "dev": true
+    },
+    "cliui": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz",
+      "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "center-align": "^0.1.1",
+        "right-align": "^0.1.1",
+        "wordwrap": "0.0.2"
+      },
+      "dependencies": {
+        "wordwrap": {
+          "version": "0.0.2",
+          "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz",
+          "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=",
+          "dev": true,
+          "optional": true
+        }
+      }
+    },
+    "clone": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
+      "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
+      "dev": true
+    },
+    "co": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
+      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
+      "dev": true
+    },
+    "coa": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/coa/-/coa-1.0.4.tgz",
+      "integrity": "sha1-qe8VNmDWqGqL3sAomlxoTSF0Mv0=",
+      "dev": true,
+      "requires": {
+        "q": "^1.1.2"
+      }
+    },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
+      "dev": true
+    },
+    "collection-visit": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
+      "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=",
+      "dev": true,
+      "requires": {
+        "map-visit": "^1.0.0",
+        "object-visit": "^1.0.0"
+      }
+    },
+    "color": {
+      "version": "0.11.4",
+      "resolved": "https://registry.npmjs.org/color/-/color-0.11.4.tgz",
+      "integrity": "sha1-bXtcdPtl6EHNSHkq0e1eB7kE12Q=",
+      "dev": true,
+      "requires": {
+        "clone": "^1.0.2",
+        "color-convert": "^1.3.0",
+        "color-string": "^0.3.0"
+      }
+    },
+    "color-convert": {
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.2.tgz",
+      "integrity": "sha512-3NUJZdhMhcdPn8vJ9v2UQJoH0qqoGUkYTgFEPZaPjEtwmmKUfNV46zZmgB2M5M4DCEQHMaCfWHCxiBflLm04Tg==",
+      "dev": true,
+      "requires": {
+        "color-name": "1.1.1"
+      }
+    },
+    "color-name": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.1.tgz",
+      "integrity": "sha1-SxQVMEz1ACjqgWQ2Q72C6gWANok=",
+      "dev": true
+    },
+    "color-string": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-0.3.0.tgz",
+      "integrity": "sha1-J9RvtnAlxcL6JZk7+/V55HhBuZE=",
+      "dev": true,
+      "requires": {
+        "color-name": "^1.0.0"
+      }
+    },
+    "colormin": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/colormin/-/colormin-1.1.2.tgz",
+      "integrity": "sha1-6i90IKcrlogaOKrlnsEkpvcpgTM=",
+      "dev": true,
+      "requires": {
+        "color": "^0.11.0",
+        "css-color-names": "0.0.4",
+        "has": "^1.0.1"
+      }
+    },
+    "colors": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
+      "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=",
+      "dev": true
+    },
+    "commander": {
+      "version": "2.13.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz",
+      "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA==",
+      "dev": true
+    },
+    "commondir": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
+      "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
+      "dev": true
+    },
+    "component-emitter": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
+      "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=",
+      "dev": true
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
+    },
+    "concat-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+      "dev": true,
+      "requires": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.2.2",
+        "typedarray": "^0.0.6"
+      }
+    },
+    "console-browserify": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz",
+      "integrity": "sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA=",
+      "dev": true,
+      "requires": {
+        "date-now": "^0.1.4"
+      }
+    },
+    "constants-browserify": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
+      "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=",
+      "dev": true
+    },
+    "contains-path": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz",
+      "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=",
+      "dev": true
+    },
+    "convert-source-map": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.5.1.tgz",
+      "integrity": "sha1-uCeAl7m8IpNl3lxiz1/K7YtVmeU=",
+      "dev": true
+    },
+    "copy-concurrently": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz",
+      "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==",
+      "dev": true,
+      "requires": {
+        "aproba": "^1.1.1",
+        "fs-write-stream-atomic": "^1.0.8",
+        "iferr": "^0.1.5",
+        "mkdirp": "^0.5.1",
+        "rimraf": "^2.5.4",
+        "run-queue": "^1.0.0"
+      }
+    },
+    "copy-descriptor": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
+      "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=",
+      "dev": true
+    },
+    "copy-webpack-plugin": {
+      "version": "4.5.2",
+      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-4.5.2.tgz",
+      "integrity": "sha512-zmC33E8FFSq3AbflTvqvPvBo621H36Afsxlui91d+QyZxPIuXghfnTsa1CuqiAaCPgJoSUWfTFbKJnadZpKEbQ==",
+      "dev": true,
+      "requires": {
+        "cacache": "^10.0.4",
+        "find-cache-dir": "^1.0.0",
+        "globby": "^7.1.1",
+        "is-glob": "^4.0.0",
+        "loader-utils": "^1.1.0",
+        "minimatch": "^3.0.4",
+        "p-limit": "^1.0.0",
+        "serialize-javascript": "^1.4.0"
+      }
+    },
+    "core-js": {
+      "version": "2.5.7",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.7.tgz",
+      "integrity": "sha512-RszJCAxg/PP6uzXVXL6BsxSXx/B05oJAQ2vkJRjyjrEcNVycaqOmNb5OTxZPE3xa5gwZduqza6L9JOCenh/Ecw==",
+      "dev": true
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
+      "dev": true
+    },
+    "create-ecdh": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz",
+      "integrity": "sha512-GbEHQPMOswGpKXM9kCWVrremUcBmjteUaQ01T9rkKCPDXfUHX0IoP9LpHYo2NPFampa4e+/pFDc3jQdxrxQLaw==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.1.0",
+        "elliptic": "^6.0.0"
+      }
+    },
+    "create-hash": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
+      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
+      "dev": true,
+      "requires": {
+        "cipher-base": "^1.0.1",
+        "inherits": "^2.0.1",
+        "md5.js": "^1.3.4",
+        "ripemd160": "^2.0.1",
+        "sha.js": "^2.4.0"
+      }
+    },
+    "create-hmac": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
+      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
+      "dev": true,
+      "requires": {
+        "cipher-base": "^1.0.3",
+        "create-hash": "^1.1.0",
+        "inherits": "^2.0.1",
+        "ripemd160": "^2.0.0",
+        "safe-buffer": "^5.0.1",
+        "sha.js": "^2.4.8"
+      }
+    },
+    "cross-spawn": {
+      "version": "6.0.5",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
+      "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
+      "dev": true,
+      "requires": {
+        "nice-try": "^1.0.4",
+        "path-key": "^2.0.1",
+        "semver": "^5.5.0",
+        "shebang-command": "^1.2.0",
+        "which": "^1.2.9"
+      }
+    },
+    "crypto-browserify": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
+      "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
+      "dev": true,
+      "requires": {
+        "browserify-cipher": "^1.0.0",
+        "browserify-sign": "^4.0.0",
+        "create-ecdh": "^4.0.0",
+        "create-hash": "^1.1.0",
+        "create-hmac": "^1.1.0",
+        "diffie-hellman": "^5.0.0",
+        "inherits": "^2.0.1",
+        "pbkdf2": "^3.0.3",
+        "public-encrypt": "^4.0.0",
+        "randombytes": "^2.0.0",
+        "randomfill": "^1.0.3"
+      }
+    },
+    "css-color-names": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz",
+      "integrity": "sha1-gIrcLnnPhHOAabZGyyDsJ762KeA=",
+      "dev": true
+    },
+    "css-loader": {
+      "version": "0.28.11",
+      "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-0.28.11.tgz",
+      "integrity": "sha512-wovHgjAx8ZIMGSL8pTys7edA1ClmzxHeY6n/d97gg5odgsxEgKjULPR0viqyC+FWMCL9sfqoC/QCUBo62tLvPg==",
+      "dev": true,
+      "requires": {
+        "babel-code-frame": "^6.26.0",
+        "css-selector-tokenizer": "^0.7.0",
+        "cssnano": "^3.10.0",
+        "icss-utils": "^2.1.0",
+        "loader-utils": "^1.0.2",
+        "lodash.camelcase": "^4.3.0",
+        "object-assign": "^4.1.1",
+        "postcss": "^5.0.6",
+        "postcss-modules-extract-imports": "^1.2.0",
+        "postcss-modules-local-by-default": "^1.2.0",
+        "postcss-modules-scope": "^1.1.0",
+        "postcss-modules-values": "^1.3.0",
+        "postcss-value-parser": "^3.3.0",
+        "source-list-map": "^2.0.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "icss-utils": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-2.1.0.tgz",
+          "integrity": "sha1-g/Cg7DeL8yRheLbCrZE28TWxyWI=",
+          "dev": true,
+          "requires": {
+            "postcss": "^6.0.1"
+          },
+          "dependencies": {
+            "postcss": {
+              "version": "6.0.23",
+              "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.23.tgz",
+              "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==",
+              "dev": true,
+              "requires": {
+                "chalk": "^2.4.1",
+                "source-map": "^0.6.1",
+                "supports-color": "^5.4.0"
+              }
+            }
+          }
+        },
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          },
+          "dependencies": {
+            "ansi-styles": {
+              "version": "2.2.1",
+              "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+              "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+              "dev": true
+            },
+            "chalk": {
+              "version": "1.1.3",
+              "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+              "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+              "dev": true,
+              "requires": {
+                "ansi-styles": "^2.2.1",
+                "escape-string-regexp": "^1.0.2",
+                "has-ansi": "^2.0.0",
+                "strip-ansi": "^3.0.0",
+                "supports-color": "^2.0.0"
+              },
+              "dependencies": {
+                "supports-color": {
+                  "version": "2.0.0",
+                  "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+                  "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+                  "dev": true
+                }
+              }
+            },
+            "has-flag": {
+              "version": "1.0.0",
+              "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz",
+              "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=",
+              "dev": true
+            },
+            "source-map": {
+              "version": "0.5.7",
+              "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
+              "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=",
+              "dev": true
+            },
+            "supports-color": {
+              "version": "3.2.3",
+              "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+              "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+              "dev": true,
+              "requires": {
+                "has-flag": "^1.0.0"
+              }
+            }
+          }
+        },
+        "postcss-modules-extract-imports": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-1.2.0.tgz",
+          "integrity": "sha1-ZhQOzs447wa/DT41XWm/WdFB6oU=",
+          "dev": true,
+          "requires": {
+            "postcss": "^6.0.1"
+          },
+          "dependencies": {
+            "postcss": {
+              "version": "6.0.23",
+              "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.23.tgz",
+              "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==",
+              "dev": true,
+              "requires": {
+                "chalk": "^2.4.1",
+                "source-map": "^0.6.1",
+                "supports-color": "^5.4.0"
+              }
+            }
+          }
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "css-modules-require-hook": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/css-modules-require-hook/-/css-modules-require-hook-4.2.3.tgz",
+      "integrity": "sha1-Z5LKQSsV4j5vm+agfc739Xf/kE0=",
+      "dev": true,
+      "requires": {
+        "debug": "^2.2.0",
+        "generic-names": "^1.0.1",
+        "glob-to-regexp": "^0.3.0",
+        "icss-replace-symbols": "^1.0.2",
+        "lodash": "^4.3.0",
+        "postcss": "^6.0.1",
+        "postcss-modules-extract-imports": "^1.0.0",
+        "postcss-modules-local-by-default": "^1.0.1",
+        "postcss-modules-resolve-imports": "^1.3.0",
+        "postcss-modules-scope": "^1.0.0",
+        "postcss-modules-values": "^1.1.1",
+        "seekout": "^1.0.1"
+      }
+    },
+    "css-selector-tokenizer": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/css-selector-tokenizer/-/css-selector-tokenizer-0.7.0.tgz",
+      "integrity": "sha1-5piEdK6MlTR3v15+/s/OzNnPTIY=",
+      "dev": true,
+      "requires": {
+        "cssesc": "^0.1.0",
+        "fastparse": "^1.1.1",
+        "regexpu-core": "^1.0.0"
+      }
+    },
+    "cssesc": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-0.1.0.tgz",
+      "integrity": "sha1-yBSQPkViM3GgR3tAEJqq++6t27Q=",
+      "dev": true
+    },
+    "cssnano": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-3.10.0.tgz",
+      "integrity": "sha1-Tzj2zqK5sX+gFJDyPx3GjqZcHDg=",
+      "dev": true,
+      "requires": {
+        "autoprefixer": "^6.3.1",
+        "decamelize": "^1.1.2",
+        "defined": "^1.0.0",
+        "has": "^1.0.1",
+        "object-assign": "^4.0.1",
+        "postcss": "^5.0.14",
+        "postcss-calc": "^5.2.0",
+        "postcss-colormin": "^2.1.8",
+        "postcss-convert-values": "^2.3.4",
+        "postcss-discard-comments": "^2.0.4",
+        "postcss-discard-duplicates": "^2.0.1",
+        "postcss-discard-empty": "^2.0.1",
+        "postcss-discard-overridden": "^0.1.1",
+        "postcss-discard-unused": "^2.2.1",
+        "postcss-filter-plugins": "^2.0.0",
+        "postcss-merge-idents": "^2.1.5",
+        "postcss-merge-longhand": "^2.0.1",
+        "postcss-merge-rules": "^2.0.3",
+        "postcss-minify-font-values": "^1.0.2",
+        "postcss-minify-gradients": "^1.0.1",
+        "postcss-minify-params": "^1.0.4",
+        "postcss-minify-selectors": "^2.0.4",
+        "postcss-normalize-charset": "^1.1.0",
+        "postcss-normalize-url": "^3.0.7",
+        "postcss-ordered-values": "^2.1.0",
+        "postcss-reduce-idents": "^2.2.2",
+        "postcss-reduce-initial": "^1.0.0",
+        "postcss-reduce-transforms": "^1.0.3",
+        "postcss-svgo": "^2.1.1",
+        "postcss-unique-selectors": "^2.0.2",
+        "postcss-value-parser": "^3.2.3",
+        "postcss-zindex": "^2.0.1"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "csso": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/csso/-/csso-2.3.2.tgz",
+      "integrity": "sha1-3dUsWHAz9J6Utx/FVWnyUuj/X4U=",
+      "dev": true,
+      "requires": {
+        "clap": "^1.0.9",
+        "source-map": "^0.5.3"
+      }
+    },
+    "cyclist": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-0.2.2.tgz",
+      "integrity": "sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA=",
+      "dev": true
+    },
+    "d": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz",
+      "integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=",
+      "dev": true,
+      "requires": {
+        "es5-ext": "^0.10.9"
+      }
+    },
+    "d3": {
+      "version": "3.5.17",
+      "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz",
+      "integrity": "sha1-vEZ0gAQ3iyGjYMn8fPUjF5B2L7g="
+    },
+    "d3-collection": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/d3-collection/-/d3-collection-1.0.4.tgz",
+      "integrity": "sha1-NC39EoN8kJdPM/HMCnha6lcNzcI="
+    },
+    "d3-selection": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.3.0.tgz",
+      "integrity": "sha512-qgpUOg9tl5CirdqESUAu0t9MU/t3O9klYfGfyKsXEmhyxyzLpzpeh08gaxBUTQw1uXIOkr/30Ut2YRjSSxlmHA=="
+    },
+    "d3-tip": {
+      "version": "0.9.1",
+      "resolved": "https://registry.npmjs.org/d3-tip/-/d3-tip-0.9.1.tgz",
+      "integrity": "sha512-EVBfG9d+HnjIoyVXfhpytWxlF59JaobwizqMX9EBXtsFmJytjwHeYiUs74ldHQjE7S9vzfKTx2LCtvUrIbuFYg==",
+      "requires": {
+        "d3-collection": "^1.0.4",
+        "d3-selection": "^1.3.0"
+      }
+    },
+    "dagre": {
+      "version": "0.7.4",
+      "resolved": "https://registry.npmjs.org/dagre/-/dagre-0.7.4.tgz",
+      "integrity": "sha1-3nLw50pVDOEc5jjwoTb+1xI5gCI=",
+      "requires": {
+        "graphlib": "^1.0.5",
+        "lodash": "^3.10.0"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y="
+        }
+      }
+    },
+    "dagre-d3": {
+      "version": "0.4.18",
+      "resolved": "https://registry.npmjs.org/dagre-d3/-/dagre-d3-0.4.18.tgz",
+      "integrity": "sha512-7tRltaOfNTIkNEZYMCL8N3Q8bCre99x/mAJL2RbuUfPu5d+4f0KOHglZm+AzOG2Z/+S2HBDYciE6iDcDtki6Tg==",
+      "requires": {
+        "d3": "^3.3.8",
+        "dagre": "^0.7.3",
+        "graphlib": "^1.0.5",
+        "lodash": "^3.10.0"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y="
+        }
+      }
+    },
+    "datatables.net": {
+      "version": "1.10.19",
+      "resolved": "https://registry.npmjs.org/datatables.net/-/datatables.net-1.10.19.tgz",
+      "integrity": "sha512-+ljXcI6Pj3PTGy5pesp3E5Dr3x3AV45EZe0o1r0gKENN2gafBKXodVnk2ypKwl2tTmivjxbkiqoWnipTefyBTA==",
+      "requires": {
+        "jquery": ">=1.7"
+      }
+    },
+    "datatables.net-bs": {
+      "version": "1.10.19",
+      "resolved": "https://registry.npmjs.org/datatables.net-bs/-/datatables.net-bs-1.10.19.tgz",
+      "integrity": "sha512-5gxoI2n+duZP06+4xVC2TtH6zcY369/TRKTZ1DdSgDcDUl4OYQsrXCuaLJmbVzna/5Y5lrMmK7CxgvYgIynICA==",
+      "requires": {
+        "datatables.net": "1.10.19",
+        "jquery": ">=1.7"
+      }
+    },
+    "date-now": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz",
+      "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs=",
+      "dev": true
+    },
+    "debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "dev": true,
+      "requires": {
+        "ms": "2.0.0"
+      }
+    },
+    "decamelize": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
+      "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
+      "dev": true
+    },
+    "decode-uri-component": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
+      "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=",
+      "dev": true
+    },
+    "deep-is": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
+      "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+      "dev": true
+    },
+    "define-properties": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
+      "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
+      "dev": true,
+      "requires": {
+        "foreach": "^2.0.5",
+        "object-keys": "^1.0.8"
+      }
+    },
+    "define-property": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
+      "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
+      "dev": true,
+      "requires": {
+        "is-descriptor": "^1.0.2",
+        "isobject": "^3.0.1"
+      },
+      "dependencies": {
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        },
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+          "dev": true
+        }
+      }
+    },
+    "defined": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz",
+      "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=",
+      "dev": true
+    },
+    "del": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/del/-/del-2.2.2.tgz",
+      "integrity": "sha1-wSyYHQZ4RshLyvhiz/kw2Qf/0ag=",
+      "dev": true,
+      "requires": {
+        "globby": "^5.0.0",
+        "is-path-cwd": "^1.0.0",
+        "is-path-in-cwd": "^1.0.0",
+        "object-assign": "^4.0.1",
+        "pify": "^2.0.0",
+        "pinkie-promise": "^2.0.0",
+        "rimraf": "^2.2.8"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "dev": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        },
+        "globby": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz",
+          "integrity": "sha1-69hGZ8oNuzMLmbz8aOrCvFQ3Dg0=",
+          "dev": true,
+          "requires": {
+            "array-union": "^1.0.1",
+            "arrify": "^1.0.0",
+            "glob": "^7.0.3",
+            "object-assign": "^4.0.1",
+            "pify": "^2.0.0",
+            "pinkie-promise": "^2.0.0"
+          }
+        },
+        "pify": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "dev": true
+        }
+      }
+    },
+    "des.js": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.0.tgz",
+      "integrity": "sha1-wHTS4qpqipoH29YfmhXCzYPsjsw=",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0"
+      }
+    },
+    "detect-indent": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-4.0.0.tgz",
+      "integrity": "sha1-920GQ1LN9Docts5hnE7jqUdd4gg=",
+      "dev": true,
+      "requires": {
+        "repeating": "^2.0.0"
+      }
+    },
+    "diffie-hellman": {
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
+      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.1.0",
+        "miller-rabin": "^4.0.0",
+        "randombytes": "^2.0.0"
+      }
+    },
+    "dir-glob": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.0.0.tgz",
+      "integrity": "sha512-37qirFDz8cA5fimp9feo43fSuRo2gHwaIn6dXL8Ber1dGwUosDrGZeCCXq57WnIqE4aQ+u3eQZzsk1yOzhdwag==",
+      "dev": true,
+      "requires": {
+        "arrify": "^1.0.1",
+        "path-type": "^3.0.0"
+      }
+    },
+    "doctrine": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
+      "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==",
+      "dev": true,
+      "requires": {
+        "esutils": "^2.0.2"
+      }
+    },
+    "dom-serializer": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz",
+      "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=",
+      "dev": true,
+      "requires": {
+        "domelementtype": "~1.1.1",
+        "entities": "~1.1.1"
+      },
+      "dependencies": {
+        "domelementtype": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
+          "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=",
+          "dev": true
+        }
+      }
+    },
+    "domain-browser": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
+      "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
+      "dev": true
+    },
+    "domelementtype": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz",
+      "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=",
+      "dev": true
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "dev": true,
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz",
+      "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==",
+      "dev": true,
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
+    "duplexify": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.0.tgz",
+      "integrity": "sha512-fO3Di4tBKJpYTFHAxTU00BcfWMY9w24r/x21a6rZRbsD/ToUgGxsMbiGRmB7uVAXeGKXD9MwiLZa5E97EVgIRQ==",
+      "dev": true,
+      "requires": {
+        "end-of-stream": "^1.0.0",
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.0",
+        "stream-shift": "^1.0.0"
+      }
+    },
+    "electron-to-chromium": {
+      "version": "1.3.51",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.51.tgz",
+      "integrity": "sha1-akK0nar38ipbN7mR2vlJ8029ubU=",
+      "dev": true
+    },
+    "elliptic": {
+      "version": "6.4.0",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.0.tgz",
+      "integrity": "sha1-ysmvh2LIWDYYcAPI3+GT5eLq5d8=",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.4.0",
+        "brorand": "^1.0.1",
+        "hash.js": "^1.0.0",
+        "hmac-drbg": "^1.0.0",
+        "inherits": "^2.0.1",
+        "minimalistic-assert": "^1.0.0",
+        "minimalistic-crypto-utils": "^1.0.0"
+      }
+    },
+    "emojis-list": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz",
+      "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k=",
+      "dev": true
+    },
+    "end-of-stream": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
+      "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
+      "dev": true,
+      "requires": {
+        "once": "^1.4.0"
+      }
+    },
+    "enhanced-resolve": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz",
+      "integrity": "sha512-F/7vkyTtyc/llOIn8oWclcB25KdRaiPBpZYDgJHgh/UHtpgT2p2eldQgtQnLtUvfMKPKxbRaQM/hHkvLHt1Vng==",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "memory-fs": "^0.4.0",
+        "tapable": "^1.0.0"
+      }
+    },
+    "entities": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
+      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=",
+      "dev": true
+    },
+    "errno": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
+      "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==",
+      "dev": true,
+      "requires": {
+        "prr": "~1.0.1"
+      }
+    },
+    "error-ex": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
+      "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
+      "dev": true,
+      "requires": {
+        "is-arrayish": "^0.2.1"
+      }
+    },
+    "es-abstract": {
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz",
+      "integrity": "sha512-C8Fx/0jFmV5IPoMOFPA9P9G5NtqW+4cOPit3MIuvR2t7Ag2K15EJTpxnHAYTzL+aYQJIESYeXZmDBfOBE1HcpA==",
+      "dev": true,
+      "requires": {
+        "es-to-primitive": "^1.1.1",
+        "function-bind": "^1.1.1",
+        "has": "^1.0.1",
+        "is-callable": "^1.1.3",
+        "is-regex": "^1.0.4"
+      }
+    },
+    "es-to-primitive": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
+      "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
+      "dev": true,
+      "requires": {
+        "is-callable": "^1.1.1",
+        "is-date-object": "^1.0.1",
+        "is-symbol": "^1.0.1"
+      }
+    },
+    "es5-ext": {
+      "version": "0.10.45",
+      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.45.tgz",
+      "integrity": "sha512-FkfM6Vxxfmztilbxxz5UKSD4ICMf5tSpRFtDNtkAhOxZ0EKtX6qwmXNyH/sFyIbX2P/nU5AMiA9jilWsUGJzCQ==",
+      "dev": true,
+      "requires": {
+        "es6-iterator": "~2.0.3",
+        "es6-symbol": "~3.1.1",
+        "next-tick": "1"
+      }
+    },
+    "es6-iterator": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
+      "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
+      "dev": true,
+      "requires": {
+        "d": "1",
+        "es5-ext": "^0.10.35",
+        "es6-symbol": "^3.1.1"
+      }
+    },
+    "es6-symbol": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz",
+      "integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=",
+      "dev": true,
+      "requires": {
+        "d": "1",
+        "es5-ext": "~0.10.14"
+      }
+    },
+    "escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
+      "dev": true
+    },
+    "escodegen": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz",
+      "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=",
+      "dev": true,
+      "requires": {
+        "esprima": "^2.7.1",
+        "estraverse": "^1.9.1",
+        "esutils": "^2.0.2",
+        "optionator": "^0.8.1",
+        "source-map": "~0.2.0"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.2.0",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz",
+          "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=",
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "amdefine": ">=0.0.4"
+          }
+        }
+      }
+    },
+    "eslint": {
+      "version": "4.19.1",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.19.1.tgz",
+      "integrity": "sha512-bT3/1x1EbZB7phzYu7vCr1v3ONuzDtX8WjuM9c0iYxe+cq+pwcKEoQjl7zd3RpC6YOLgnSy3cTN58M2jcoPDIQ==",
+      "dev": true,
+      "requires": {
+        "ajv": "^5.3.0",
+        "babel-code-frame": "^6.22.0",
+        "chalk": "^2.1.0",
+        "concat-stream": "^1.6.0",
+        "cross-spawn": "^5.1.0",
+        "debug": "^3.1.0",
+        "doctrine": "^2.1.0",
+        "eslint-scope": "^3.7.1",
+        "eslint-visitor-keys": "^1.0.0",
+        "espree": "^3.5.4",
+        "esquery": "^1.0.0",
+        "esutils": "^2.0.2",
+        "file-entry-cache": "^2.0.0",
+        "functional-red-black-tree": "^1.0.1",
+        "glob": "^7.1.2",
+        "globals": "^11.0.1",
+        "ignore": "^3.3.3",
+        "imurmurhash": "^0.1.4",
+        "inquirer": "^3.0.6",
+        "is-resolvable": "^1.0.0",
+        "js-yaml": "^3.9.1",
+        "json-stable-stringify-without-jsonify": "^1.0.1",
+        "levn": "^0.3.0",
+        "lodash": "^4.17.4",
+        "minimatch": "^3.0.2",
+        "mkdirp": "^0.5.1",
+        "natural-compare": "^1.4.0",
+        "optionator": "^0.8.2",
+        "path-is-inside": "^1.0.2",
+        "pluralize": "^7.0.0",
+        "progress": "^2.0.0",
+        "regexpp": "^1.0.1",
+        "require-uncached": "^1.0.3",
+        "semver": "^5.3.0",
+        "strip-ansi": "^4.0.0",
+        "strip-json-comments": "~2.0.1",
+        "table": "4.0.2",
+        "text-table": "~0.2.0"
+      },
+      "dependencies": {
+        "ajv": {
+          "version": "5.5.2",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
+          "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
+          "dev": true,
+          "requires": {
+            "co": "^4.6.0",
+            "fast-deep-equal": "^1.0.0",
+            "fast-json-stable-stringify": "^2.0.0",
+            "json-schema-traverse": "^0.3.0"
+          }
+        },
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "chardet": {
+          "version": "0.4.2",
+          "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz",
+          "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=",
+          "dev": true
+        },
+        "cross-spawn": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
+          "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
+          "dev": true,
+          "requires": {
+            "lru-cache": "^4.0.1",
+            "shebang-command": "^1.2.0",
+            "which": "^1.2.9"
+          }
+        },
+        "debug": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "external-editor": {
+          "version": "2.2.0",
+          "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz",
+          "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==",
+          "dev": true,
+          "requires": {
+            "chardet": "^0.4.0",
+            "iconv-lite": "^0.4.17",
+            "tmp": "^0.0.33"
+          }
+        },
+        "fast-deep-equal": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
+          "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=",
+          "dev": true
+        },
+        "glob": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "dev": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        },
+        "globals": {
+          "version": "11.7.0",
+          "resolved": "https://registry.npmjs.org/globals/-/globals-11.7.0.tgz",
+          "integrity": "sha512-K8BNSPySfeShBQXsahYB/AbbWruVOTyVpgoIDnl8odPpeSfP2J5QO2oLFFdl2j7GfDCtZj2bMKar2T49itTPCg==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "inquirer": {
+          "version": "3.3.0",
+          "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz",
+          "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==",
+          "dev": true,
+          "requires": {
+            "ansi-escapes": "^3.0.0",
+            "chalk": "^2.0.0",
+            "cli-cursor": "^2.1.0",
+            "cli-width": "^2.0.0",
+            "external-editor": "^2.0.4",
+            "figures": "^2.0.0",
+            "lodash": "^4.3.0",
+            "mute-stream": "0.0.7",
+            "run-async": "^2.2.0",
+            "rx-lite": "^4.0.8",
+            "rx-lite-aggregates": "^4.0.8",
+            "string-width": "^2.1.0",
+            "strip-ansi": "^4.0.0",
+            "through": "^2.3.6"
+          }
+        },
+        "json-schema-traverse": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
+          "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=",
+          "dev": true
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "eslint-config-airbnb-base": {
+      "version": "13.0.0",
+      "resolved": "https://registry.npmjs.org/eslint-config-airbnb-base/-/eslint-config-airbnb-base-13.0.0.tgz",
+      "integrity": "sha512-hUFXRlE6AY84z0qYh4wKdtSF4EqDnyT8sxrvTpcXCV4ENSLF8li5yNA1yDM26iinH8Ierbpc4lv8Rp62uX6VSQ==",
+      "dev": true,
+      "requires": {
+        "eslint-restricted-globals": "^0.1.1",
+        "object.assign": "^4.1.0",
+        "object.entries": "^1.0.4"
+      }
+    },
+    "eslint-import-resolver-node": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.2.tgz",
+      "integrity": "sha512-sfmTqJfPSizWu4aymbPr4Iidp5yKm8yDkHp+Ir3YiTHiiDfxh69mOUsmiqW6RZ9zRXFaF64GtYmN7e+8GHBv6Q==",
+      "dev": true,
+      "requires": {
+        "debug": "^2.6.9",
+        "resolve": "^1.5.0"
+      }
+    },
+    "eslint-module-utils": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.2.0.tgz",
+      "integrity": "sha1-snA2LNiLGkitMIl2zn+lTphBF0Y=",
+      "dev": true,
+      "requires": {
+        "debug": "^2.6.8",
+        "pkg-dir": "^1.0.0"
+      },
+      "dependencies": {
+        "find-up": {
+          "version": "1.1.2",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
+          "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=",
+          "dev": true,
+          "requires": {
+            "path-exists": "^2.0.0",
+            "pinkie-promise": "^2.0.0"
+          }
+        },
+        "path-exists": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz",
+          "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=",
+          "dev": true,
+          "requires": {
+            "pinkie-promise": "^2.0.0"
+          }
+        },
+        "pkg-dir": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-1.0.0.tgz",
+          "integrity": "sha1-ektQio1bstYp1EcFb/TpyTFM89Q=",
+          "dev": true,
+          "requires": {
+            "find-up": "^1.0.0"
+          }
+        }
+      }
+    },
+    "eslint-plugin-es": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-1.3.1.tgz",
+      "integrity": "sha512-9XcVyZiQRVeFjqHw8qHNDAZcQLqaHlOGGpeYqzYh8S4JYCWTCO3yzyen8yVmA5PratfzTRWDwCOFphtDEG+w/w==",
+      "dev": true,
+      "requires": {
+        "eslint-utils": "^1.3.0",
+        "regexpp": "^2.0.0"
+      },
+      "dependencies": {
+        "regexpp": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.0.tgz",
+          "integrity": "sha512-g2FAVtR8Uh8GO1Nv5wpxW7VFVwHcCEr4wyA8/MHiRkO8uHoR5ntAA8Uq3P1vvMTX/BeQiRVSpDGLd+Wn5HNOTA==",
+          "dev": true
+        }
+      }
+    },
+    "eslint-plugin-html": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-4.0.5.tgz",
+      "integrity": "sha512-yULqYldzhYXTwZEaJXM30HhfgJdtTzuVH3LeoANybESHZ5+2ztLD72BsB2wR124/kk/PvQqZofDFSdNIk+kykw==",
+      "dev": true,
+      "requires": {
+        "htmlparser2": "^3.8.2"
+      }
+    },
+    "eslint-plugin-import": {
+      "version": "2.13.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.13.0.tgz",
+      "integrity": "sha512-t6hGKQDMIt9N8R7vLepsYXgDfeuhp6ZJSgtrLEDxonpSubyxUZHjhm6LsAaZX8q6GYVxkbT3kTsV9G5mBCFR6A==",
+      "dev": true,
+      "requires": {
+        "contains-path": "^0.1.0",
+        "debug": "^2.6.8",
+        "doctrine": "1.5.0",
+        "eslint-import-resolver-node": "^0.3.1",
+        "eslint-module-utils": "^2.2.0",
+        "has": "^1.0.1",
+        "lodash": "^4.17.4",
+        "minimatch": "^3.0.3",
+        "read-pkg-up": "^2.0.0",
+        "resolve": "^1.6.0"
+      },
+      "dependencies": {
+        "doctrine": {
+          "version": "1.5.0",
+          "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz",
+          "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=",
+          "dev": true,
+          "requires": {
+            "esutils": "^2.0.2",
+            "isarray": "^1.0.0"
+          }
+        }
+      }
+    },
+    "eslint-plugin-node": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-7.0.1.tgz",
+      "integrity": "sha512-lfVw3TEqThwq0j2Ba/Ckn2ABdwmL5dkOgAux1rvOk6CO7A6yGyPI2+zIxN6FyNkp1X1X/BSvKOceD6mBWSj4Yw==",
+      "dev": true,
+      "requires": {
+        "eslint-plugin-es": "^1.3.1",
+        "eslint-utils": "^1.3.1",
+        "ignore": "^4.0.2",
+        "minimatch": "^3.0.4",
+        "resolve": "^1.8.1",
+        "semver": "^5.5.0"
+      },
+      "dependencies": {
+        "ignore": {
+          "version": "4.0.2",
+          "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.2.tgz",
+          "integrity": "sha512-uoxnT7PYpyEnsja+yX+7v49B7LXxmzDJ2JALqHH3oEGzpM2U1IGcbfnOr8Dt57z3B/UWs7/iAgPFbmye8m4I0g==",
+          "dev": true
+        }
+      }
+    },
+    "eslint-plugin-promise": {
+      "version": "3.8.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-3.8.0.tgz",
+      "integrity": "sha512-JiFL9UFR15NKpHyGii1ZcvmtIqa3UTwiDAGb8atSffe43qJ3+1czVGN6UtkklpcJ2DVnqvTMzEKRaJdBkAL2aQ==",
+      "dev": true
+    },
+    "eslint-plugin-standard": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-3.1.0.tgz",
+      "integrity": "sha512-fVcdyuKRr0EZ4fjWl3c+gp1BANFJD1+RaWa2UPYfMZ6jCtp5RG00kSaXnK/dE5sYzt4kaWJ9qdxqUfc0d9kX0w==",
+      "dev": true
+    },
+    "eslint-restricted-globals": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/eslint-restricted-globals/-/eslint-restricted-globals-0.1.1.tgz",
+      "integrity": "sha1-NfDVy8ZMLj7WLpO0saevBbp+1Nc=",
+      "dev": true
+    },
+    "eslint-scope": {
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.1.tgz",
+      "integrity": "sha1-PWPD7f2gLgbgGkUq2IyqzHzctug=",
+      "dev": true,
+      "requires": {
+        "esrecurse": "^4.1.0",
+        "estraverse": "^4.1.1"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
+          "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=",
+          "dev": true
+        }
+      }
+    },
+    "eslint-utils": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.3.1.tgz",
+      "integrity": "sha512-Z7YjnIldX+2XMcjr7ZkgEsOj/bREONV60qYeB/bjMAqqqZ4zxKyWX+BOUkdmRmA9riiIPVvo5x86m5elviOk0Q==",
+      "dev": true
+    },
+    "eslint-visitor-keys": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz",
+      "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==",
+      "dev": true
+    },
+    "espree": {
+      "version": "3.5.4",
+      "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz",
+      "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==",
+      "dev": true,
+      "requires": {
+        "acorn": "^5.5.0",
+        "acorn-jsx": "^3.0.0"
+      }
+    },
+    "esprima": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz",
+      "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=",
+      "dev": true
+    },
+    "esquery": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz",
+      "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==",
+      "dev": true,
+      "requires": {
+        "estraverse": "^4.0.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
+          "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=",
+          "dev": true
+        }
+      }
+    },
+    "esrecurse": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
+      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+      "dev": true,
+      "requires": {
+        "estraverse": "^4.1.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
+          "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=",
+          "dev": true
+        }
+      }
+    },
+    "estraverse": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz",
+      "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=",
+      "dev": true
+    },
+    "esutils": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz",
+      "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=",
+      "dev": true
+    },
+    "events": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
+      "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
+      "dev": true
+    },
+    "evp_bytestokey": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz",
+      "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==",
+      "dev": true,
+      "requires": {
+        "md5.js": "^1.3.4",
+        "safe-buffer": "^5.1.1"
+      }
+    },
+    "execa": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
+      "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
+      "dev": true,
+      "requires": {
+        "cross-spawn": "^5.0.1",
+        "get-stream": "^3.0.0",
+        "is-stream": "^1.1.0",
+        "npm-run-path": "^2.0.0",
+        "p-finally": "^1.0.0",
+        "signal-exit": "^3.0.0",
+        "strip-eof": "^1.0.0"
+      },
+      "dependencies": {
+        "cross-spawn": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
+          "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
+          "dev": true,
+          "requires": {
+            "lru-cache": "^4.0.1",
+            "shebang-command": "^1.2.0",
+            "which": "^1.2.9"
+          }
+        }
+      }
+    },
+    "expand-brackets": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
+      "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
+      "dev": true,
+      "requires": {
+        "debug": "^2.3.3",
+        "define-property": "^0.2.5",
+        "extend-shallow": "^2.0.1",
+        "posix-character-classes": "^0.1.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "extend-shallow": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
+      "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
+      "dev": true,
+      "requires": {
+        "assign-symbols": "^1.0.0",
+        "is-extendable": "^1.0.1"
+      },
+      "dependencies": {
+        "is-extendable": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+          "dev": true,
+          "requires": {
+            "is-plain-object": "^2.0.4"
+          }
+        }
+      }
+    },
+    "external-editor": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.0.0.tgz",
+      "integrity": "sha512-mpkfj0FEdxrIhOC04zk85X7StNtr0yXnG7zCb+8ikO8OJi2jsHh5YGoknNTyXgsbHOf1WOOcVU3kPFWT2WgCkQ==",
+      "dev": true,
+      "requires": {
+        "chardet": "^0.5.0",
+        "iconv-lite": "^0.4.22",
+        "tmp": "^0.0.33"
+      }
+    },
+    "extglob": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
+      "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
+      "dev": true,
+      "requires": {
+        "array-unique": "^0.3.2",
+        "define-property": "^1.0.0",
+        "expand-brackets": "^2.1.4",
+        "extend-shallow": "^2.0.1",
+        "fragment-cache": "^0.2.1",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        },
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+          "dev": true
+        }
+      }
+    },
+    "fast-deep-equal": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
+      "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=",
+      "dev": true
+    },
+    "fast-json-stable-stringify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
+      "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=",
+      "dev": true
+    },
+    "fast-levenshtein": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+      "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
+      "dev": true
+    },
+    "fastparse": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/fastparse/-/fastparse-1.1.1.tgz",
+      "integrity": "sha1-0eJkOzipTXWDtHkGDmxK/8lAcfg=",
+      "dev": true
+    },
+    "figures": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
+      "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=",
+      "dev": true,
+      "requires": {
+        "escape-string-regexp": "^1.0.5"
+      }
+    },
+    "file-entry-cache": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz",
+      "integrity": "sha1-w5KZDD5oR4PYOLjISkXYoEhFg2E=",
+      "dev": true,
+      "requires": {
+        "flat-cache": "^1.2.1",
+        "object-assign": "^4.0.1"
+      }
+    },
+    "file-loader": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-1.1.11.tgz",
+      "integrity": "sha512-TGR4HU7HUsGg6GCOPJnFk06RhWgEWFLAGWiT6rcD+GRC2keU3s9RGJ+b3Z6/U73jwwNb2gKLJ7YCrp+jvU4ALg==",
+      "dev": true,
+      "requires": {
+        "loader-utils": "^1.0.2",
+        "schema-utils": "^0.4.5"
+      }
+    },
+    "fill-range": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
+      "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
+      "dev": true,
+      "requires": {
+        "extend-shallow": "^2.0.1",
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1",
+        "to-regex-range": "^2.1.0"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "find-cache-dir": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-1.0.0.tgz",
+      "integrity": "sha1-kojj6ePMN0hxfTnq3hfPcfww7m8=",
+      "dev": true,
+      "requires": {
+        "commondir": "^1.0.1",
+        "make-dir": "^1.0.0",
+        "pkg-dir": "^2.0.0"
+      }
+    },
+    "find-up": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
+      "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
+      "dev": true,
+      "requires": {
+        "locate-path": "^2.0.0"
+      }
+    },
+    "flat-cache": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.0.tgz",
+      "integrity": "sha1-0wMLMrOBVPTjt+nHCfSQ9++XxIE=",
+      "dev": true,
+      "requires": {
+        "circular-json": "^0.3.1",
+        "del": "^2.0.2",
+        "graceful-fs": "^4.1.2",
+        "write": "^0.2.1"
+      }
+    },
+    "flatten": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/flatten/-/flatten-1.0.2.tgz",
+      "integrity": "sha1-2uRqnXj74lKSJYzB54CkHZXAN4I=",
+      "dev": true
+    },
+    "flush-write-stream": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz",
+      "integrity": "sha512-calZMC10u0FMUqoiunI2AiGIIUtUIvifNwkHhNupZH4cbNnW1Itkoh/Nf5HFYmDrwWPjrUxpkZT0KhuCq0jmGw==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.4"
+      }
+    },
+    "for-in": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
+      "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=",
+      "dev": true
+    },
+    "foreach": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
+      "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=",
+      "dev": true
+    },
+    "fragment-cache": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
+      "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=",
+      "dev": true,
+      "requires": {
+        "map-cache": "^0.2.2"
+      }
+    },
+    "from2": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
+      "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.0"
+      }
+    },
+    "fs-extra": {
+      "version": "0.30.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.30.0.tgz",
+      "integrity": "sha1-8jP/zAjU2n1DLapEl3aYnbHfk/A=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "jsonfile": "^2.1.0",
+        "klaw": "^1.0.0",
+        "path-is-absolute": "^1.0.0",
+        "rimraf": "^2.2.8"
+      }
+    },
+    "fs-write-stream-atomic": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz",
+      "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "iferr": "^0.1.5",
+        "imurmurhash": "^0.1.4",
+        "readable-stream": "1 || 2"
+      }
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "fsevents": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.4.tgz",
+      "integrity": "sha512-z8H8/diyk76B7q5wg+Ud0+CqzcAF3mBBI/bA5ne5zrRUUIvNkJY//D3BqyH571KuAC4Nr7Rw7CjWX4r0y9DvNg==",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "nan": "^2.9.2",
+        "node-pre-gyp": "^0.10.0"
+      },
+      "dependencies": {
+        "abbrev": {
+          "version": "1.1.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "ansi-regex": {
+          "version": "2.1.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "aproba": {
+          "version": "1.2.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "are-we-there-yet": {
+          "version": "1.1.4",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "delegates": "^1.0.0",
+            "readable-stream": "^2.0.6"
+          }
+        },
+        "balanced-match": {
+          "version": "1.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "brace-expansion": {
+          "version": "1.1.11",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "balanced-match": "^1.0.0",
+            "concat-map": "0.0.1"
+          }
+        },
+        "chownr": {
+          "version": "1.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "code-point-at": {
+          "version": "1.1.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "concat-map": {
+          "version": "0.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "console-control-strings": {
+          "version": "1.1.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "core-util-is": {
+          "version": "1.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "debug": {
+          "version": "2.6.9",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "deep-extend": {
+          "version": "0.5.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "delegates": {
+          "version": "1.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "detect-libc": {
+          "version": "1.0.3",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "fs-minipass": {
+          "version": "1.2.5",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "minipass": "^2.2.1"
+          }
+        },
+        "fs.realpath": {
+          "version": "1.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "gauge": {
+          "version": "2.7.4",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "aproba": "^1.0.3",
+            "console-control-strings": "^1.0.0",
+            "has-unicode": "^2.0.0",
+            "object-assign": "^4.1.0",
+            "signal-exit": "^3.0.0",
+            "string-width": "^1.0.1",
+            "strip-ansi": "^3.0.1",
+            "wide-align": "^1.1.0"
+          }
+        },
+        "glob": {
+          "version": "7.1.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        },
+        "has-unicode": {
+          "version": "2.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "iconv-lite": {
+          "version": "0.4.21",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "safer-buffer": "^2.1.0"
+          }
+        },
+        "ignore-walk": {
+          "version": "3.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "minimatch": "^3.0.4"
+          }
+        },
+        "inflight": {
+          "version": "1.0.6",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "once": "^1.3.0",
+            "wrappy": "1"
+          }
+        },
+        "inherits": {
+          "version": "2.0.3",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "ini": {
+          "version": "1.3.5",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "1.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "number-is-nan": "^1.0.0"
+          }
+        },
+        "isarray": {
+          "version": "1.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "minimatch": {
+          "version": "3.0.4",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "brace-expansion": "^1.1.7"
+          }
+        },
+        "minimist": {
+          "version": "0.0.8",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "minipass": {
+          "version": "2.2.4",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "safe-buffer": "^5.1.1",
+            "yallist": "^3.0.0"
+          }
+        },
+        "minizlib": {
+          "version": "1.1.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "minipass": "^2.2.1"
+          }
+        },
+        "mkdirp": {
+          "version": "0.5.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "minimist": "0.0.8"
+          }
+        },
+        "ms": {
+          "version": "2.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "needle": {
+          "version": "2.2.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "debug": "^2.1.2",
+            "iconv-lite": "^0.4.4",
+            "sax": "^1.2.4"
+          }
+        },
+        "node-pre-gyp": {
+          "version": "0.10.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "detect-libc": "^1.0.2",
+            "mkdirp": "^0.5.1",
+            "needle": "^2.2.0",
+            "nopt": "^4.0.1",
+            "npm-packlist": "^1.1.6",
+            "npmlog": "^4.0.2",
+            "rc": "^1.1.7",
+            "rimraf": "^2.6.1",
+            "semver": "^5.3.0",
+            "tar": "^4"
+          }
+        },
+        "nopt": {
+          "version": "4.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "abbrev": "1",
+            "osenv": "^0.1.4"
+          }
+        },
+        "npm-bundled": {
+          "version": "1.0.3",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "npm-packlist": {
+          "version": "1.1.10",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "ignore-walk": "^3.0.1",
+            "npm-bundled": "^1.0.1"
+          }
+        },
+        "npmlog": {
+          "version": "4.1.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "are-we-there-yet": "~1.1.2",
+            "console-control-strings": "~1.1.0",
+            "gauge": "~2.7.3",
+            "set-blocking": "~2.0.0"
+          }
+        },
+        "number-is-nan": {
+          "version": "1.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "object-assign": {
+          "version": "4.1.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "once": {
+          "version": "1.4.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "wrappy": "1"
+          }
+        },
+        "os-homedir": {
+          "version": "1.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "os-tmpdir": {
+          "version": "1.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "osenv": {
+          "version": "0.1.5",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "os-homedir": "^1.0.0",
+            "os-tmpdir": "^1.0.0"
+          }
+        },
+        "path-is-absolute": {
+          "version": "1.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "process-nextick-args": {
+          "version": "2.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "rc": {
+          "version": "1.2.7",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "deep-extend": "^0.5.1",
+            "ini": "~1.3.0",
+            "minimist": "^1.2.0",
+            "strip-json-comments": "~2.0.1"
+          },
+          "dependencies": {
+            "minimist": {
+              "version": "1.2.0",
+              "bundled": true,
+              "dev": true,
+              "optional": true
+            }
+          }
+        },
+        "readable-stream": {
+          "version": "2.3.6",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "rimraf": {
+          "version": "2.6.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "glob": "^7.0.5"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "safer-buffer": {
+          "version": "2.1.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "sax": {
+          "version": "1.2.4",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "semver": {
+          "version": "5.5.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "set-blocking": {
+          "version": "2.0.0",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "signal-exit": {
+          "version": "3.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "string-width": {
+          "version": "1.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "code-point-at": "^1.0.0",
+            "is-fullwidth-code-point": "^1.0.0",
+            "strip-ansi": "^3.0.0"
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "3.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "ansi-regex": "^2.0.0"
+          }
+        },
+        "strip-json-comments": {
+          "version": "2.0.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "tar": {
+          "version": "4.4.1",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "chownr": "^1.0.1",
+            "fs-minipass": "^1.2.5",
+            "minipass": "^2.2.4",
+            "minizlib": "^1.1.0",
+            "mkdirp": "^0.5.0",
+            "safe-buffer": "^5.1.1",
+            "yallist": "^3.0.2"
+          }
+        },
+        "util-deprecate": {
+          "version": "1.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "wide-align": {
+          "version": "1.1.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true,
+          "requires": {
+            "string-width": "^1.0.2"
+          }
+        },
+        "wrappy": {
+          "version": "1.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        },
+        "yallist": {
+          "version": "3.0.2",
+          "bundled": true,
+          "dev": true,
+          "optional": true
+        }
+      }
+    },
+    "function-bind": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
+      "dev": true
+    },
+    "functional-red-black-tree": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
+      "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
+      "dev": true
+    },
+    "generic-names": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/generic-names/-/generic-names-1.0.3.tgz",
+      "integrity": "sha1-LXhqEhruUIh2eWk56OO/+DbCCRc=",
+      "dev": true,
+      "requires": {
+        "loader-utils": "^0.2.16"
+      },
+      "dependencies": {
+        "loader-utils": {
+          "version": "0.2.17",
+          "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.17.tgz",
+          "integrity": "sha1-+G5jdNQyBabmxg6RlvF8Apm/s0g=",
+          "dev": true,
+          "requires": {
+            "big.js": "^3.1.3",
+            "emojis-list": "^2.0.0",
+            "json5": "^0.5.0",
+            "object-assign": "^4.0.1"
+          }
+        }
+      }
+    },
+    "get-caller-file": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
+      "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
+      "dev": true
+    },
+    "get-stream": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+      "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
+      "dev": true
+    },
+    "get-value": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
+      "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=",
+      "dev": true
+    },
+    "glob": {
+      "version": "5.0.15",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz",
+      "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=",
+      "dev": true,
+      "requires": {
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "2 || 3",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "glob-parent": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz",
+      "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=",
+      "dev": true,
+      "requires": {
+        "is-glob": "^3.1.0",
+        "path-dirname": "^1.0.0"
+      },
+      "dependencies": {
+        "is-glob": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
+          "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=",
+          "dev": true,
+          "requires": {
+            "is-extglob": "^2.1.0"
+          }
+        }
+      }
+    },
+    "glob-to-regexp": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz",
+      "integrity": "sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=",
+      "dev": true
+    },
+    "global-modules-path": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/global-modules-path/-/global-modules-path-2.3.0.tgz",
+      "integrity": "sha512-HchvMJNYh9dGSCy8pOQ2O8u/hoXaL+0XhnrwH0RyLiSXMMTl9W3N6KUU73+JFOg5PGjtzl6VZzUQsnrpm7Szag==",
+      "dev": true
+    },
+    "globals": {
+      "version": "9.18.0",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-9.18.0.tgz",
+      "integrity": "sha512-S0nG3CLEQiY/ILxqtztTWH/3iRRdyBLw6KMDxnKMchrtbj2OFmehVh0WUCfW3DUrIgx/qFrJPICrq4Z4sTR9UQ==",
+      "dev": true
+    },
+    "globby": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-7.1.1.tgz",
+      "integrity": "sha1-+yzP+UAfhgCUXfral0QMypcrhoA=",
+      "dev": true,
+      "requires": {
+        "array-union": "^1.0.1",
+        "dir-glob": "^2.0.0",
+        "glob": "^7.1.2",
+        "ignore": "^3.3.5",
+        "pify": "^3.0.0",
+        "slash": "^1.0.0"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "dev": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        }
+      }
+    },
+    "graceful-fs": {
+      "version": "4.1.11",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz",
+      "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=",
+      "dev": true
+    },
+    "graphlib": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/graphlib/-/graphlib-1.0.7.tgz",
+      "integrity": "sha1-DKst8P/mq+BwsmJb+h7bbslnuLE=",
+      "requires": {
+        "lodash": "^3.10.0"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y="
+        }
+      }
+    },
+    "handlebars": {
+      "version": "4.0.11",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.11.tgz",
+      "integrity": "sha1-Ywo13+ApS8KB7a5v/F0yn8eYLcw=",
+      "dev": true,
+      "requires": {
+        "async": "^1.4.0",
+        "optimist": "^0.6.1",
+        "source-map": "^0.4.4",
+        "uglify-js": "^2.6"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.4.4",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
+          "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=",
+          "dev": true,
+          "requires": {
+            "amdefine": ">=0.0.4"
+          }
+        }
+      }
+    },
+    "has": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
+      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+      "dev": true,
+      "requires": {
+        "function-bind": "^1.1.1"
+      }
+    },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dev": true,
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
+    "has-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz",
+      "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=",
+      "dev": true
+    },
+    "has-symbols": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz",
+      "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=",
+      "dev": true
+    },
+    "has-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
+      "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=",
+      "dev": true,
+      "requires": {
+        "get-value": "^2.0.6",
+        "has-values": "^1.0.0",
+        "isobject": "^3.0.0"
+      }
+    },
+    "has-values": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
+      "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=",
+      "dev": true,
+      "requires": {
+        "is-number": "^3.0.0",
+        "kind-of": "^4.0.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
+          "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=",
+          "dev": true,
+          "requires": {
+            "is-buffer": "^1.1.5"
+          }
+        }
+      }
+    },
+    "hash-base": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz",
+      "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "hash.js": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.5.tgz",
+      "integrity": "sha512-eWI5HG9Np+eHV1KQhisXWwM+4EPPYe5dFX1UZZH7k/E3JzDEazVH+VGlZi6R94ZqImq+A3D1mCEtrFIfg/E7sA==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.3",
+        "minimalistic-assert": "^1.0.1"
+      }
+    },
+    "hmac-drbg": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
+      "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=",
+      "dev": true,
+      "requires": {
+        "hash.js": "^1.0.3",
+        "minimalistic-assert": "^1.0.0",
+        "minimalistic-crypto-utils": "^1.0.1"
+      }
+    },
+    "home-or-tmp": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-2.0.0.tgz",
+      "integrity": "sha1-42w/LSyufXRqhX440Y1fMqeILbg=",
+      "dev": true,
+      "requires": {
+        "os-homedir": "^1.0.0",
+        "os-tmpdir": "^1.0.1"
+      }
+    },
+    "hosted-git-info": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz",
+      "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==",
+      "dev": true
+    },
+    "html-comment-regex": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.1.tgz",
+      "integrity": "sha1-ZouTd26q5V696POtRkswekljYl4=",
+      "dev": true
+    },
+    "htmlparser2": {
+      "version": "3.9.2",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz",
+      "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=",
+      "dev": true,
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.2"
+      }
+    },
+    "https-browserify": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
+      "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
+      "dev": true
+    },
+    "iconv-lite": {
+      "version": "0.4.23",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
+      "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==",
+      "dev": true,
+      "requires": {
+        "safer-buffer": ">= 2.1.2 < 3"
+      }
+    },
+    "icss-replace-symbols": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz",
+      "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0=",
+      "dev": true
+    },
+    "icss-utils": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-3.0.1.tgz",
+      "integrity": "sha1-7nDTroysOMa+XtkehRsn7tNDrQ8=",
+      "dev": true,
+      "requires": {
+        "postcss": "^6.0.2"
+      }
+    },
+    "ieee754": {
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.12.tgz",
+      "integrity": "sha512-GguP+DRY+pJ3soyIiGPTvdiVXjZ+DbXOxGpXn3eMvNW4x4irjqXm4wHKscC+TfxSJ0yw/S1F24tqdMNsMZTiLA==",
+      "dev": true
+    },
+    "iferr": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz",
+      "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
+      "dev": true
+    },
+    "ignore": {
+      "version": "3.3.10",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz",
+      "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==",
+      "dev": true
+    },
+    "import-local": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/import-local/-/import-local-1.0.0.tgz",
+      "integrity": "sha512-vAaZHieK9qjGo58agRBg+bhHX3hoTZU/Oa3GESWLz7t1U62fk63aHuDJJEteXoDeTCcPmUT+z38gkHPZkkmpmQ==",
+      "dev": true,
+      "requires": {
+        "pkg-dir": "^2.0.0",
+        "resolve-cwd": "^2.0.0"
+      }
+    },
+    "imports-loader": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/imports-loader/-/imports-loader-0.8.0.tgz",
+      "integrity": "sha512-kXWL7Scp8KQ4552ZcdVTeaQCZSLW+e6nJfp3cwUMB673T7Hr98Xjx5JK+ql7ADlJUvj1JS5O01RLbKoutN5QDQ==",
+      "dev": true,
+      "requires": {
+        "loader-utils": "^1.0.2",
+        "source-map": "^0.6.1"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
+      }
+    },
+    "imurmurhash": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+      "dev": true
+    },
+    "indexes-of": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/indexes-of/-/indexes-of-1.0.1.tgz",
+      "integrity": "sha1-8w9xbI4r00bHtn0985FVZqfAVgc=",
+      "dev": true
+    },
+    "indexof": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz",
+      "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=",
+      "dev": true
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
+      "dev": true
+    },
+    "inquirer": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.0.0.tgz",
+      "integrity": "sha512-tISQWRwtcAgrz+SHPhTH7d3e73k31gsOy6i1csonLc0u1dVK/wYvuOnFeiWqC5OXFIYbmrIFInef31wbT8MEJg==",
+      "dev": true,
+      "requires": {
+        "ansi-escapes": "^3.0.0",
+        "chalk": "^2.0.0",
+        "cli-cursor": "^2.1.0",
+        "cli-width": "^2.0.0",
+        "external-editor": "^3.0.0",
+        "figures": "^2.0.0",
+        "lodash": "^4.3.0",
+        "mute-stream": "0.0.7",
+        "run-async": "^2.2.0",
+        "rxjs": "^6.1.0",
+        "string-width": "^2.1.0",
+        "strip-ansi": "^4.0.0",
+        "through": "^2.3.6"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "interpret": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.1.0.tgz",
+      "integrity": "sha1-ftGxQQxqDg94z5XTuEQMY/eLhhQ=",
+      "dev": true
+    },
+    "invariant": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
+      "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==",
+      "dev": true,
+      "requires": {
+        "loose-envify": "^1.0.0"
+      }
+    },
+    "invert-kv": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
+      "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=",
+      "dev": true
+    },
+    "is-absolute-url": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-2.1.0.tgz",
+      "integrity": "sha1-UFMN+4T8yap9vnhS6Do3uTufKqY=",
+      "dev": true
+    },
+    "is-accessor-descriptor": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+      "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+      "dev": true,
+      "requires": {
+        "kind-of": "^3.0.2"
+      }
+    },
+    "is-arrayish": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
+      "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=",
+      "dev": true
+    },
+    "is-binary-path": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz",
+      "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=",
+      "dev": true,
+      "requires": {
+        "binary-extensions": "^1.0.0"
+      }
+    },
+    "is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+      "dev": true
+    },
+    "is-builtin-module": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
+      "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=",
+      "dev": true,
+      "requires": {
+        "builtin-modules": "^1.0.0"
+      }
+    },
+    "is-callable": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz",
+      "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==",
+      "dev": true
+    },
+    "is-data-descriptor": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+      "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+      "dev": true,
+      "requires": {
+        "kind-of": "^3.0.2"
+      }
+    },
+    "is-date-object": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
+      "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=",
+      "dev": true
+    },
+    "is-descriptor": {
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
+      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
+      "dev": true,
+      "requires": {
+        "is-accessor-descriptor": "^0.1.6",
+        "is-data-descriptor": "^0.1.4",
+        "kind-of": "^5.0.0"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
+          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
+          "dev": true
+        }
+      }
+    },
+    "is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=",
+      "dev": true
+    },
+    "is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
+      "dev": true
+    },
+    "is-finite": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz",
+      "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=",
+      "dev": true,
+      "requires": {
+        "number-is-nan": "^1.0.0"
+      }
+    },
+    "is-fullwidth-code-point": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+      "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+      "dev": true
+    },
+    "is-glob": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
+      "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=",
+      "dev": true,
+      "requires": {
+        "is-extglob": "^2.1.1"
+      }
+    },
+    "is-number": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
+      "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
+      "dev": true,
+      "requires": {
+        "kind-of": "^3.0.2"
+      }
+    },
+    "is-path-cwd": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz",
+      "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=",
+      "dev": true
+    },
+    "is-path-in-cwd": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz",
+      "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==",
+      "dev": true,
+      "requires": {
+        "is-path-inside": "^1.0.0"
+      }
+    },
+    "is-path-inside": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz",
+      "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=",
+      "dev": true,
+      "requires": {
+        "path-is-inside": "^1.0.1"
+      }
+    },
+    "is-plain-obj": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
+      "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
+      "dev": true
+    },
+    "is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "dev": true,
+      "requires": {
+        "isobject": "^3.0.1"
+      }
+    },
+    "is-promise": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
+      "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=",
+      "dev": true
+    },
+    "is-regex": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
+      "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
+      "dev": true,
+      "requires": {
+        "has": "^1.0.1"
+      }
+    },
+    "is-resolvable": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.1.0.tgz",
+      "integrity": "sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==",
+      "dev": true
+    },
+    "is-stream": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
+      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
+      "dev": true
+    },
+    "is-svg": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-svg/-/is-svg-2.1.0.tgz",
+      "integrity": "sha1-z2EJDaDZ77yrhyLeum8DIgjbsOk=",
+      "dev": true,
+      "requires": {
+        "html-comment-regex": "^1.1.0"
+      }
+    },
+    "is-symbol": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
+      "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=",
+      "dev": true
+    },
+    "is-windows": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
+      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
+      "dev": true
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+      "dev": true
+    },
+    "isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
+      "dev": true
+    },
+    "isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
+      "dev": true
+    },
+    "jquery": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.3.1.tgz",
+      "integrity": "sha512-Ubldcmxp5np52/ENotGxlLe6aGMvmF4R8S6tZjsP6Knsaxd/xp3Zrh50cG93lR6nPXyUFwzN3ZSOQI0wRJNdGg=="
+    },
+    "js-base64": {
+      "version": "2.4.5",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.4.5.tgz",
+      "integrity": "sha512-aUnNwqMOXw3yvErjMPSQu6qIIzUmT1e5KcU1OZxRDU1g/am6mzBvcrmLAYwzmB59BHPrh5/tKaiF4OPhqRWESQ==",
+      "dev": true
+    },
+    "js-tokens": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
+      "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=",
+      "dev": true
+    },
+    "js-yaml": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz",
+      "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==",
+      "dev": true,
+      "requires": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      },
+      "dependencies": {
+        "esprima": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
+          "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==",
+          "dev": true
+        }
+      }
+    },
+    "jsesc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz",
+      "integrity": "sha1-RsP+yMGJKxKwgz25vHYiF226s0s=",
+      "dev": true
+    },
+    "json-parse-better-errors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz",
+      "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==",
+      "dev": true
+    },
+    "json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
+    },
+    "json-stable-stringify-without-jsonify": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
+      "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
+      "dev": true
+    },
+    "json5": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz",
+      "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=",
+      "dev": true
+    },
+    "jsonfile": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
+      "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
+    "kind-of": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+      "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+      "dev": true,
+      "requires": {
+        "is-buffer": "^1.1.5"
+      }
+    },
+    "klaw": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz",
+      "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.9"
+      }
+    },
+    "lazy-cache": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
+      "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=",
+      "dev": true,
+      "optional": true
+    },
+    "lcid": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
+      "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=",
+      "dev": true,
+      "requires": {
+        "invert-kv": "^1.0.0"
+      }
+    },
+    "levn": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
+      "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
+      "dev": true,
+      "requires": {
+        "prelude-ls": "~1.1.2",
+        "type-check": "~0.3.2"
+      }
+    },
+    "load-json-file": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
+      "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "parse-json": "^2.2.0",
+        "pify": "^2.0.0",
+        "strip-bom": "^3.0.0"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "dev": true
+        }
+      }
+    },
+    "loader-runner": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.0.tgz",
+      "integrity": "sha1-9IKuqC1UPgeSFwDVpG7yb9rGuKI=",
+      "dev": true
+    },
+    "loader-utils": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.1.0.tgz",
+      "integrity": "sha1-yYrvSIvM7aL/teLeZG1qdUQp9c0=",
+      "dev": true,
+      "requires": {
+        "big.js": "^3.1.3",
+        "emojis-list": "^2.0.0",
+        "json5": "^0.5.0"
+      }
+    },
+    "locate-path": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
+      "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
+      "dev": true,
+      "requires": {
+        "p-locate": "^2.0.0",
+        "path-exists": "^3.0.0"
+      }
+    },
+    "lodash": {
+      "version": "4.17.11",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
+      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+    },
+    "lodash.camelcase": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
+      "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=",
+      "dev": true
+    },
+    "lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=",
+      "dev": true
+    },
+    "lodash.memoize": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
+      "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=",
+      "dev": true
+    },
+    "lodash.uniq": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
+      "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=",
+      "dev": true
+    },
+    "log-symbols": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
+      "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==",
+      "dev": true,
+      "requires": {
+        "chalk": "^2.0.1"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "loglevelnext": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/loglevelnext/-/loglevelnext-1.0.5.tgz",
+      "integrity": "sha512-V/73qkPuJmx4BcBF19xPBr+0ZRVBhc4POxvZTZdMeXpJ4NItXSJ/MSwuFT0kQJlCbXvdlZoQQ/418bS1y9Jh6A==",
+      "dev": true,
+      "requires": {
+        "es6-symbol": "^3.1.1",
+        "object.assign": "^4.1.0"
+      }
+    },
+    "long": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz",
+      "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s=",
+      "dev": true
+    },
+    "longest": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz",
+      "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=",
+      "dev": true,
+      "optional": true
+    },
+    "loose-envify": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.3.1.tgz",
+      "integrity": "sha1-0aitM/qc4OcT1l/dCsi3SNR4yEg=",
+      "dev": true,
+      "requires": {
+        "js-tokens": "^3.0.0"
+      }
+    },
+    "lru-cache": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz",
+      "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==",
+      "dev": true,
+      "requires": {
+        "pseudomap": "^1.0.2",
+        "yallist": "^2.1.2"
+      }
+    },
+    "make-dir": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
+      "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==",
+      "dev": true,
+      "requires": {
+        "pify": "^3.0.0"
+      }
+    },
+    "mamacro": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/mamacro/-/mamacro-0.0.3.tgz",
+      "integrity": "sha512-qMEwh+UujcQ+kbz3T6V+wAmO2U8veoq2w+3wY8MquqwVA3jChfwY+Tk52GZKDfACEPjuZ7r2oJLejwpt8jtwTA==",
+      "dev": true
+    },
+    "map-cache": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz",
+      "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=",
+      "dev": true
+    },
+    "map-visit": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz",
+      "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=",
+      "dev": true,
+      "requires": {
+        "object-visit": "^1.0.0"
+      }
+    },
+    "math-expression-evaluator": {
+      "version": "1.2.17",
+      "resolved": "https://registry.npmjs.org/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz",
+      "integrity": "sha1-3oGf282E3M2PrlnGrreWFbnSZqw=",
+      "dev": true
+    },
+    "md5.js": {
+      "version": "1.3.4",
+      "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.4.tgz",
+      "integrity": "sha1-6b296UogpawYsENA/Fdk1bCdkB0=",
+      "dev": true,
+      "requires": {
+        "hash-base": "^3.0.0",
+        "inherits": "^2.0.1"
+      }
+    },
+    "mem": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz",
+      "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=",
+      "dev": true,
+      "requires": {
+        "mimic-fn": "^1.0.0"
+      }
+    },
+    "memory-fs": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz",
+      "integrity": "sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=",
+      "dev": true,
+      "requires": {
+        "errno": "^0.1.3",
+        "readable-stream": "^2.0.1"
+      }
+    },
+    "micromatch": {
+      "version": "3.1.10",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
+      "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
+      "dev": true,
+      "requires": {
+        "arr-diff": "^4.0.0",
+        "array-unique": "^0.3.2",
+        "braces": "^2.3.1",
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "extglob": "^2.0.4",
+        "fragment-cache": "^0.2.1",
+        "kind-of": "^6.0.2",
+        "nanomatch": "^1.2.9",
+        "object.pick": "^1.3.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.2"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+          "dev": true
+        }
+      }
+    },
+    "miller-rabin": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz",
+      "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.0.0",
+        "brorand": "^1.0.1"
+      }
+    },
+    "mime": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz",
+      "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==",
+      "dev": true
+    },
+    "mimic-fn": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
+      "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
+      "dev": true
+    },
+    "mini-css-extract-plugin": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-0.4.1.tgz",
+      "integrity": "sha512-XWuB3G61Rtasq/gLe7cp5cuozehE6hN+E4sxCamRR/WDiHTg+f7ZIAS024r8UJQffY+e2gGELXQZgQoFDfNDCg==",
+      "dev": true,
+      "requires": {
+        "@webpack-contrib/schema-utils": "^1.0.0-beta.0",
+        "loader-utils": "^1.1.0",
+        "webpack-sources": "^1.1.0"
+      }
+    },
+    "minimalistic-assert": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz",
+      "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==",
+      "dev": true
+    },
+    "minimalistic-crypto-utils": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz",
+      "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=",
+      "dev": true
+    },
+    "minimatch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "dev": true,
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "minimist": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
+      "dev": true
+    },
+    "mississippi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-2.0.0.tgz",
+      "integrity": "sha512-zHo8v+otD1J10j/tC+VNoGK9keCuByhKovAvdn74dmxJl9+mWHnx6EMsDN4lgRoMI/eYo2nchAxniIbUPb5onw==",
+      "dev": true,
+      "requires": {
+        "concat-stream": "^1.5.0",
+        "duplexify": "^3.4.2",
+        "end-of-stream": "^1.1.0",
+        "flush-write-stream": "^1.0.0",
+        "from2": "^2.1.0",
+        "parallel-transform": "^1.1.0",
+        "pump": "^2.0.1",
+        "pumpify": "^1.3.3",
+        "stream-each": "^1.1.0",
+        "through2": "^2.0.0"
+      }
+    },
+    "mixin-deep": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
+      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "dev": true,
+      "requires": {
+        "for-in": "^1.0.2",
+        "is-extendable": "^1.0.1"
+      },
+      "dependencies": {
+        "is-extendable": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
+          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
+          "dev": true,
+          "requires": {
+            "is-plain-object": "^2.0.4"
+          }
+        }
+      }
+    },
+    "mkdirp": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "dev": true,
+      "requires": {
+        "minimist": "0.0.8"
+      }
+    },
+    "moment": {
+      "version": "2.22.2",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz",
+      "integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y="
+    },
+    "moment-timezone": {
+      "version": "0.5.21",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.21.tgz",
+      "integrity": "sha512-j96bAh4otsgj3lKydm3K7kdtA3iKf2m6MY2iSYCzCm5a1zmHo1g+aK3068dDEeocLZQIS9kU8bsdQHLqEvgW0A==",
+      "requires": {
+        "moment": ">= 2.9.0"
+      }
+    },
+    "move-concurrently": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz",
+      "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=",
+      "dev": true,
+      "requires": {
+        "aproba": "^1.1.1",
+        "copy-concurrently": "^1.0.0",
+        "fs-write-stream-atomic": "^1.0.8",
+        "mkdirp": "^0.5.1",
+        "rimraf": "^2.5.4",
+        "run-queue": "^1.0.3"
+      }
+    },
+    "ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+      "dev": true
+    },
+    "multi-glob": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/multi-glob/-/multi-glob-1.0.1.tgz",
+      "integrity": "sha1-5n0qtEKdJ2BubrTbNQlK/JF4h1A=",
+      "dev": true,
+      "requires": {
+        "async": "1.x",
+        "glob": "5.x",
+        "lodash": "3.x"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=",
+          "dev": true
+        }
+      }
+    },
+    "mute-stream": {
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
+      "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=",
+      "dev": true
+    },
+    "nan": {
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
+      "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==",
+      "dev": true,
+      "optional": true
+    },
+    "nanomatch": {
+      "version": "1.2.13",
+      "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
+      "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==",
+      "dev": true,
+      "requires": {
+        "arr-diff": "^4.0.0",
+        "array-unique": "^0.3.2",
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "fragment-cache": "^0.2.1",
+        "is-windows": "^1.0.2",
+        "kind-of": "^6.0.2",
+        "object.pick": "^1.3.0",
+        "regex-not": "^1.0.0",
+        "snapdragon": "^0.8.1",
+        "to-regex": "^3.0.1"
+      },
+      "dependencies": {
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+          "dev": true
+        }
+      }
+    },
+    "natural-compare": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
+      "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
+      "dev": true
+    },
+    "neo-async": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.5.1.tgz",
+      "integrity": "sha512-3KL3fvuRkZ7s4IFOMfztb7zJp3QaVWnBeGoJlgB38XnCRPj/0tLzzLG5IB8NYOHbJ8g8UGrgZv44GLDk6CxTxA==",
+      "dev": true
+    },
+    "next-tick": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
+      "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=",
+      "dev": true
+    },
+    "nice-try": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.4.tgz",
+      "integrity": "sha512-2NpiFHqC87y/zFke0fC0spBXL3bBsoh/p5H1EFhshxjCR5+0g2d6BiXbUFz9v1sAcxsk2htp2eQnNIci2dIYcA==",
+      "dev": true
+    },
+    "node-libs-browser": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.1.0.tgz",
+      "integrity": "sha512-5AzFzdoIMb89hBGMZglEegffzgRg+ZFoUmisQ8HI4j1KDdpx13J0taNp2y9xPbur6W61gepGDDotGBVQ7mfUCg==",
+      "dev": true,
+      "requires": {
+        "assert": "^1.1.1",
+        "browserify-zlib": "^0.2.0",
+        "buffer": "^4.3.0",
+        "console-browserify": "^1.1.0",
+        "constants-browserify": "^1.0.0",
+        "crypto-browserify": "^3.11.0",
+        "domain-browser": "^1.1.1",
+        "events": "^1.0.0",
+        "https-browserify": "^1.0.0",
+        "os-browserify": "^0.3.0",
+        "path-browserify": "0.0.0",
+        "process": "^0.11.10",
+        "punycode": "^1.2.4",
+        "querystring-es3": "^0.2.0",
+        "readable-stream": "^2.3.3",
+        "stream-browserify": "^2.0.1",
+        "stream-http": "^2.7.2",
+        "string_decoder": "^1.0.0",
+        "timers-browserify": "^2.0.4",
+        "tty-browserify": "0.0.0",
+        "url": "^0.11.0",
+        "util": "^0.10.3",
+        "vm-browserify": "0.0.4"
+      },
+      "dependencies": {
+        "punycode": {
+          "version": "1.4.1",
+          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+          "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
+          "dev": true
+        }
+      }
+    },
+    "nopt": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
+      "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=",
+      "dev": true,
+      "requires": {
+        "abbrev": "1"
+      }
+    },
+    "normalize-package-data": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
+      "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==",
+      "dev": true,
+      "requires": {
+        "hosted-git-info": "^2.1.4",
+        "is-builtin-module": "^1.0.0",
+        "semver": "2 || 3 || 4 || 5",
+        "validate-npm-package-license": "^3.0.1"
+      }
+    },
+    "normalize-path": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
+      "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
+      "dev": true,
+      "requires": {
+        "remove-trailing-separator": "^1.0.1"
+      }
+    },
+    "normalize-range": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz",
+      "integrity": "sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=",
+      "dev": true
+    },
+    "normalize-url": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.9.1.tgz",
+      "integrity": "sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=",
+      "dev": true,
+      "requires": {
+        "object-assign": "^4.0.1",
+        "prepend-http": "^1.0.0",
+        "query-string": "^4.1.0",
+        "sort-keys": "^1.0.0"
+      }
+    },
+    "npm": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/npm/-/npm-6.3.0.tgz",
+      "integrity": "sha512-oDtLFo3wXue/xe3pU/oks9VHS5501OAWlYrZrApZkFv7l2LXk+9CfPMbjbfZWK7Jqlc1jbNcJMkB6KZC7K/vEA==",
+      "requires": {
+        "JSONStream": "^1.3.3",
+        "abbrev": "~1.1.1",
+        "ansicolors": "~0.3.2",
+        "ansistyles": "~0.1.3",
+        "aproba": "~1.2.0",
+        "archy": "~1.0.0",
+        "bin-links": "^1.1.2",
+        "bluebird": "~3.5.1",
+        "byte-size": "^4.0.3",
+        "cacache": "^11.1.0",
+        "call-limit": "~1.1.0",
+        "chownr": "~1.0.1",
+        "cli-columns": "^3.1.2",
+        "cli-table3": "^0.5.0",
+        "cmd-shim": "~2.0.2",
+        "columnify": "~1.5.4",
+        "config-chain": "~1.1.11",
+        "debuglog": "*",
+        "detect-indent": "~5.0.0",
+        "detect-newline": "^2.1.0",
+        "dezalgo": "~1.0.3",
+        "editor": "~1.0.0",
+        "figgy-pudding": "^3.2.0",
+        "find-npm-prefix": "^1.0.2",
+        "fs-vacuum": "~1.2.10",
+        "fs-write-stream-atomic": "~1.0.10",
+        "gentle-fs": "^2.0.1",
+        "glob": "~7.1.2",
+        "graceful-fs": "~4.1.11",
+        "has-unicode": "~2.0.1",
+        "hosted-git-info": "^2.6.0",
+        "iferr": "^1.0.0",
+        "imurmurhash": "*",
+        "inflight": "~1.0.6",
+        "inherits": "~2.0.3",
+        "ini": "^1.3.5",
+        "init-package-json": "^1.10.3",
+        "is-cidr": "^2.0.6",
+        "json-parse-better-errors": "^1.0.2",
+        "lazy-property": "~1.0.0",
+        "libcipm": "^2.0.0",
+        "libnpmhook": "^4.0.1",
+        "libnpx": "^10.2.0",
+        "lock-verify": "^2.0.2",
+        "lockfile": "^1.0.4",
+        "lodash._baseindexof": "*",
+        "lodash._baseuniq": "~4.6.0",
+        "lodash._bindcallback": "*",
+        "lodash._cacheindexof": "*",
+        "lodash._createcache": "*",
+        "lodash._getnative": "*",
+        "lodash.clonedeep": "~4.5.0",
+        "lodash.restparam": "*",
+        "lodash.union": "~4.6.0",
+        "lodash.uniq": "~4.5.0",
+        "lodash.without": "~4.4.0",
+        "lru-cache": "^4.1.3",
+        "meant": "~1.0.1",
+        "mississippi": "^3.0.0",
+        "mkdirp": "~0.5.1",
+        "move-concurrently": "^1.0.1",
+        "node-gyp": "^3.7.0",
+        "nopt": "~4.0.1",
+        "normalize-package-data": "~2.4.0",
+        "npm-audit-report": "^1.3.1",
+        "npm-cache-filename": "~1.0.2",
+        "npm-install-checks": "~3.0.0",
+        "npm-lifecycle": "^2.0.3",
+        "npm-package-arg": "^6.1.0",
+        "npm-packlist": "~1.1.10",
+        "npm-pick-manifest": "^2.1.0",
+        "npm-profile": "^3.0.2",
+        "npm-registry-client": "^8.5.1",
+        "npm-registry-fetch": "^1.1.0",
+        "npm-user-validate": "~1.0.0",
+        "npmlog": "~4.1.2",
+        "once": "~1.4.0",
+        "opener": "~1.4.3",
+        "osenv": "^0.1.5",
+        "pacote": "^8.1.6",
+        "path-is-inside": "~1.0.2",
+        "promise-inflight": "~1.0.1",
+        "qrcode-terminal": "^0.12.0",
+        "query-string": "^6.1.0",
+        "qw": "~1.0.1",
+        "read": "~1.0.7",
+        "read-cmd-shim": "~1.0.1",
+        "read-installed": "~4.0.3",
+        "read-package-json": "^2.0.13",
+        "read-package-tree": "^5.2.1",
+        "readable-stream": "^2.3.6",
+        "readdir-scoped-modules": "*",
+        "request": "^2.81.0",
+        "retry": "^0.12.0",
+        "rimraf": "~2.6.2",
+        "safe-buffer": "^5.1.2",
+        "semver": "^5.5.0",
+        "sha": "~2.0.1",
+        "slide": "~1.1.6",
+        "sorted-object": "~2.0.1",
+        "sorted-union-stream": "~2.1.3",
+        "ssri": "^6.0.0",
+        "stringify-package": "^1.0.0",
+        "tar": "^4.4.4",
+        "text-table": "~0.2.0",
+        "tiny-relative-date": "^1.3.0",
+        "uid-number": "0.0.6",
+        "umask": "~1.1.0",
+        "unique-filename": "~1.1.0",
+        "unpipe": "~1.0.0",
+        "update-notifier": "^2.5.0",
+        "uuid": "^3.3.2",
+        "validate-npm-package-license": "^3.0.3",
+        "validate-npm-package-name": "~3.0.0",
+        "which": "^1.3.1",
+        "worker-farm": "^1.6.0",
+        "write-file-atomic": "^2.3.0"
+      },
+      "dependencies": {
+        "JSONStream": {
+          "version": "1.3.3",
+          "bundled": true,
+          "requires": {
+            "jsonparse": "^1.2.0",
+            "through": ">=2.2.7 <3"
+          }
+        },
+        "abbrev": {
+          "version": "1.1.1",
+          "bundled": true
+        },
+        "agent-base": {
+          "version": "4.2.0",
+          "bundled": true,
+          "requires": {
+            "es6-promisify": "^5.0.0"
+          }
+        },
+        "agentkeepalive": {
+          "version": "3.4.1",
+          "bundled": true,
+          "requires": {
+            "humanize-ms": "^1.2.1"
+          }
+        },
+        "ansi-align": {
+          "version": "2.0.0",
+          "bundled": true,
+          "requires": {
+            "string-width": "^2.0.0"
+          }
+        },
+        "ansi-regex": {
+          "version": "2.1.1",
+          "bundled": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "bundled": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "ansicolors": {
+          "version": "0.3.2",
+          "bundled": true
+        },
+        "ansistyles": {
+          "version": "0.1.3",
+          "bundled": true
+        },
+        "aproba": {
+          "version": "1.2.0",
+          "bundled": true
+        },
+        "archy": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "are-we-there-yet": {
+          "version": "1.1.4",
+          "bundled": true,
+          "requires": {
+            "delegates": "^1.0.0",
+            "readable-stream": "^2.0.6"
+          }
+        },
+        "asap": {
+          "version": "2.0.6",
+          "bundled": true
+        },
+        "asn1": {
+          "version": "0.2.3",
+          "bundled": true
+        },
+        "assert-plus": {
+          "version": "0.2.0",
+          "bundled": true
+        },
+        "asynckit": {
+          "version": "0.4.0",
+          "bundled": true
+        },
+        "aws-sign2": {
+          "version": "0.6.0",
+          "bundled": true
+        },
+        "aws4": {
+          "version": "1.7.0",
+          "bundled": true
+        },
+        "balanced-match": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "bcrypt-pbkdf": {
+          "version": "1.0.2",
+          "bundled": true,
+          "optional": true,
+          "requires": {
+            "tweetnacl": "^0.14.3"
+          }
+        },
+        "bin-links": {
+          "version": "1.1.2",
+          "bundled": true,
+          "requires": {
+            "bluebird": "^3.5.0",
+            "cmd-shim": "^2.0.2",
+            "gentle-fs": "^2.0.0",
+            "graceful-fs": "^4.1.11",
+            "write-file-atomic": "^2.3.0"
+          }
+        },
+        "block-stream": {
+          "version": "0.0.9",
+          "bundled": true,
+          "requires": {
+            "inherits": "~2.0.0"
+          }
+        },
+        "bluebird": {
+          "version": "3.5.1",
+          "bundled": true
+        },
+        "boom": {
+          "version": "2.10.1",
+          "bundled": true,
+          "requires": {
+            "hoek": "2.x.x"
+          }
+        },
+        "boxen": {
+          "version": "1.3.0",
+          "bundled": true,
+          "requires": {
+            "ansi-align": "^2.0.0",
+            "camelcase": "^4.0.0",
+            "chalk": "^2.0.1",
+            "cli-boxes": "^1.0.0",
+            "string-width": "^2.0.0",
+            "term-size": "^1.2.0",
+            "widest-line": "^2.0.0"
+          }
+        },
+        "brace-expansion": {
+          "version": "1.1.11",
+          "bundled": true,
+          "requires": {
+            "balanced-match": "^1.0.0",
+            "concat-map": "0.0.1"
+          }
+        },
+        "buffer-from": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "builtin-modules": {
+          "version": "1.1.1",
+          "bundled": true
+        },
+        "builtins": {
+          "version": "1.0.3",
+          "bundled": true
+        },
+        "byline": {
+          "version": "5.0.0",
+          "bundled": true
+        },
+        "byte-size": {
+          "version": "4.0.3",
+          "bundled": true
+        },
+        "cacache": {
+          "version": "11.1.0",
+          "bundled": true,
+          "requires": {
+            "bluebird": "^3.5.1",
+            "chownr": "^1.0.1",
+            "figgy-pudding": "^3.1.0",
+            "glob": "^7.1.2",
+            "graceful-fs": "^4.1.11",
+            "lru-cache": "^4.1.3",
+            "mississippi": "^3.0.0",
+            "mkdirp": "^0.5.1",
+            "move-concurrently": "^1.0.1",
+            "promise-inflight": "^1.0.1",
+            "rimraf": "^2.6.2",
+            "ssri": "^6.0.0",
+            "unique-filename": "^1.1.0",
+            "y18n": "^4.0.0"
+          }
+        },
+        "call-limit": {
+          "version": "1.1.0",
+          "bundled": true
+        },
+        "camelcase": {
+          "version": "4.1.0",
+          "bundled": true
+        },
+        "capture-stack-trace": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "caseless": {
+          "version": "0.12.0",
+          "bundled": true
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "bundled": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "chownr": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "ci-info": {
+          "version": "1.1.3",
+          "bundled": true
+        },
+        "cidr-regex": {
+          "version": "2.0.9",
+          "bundled": true,
+          "requires": {
+            "ip-regex": "^2.1.0"
+          }
+        },
+        "cli-boxes": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "cli-columns": {
+          "version": "3.1.2",
+          "bundled": true,
+          "requires": {
+            "string-width": "^2.0.0",
+            "strip-ansi": "^3.0.1"
+          }
+        },
+        "cli-table3": {
+          "version": "0.5.0",
+          "bundled": true,
+          "requires": {
+            "colors": "^1.1.2",
+            "object-assign": "^4.1.0",
+            "string-width": "^2.1.1"
+          }
+        },
+        "cliui": {
+          "version": "4.1.0",
+          "bundled": true,
+          "requires": {
+            "string-width": "^2.1.1",
+            "strip-ansi": "^4.0.0",
+            "wrap-ansi": "^2.0.0"
+          },
+          "dependencies": {
+            "ansi-regex": {
+              "version": "3.0.0",
+              "bundled": true
+            },
+            "strip-ansi": {
+              "version": "4.0.0",
+              "bundled": true,
+              "requires": {
+                "ansi-regex": "^3.0.0"
+              }
+            }
+          }
+        },
+        "clone": {
+          "version": "1.0.4",
+          "bundled": true
+        },
+        "cmd-shim": {
+          "version": "2.0.2",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.2",
+            "mkdirp": "~0.5.0"
+          }
+        },
+        "co": {
+          "version": "4.6.0",
+          "bundled": true
+        },
+        "code-point-at": {
+          "version": "1.1.0",
+          "bundled": true
+        },
+        "color-convert": {
+          "version": "1.9.1",
+          "bundled": true,
+          "requires": {
+            "color-name": "^1.1.1"
+          }
+        },
+        "color-name": {
+          "version": "1.1.3",
+          "bundled": true
+        },
+        "colors": {
+          "version": "1.3.0",
+          "bundled": true,
+          "optional": true
+        },
+        "columnify": {
+          "version": "1.5.4",
+          "bundled": true,
+          "requires": {
+            "strip-ansi": "^3.0.0",
+            "wcwidth": "^1.0.0"
+          }
+        },
+        "combined-stream": {
+          "version": "1.0.6",
+          "bundled": true,
+          "requires": {
+            "delayed-stream": "~1.0.0"
+          }
+        },
+        "concat-map": {
+          "version": "0.0.1",
+          "bundled": true
+        },
+        "concat-stream": {
+          "version": "1.6.2",
+          "bundled": true,
+          "requires": {
+            "buffer-from": "^1.0.0",
+            "inherits": "^2.0.3",
+            "readable-stream": "^2.2.2",
+            "typedarray": "^0.0.6"
+          }
+        },
+        "config-chain": {
+          "version": "1.1.11",
+          "bundled": true,
+          "requires": {
+            "ini": "^1.3.4",
+            "proto-list": "~1.2.1"
+          }
+        },
+        "configstore": {
+          "version": "3.1.2",
+          "bundled": true,
+          "requires": {
+            "dot-prop": "^4.1.0",
+            "graceful-fs": "^4.1.2",
+            "make-dir": "^1.0.0",
+            "unique-string": "^1.0.0",
+            "write-file-atomic": "^2.0.0",
+            "xdg-basedir": "^3.0.0"
+          }
+        },
+        "console-control-strings": {
+          "version": "1.1.0",
+          "bundled": true
+        },
+        "copy-concurrently": {
+          "version": "1.0.5",
+          "bundled": true,
+          "requires": {
+            "aproba": "^1.1.1",
+            "fs-write-stream-atomic": "^1.0.8",
+            "iferr": "^0.1.5",
+            "mkdirp": "^0.5.1",
+            "rimraf": "^2.5.4",
+            "run-queue": "^1.0.0"
+          },
+          "dependencies": {
+            "iferr": {
+              "version": "0.1.5",
+              "bundled": true
+            }
+          }
+        },
+        "core-util-is": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "create-error-class": {
+          "version": "3.0.2",
+          "bundled": true,
+          "requires": {
+            "capture-stack-trace": "^1.0.0"
+          }
+        },
+        "cross-spawn": {
+          "version": "5.1.0",
+          "bundled": true,
+          "requires": {
+            "lru-cache": "^4.0.1",
+            "shebang-command": "^1.2.0",
+            "which": "^1.2.9"
+          }
+        },
+        "cryptiles": {
+          "version": "2.0.5",
+          "bundled": true,
+          "requires": {
+            "boom": "2.x.x"
+          }
+        },
+        "crypto-random-string": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "cyclist": {
+          "version": "0.2.2",
+          "bundled": true
+        },
+        "dashdash": {
+          "version": "1.14.1",
+          "bundled": true,
+          "requires": {
+            "assert-plus": "^1.0.0"
+          },
+          "dependencies": {
+            "assert-plus": {
+              "version": "1.0.0",
+              "bundled": true
+            }
+          }
+        },
+        "debug": {
+          "version": "3.1.0",
+          "bundled": true,
+          "requires": {
+            "ms": "2.0.0"
+          },
+          "dependencies": {
+            "ms": {
+              "version": "2.0.0",
+              "bundled": true
+            }
+          }
+        },
+        "debuglog": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "decamelize": {
+          "version": "1.2.0",
+          "bundled": true
+        },
+        "decode-uri-component": {
+          "version": "0.2.0",
+          "bundled": true
+        },
+        "deep-extend": {
+          "version": "0.5.1",
+          "bundled": true
+        },
+        "defaults": {
+          "version": "1.0.3",
+          "bundled": true,
+          "requires": {
+            "clone": "^1.0.2"
+          }
+        },
+        "delayed-stream": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "delegates": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "detect-indent": {
+          "version": "5.0.0",
+          "bundled": true
+        },
+        "detect-newline": {
+          "version": "2.1.0",
+          "bundled": true
+        },
+        "dezalgo": {
+          "version": "1.0.3",
+          "bundled": true,
+          "requires": {
+            "asap": "^2.0.0",
+            "wrappy": "1"
+          }
+        },
+        "dot-prop": {
+          "version": "4.2.0",
+          "bundled": true,
+          "requires": {
+            "is-obj": "^1.0.0"
+          }
+        },
+        "dotenv": {
+          "version": "5.0.1",
+          "bundled": true
+        },
+        "duplexer3": {
+          "version": "0.1.4",
+          "bundled": true
+        },
+        "duplexify": {
+          "version": "3.6.0",
+          "bundled": true,
+          "requires": {
+            "end-of-stream": "^1.0.0",
+            "inherits": "^2.0.1",
+            "readable-stream": "^2.0.0",
+            "stream-shift": "^1.0.0"
+          }
+        },
+        "ecc-jsbn": {
+          "version": "0.1.1",
+          "bundled": true,
+          "optional": true,
+          "requires": {
+            "jsbn": "~0.1.0"
+          }
+        },
+        "editor": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "encoding": {
+          "version": "0.1.12",
+          "bundled": true,
+          "requires": {
+            "iconv-lite": "~0.4.13"
+          }
+        },
+        "end-of-stream": {
+          "version": "1.4.1",
+          "bundled": true,
+          "requires": {
+            "once": "^1.4.0"
+          }
+        },
+        "err-code": {
+          "version": "1.1.2",
+          "bundled": true
+        },
+        "errno": {
+          "version": "0.1.7",
+          "bundled": true,
+          "requires": {
+            "prr": "~1.0.1"
+          }
+        },
+        "es6-promise": {
+          "version": "4.2.4",
+          "bundled": true
+        },
+        "es6-promisify": {
+          "version": "5.0.0",
+          "bundled": true,
+          "requires": {
+            "es6-promise": "^4.0.3"
+          }
+        },
+        "escape-string-regexp": {
+          "version": "1.0.5",
+          "bundled": true
+        },
+        "execa": {
+          "version": "0.7.0",
+          "bundled": true,
+          "requires": {
+            "cross-spawn": "^5.0.1",
+            "get-stream": "^3.0.0",
+            "is-stream": "^1.1.0",
+            "npm-run-path": "^2.0.0",
+            "p-finally": "^1.0.0",
+            "signal-exit": "^3.0.0",
+            "strip-eof": "^1.0.0"
+          }
+        },
+        "extend": {
+          "version": "3.0.1",
+          "bundled": true
+        },
+        "extsprintf": {
+          "version": "1.3.0",
+          "bundled": true
+        },
+        "figgy-pudding": {
+          "version": "3.2.0",
+          "bundled": true
+        },
+        "find-npm-prefix": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "find-up": {
+          "version": "2.1.0",
+          "bundled": true,
+          "requires": {
+            "locate-path": "^2.0.0"
+          }
+        },
+        "flush-write-stream": {
+          "version": "1.0.3",
+          "bundled": true,
+          "requires": {
+            "inherits": "^2.0.1",
+            "readable-stream": "^2.0.4"
+          }
+        },
+        "forever-agent": {
+          "version": "0.6.1",
+          "bundled": true
+        },
+        "form-data": {
+          "version": "2.1.4",
+          "bundled": true,
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.5",
+            "mime-types": "^2.1.12"
+          }
+        },
+        "from2": {
+          "version": "2.3.0",
+          "bundled": true,
+          "requires": {
+            "inherits": "^2.0.1",
+            "readable-stream": "^2.0.0"
+          }
+        },
+        "fs-minipass": {
+          "version": "1.2.5",
+          "bundled": true,
+          "requires": {
+            "minipass": "^2.2.1"
+          }
+        },
+        "fs-vacuum": {
+          "version": "1.2.10",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.2",
+            "path-is-inside": "^1.0.1",
+            "rimraf": "^2.5.2"
+          }
+        },
+        "fs-write-stream-atomic": {
+          "version": "1.0.10",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.2",
+            "iferr": "^0.1.5",
+            "imurmurhash": "^0.1.4",
+            "readable-stream": "1 || 2"
+          },
+          "dependencies": {
+            "iferr": {
+              "version": "0.1.5",
+              "bundled": true
+            }
+          }
+        },
+        "fs.realpath": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "fstream": {
+          "version": "1.0.11",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.2",
+            "inherits": "~2.0.0",
+            "mkdirp": ">=0.5 0",
+            "rimraf": "2"
+          }
+        },
+        "gauge": {
+          "version": "2.7.4",
+          "bundled": true,
+          "requires": {
+            "aproba": "^1.0.3",
+            "console-control-strings": "^1.0.0",
+            "has-unicode": "^2.0.0",
+            "object-assign": "^4.1.0",
+            "signal-exit": "^3.0.0",
+            "string-width": "^1.0.1",
+            "strip-ansi": "^3.0.1",
+            "wide-align": "^1.1.0"
+          },
+          "dependencies": {
+            "string-width": {
+              "version": "1.0.2",
+              "bundled": true,
+              "requires": {
+                "code-point-at": "^1.0.0",
+                "is-fullwidth-code-point": "^1.0.0",
+                "strip-ansi": "^3.0.0"
+              }
+            }
+          }
+        },
+        "genfun": {
+          "version": "4.0.1",
+          "bundled": true
+        },
+        "gentle-fs": {
+          "version": "2.0.1",
+          "bundled": true,
+          "requires": {
+            "aproba": "^1.1.2",
+            "fs-vacuum": "^1.2.10",
+            "graceful-fs": "^4.1.11",
+            "iferr": "^0.1.5",
+            "mkdirp": "^0.5.1",
+            "path-is-inside": "^1.0.2",
+            "read-cmd-shim": "^1.0.1",
+            "slide": "^1.1.6"
+          },
+          "dependencies": {
+            "iferr": {
+              "version": "0.1.5",
+              "bundled": true
+            }
+          }
+        },
+        "get-caller-file": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "get-stream": {
+          "version": "3.0.0",
+          "bundled": true
+        },
+        "getpass": {
+          "version": "0.1.7",
+          "bundled": true,
+          "requires": {
+            "assert-plus": "^1.0.0"
+          },
+          "dependencies": {
+            "assert-plus": {
+              "version": "1.0.0",
+              "bundled": true
+            }
+          }
+        },
+        "glob": {
+          "version": "7.1.2",
+          "bundled": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        },
+        "global-dirs": {
+          "version": "0.1.1",
+          "bundled": true,
+          "requires": {
+            "ini": "^1.3.4"
+          }
+        },
+        "got": {
+          "version": "6.7.1",
+          "bundled": true,
+          "requires": {
+            "create-error-class": "^3.0.0",
+            "duplexer3": "^0.1.4",
+            "get-stream": "^3.0.0",
+            "is-redirect": "^1.0.0",
+            "is-retry-allowed": "^1.0.0",
+            "is-stream": "^1.0.0",
+            "lowercase-keys": "^1.0.0",
+            "safe-buffer": "^5.0.1",
+            "timed-out": "^4.0.0",
+            "unzip-response": "^2.0.1",
+            "url-parse-lax": "^1.0.0"
+          }
+        },
+        "graceful-fs": {
+          "version": "4.1.11",
+          "bundled": true
+        },
+        "har-schema": {
+          "version": "1.0.5",
+          "bundled": true
+        },
+        "har-validator": {
+          "version": "4.2.1",
+          "bundled": true,
+          "requires": {
+            "ajv": "^4.9.1",
+            "har-schema": "^1.0.5"
+          },
+          "dependencies": {
+            "ajv": {
+              "version": "4.11.8",
+              "bundled": true,
+              "requires": {
+                "co": "^4.6.0",
+                "json-stable-stringify": "^1.0.1"
+              }
+            }
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "bundled": true
+        },
+        "has-unicode": {
+          "version": "2.0.1",
+          "bundled": true
+        },
+        "hawk": {
+          "version": "3.1.3",
+          "bundled": true,
+          "requires": {
+            "boom": "2.x.x",
+            "cryptiles": "2.x.x",
+            "hoek": "2.x.x",
+            "sntp": "1.x.x"
+          }
+        },
+        "hoek": {
+          "version": "2.16.3",
+          "bundled": true
+        },
+        "hosted-git-info": {
+          "version": "2.6.0",
+          "bundled": true
+        },
+        "http-cache-semantics": {
+          "version": "3.8.1",
+          "bundled": true
+        },
+        "http-proxy-agent": {
+          "version": "2.1.0",
+          "bundled": true,
+          "requires": {
+            "agent-base": "4",
+            "debug": "3.1.0"
+          }
+        },
+        "http-signature": {
+          "version": "1.1.1",
+          "bundled": true,
+          "requires": {
+            "assert-plus": "^0.2.0",
+            "jsprim": "^1.2.2",
+            "sshpk": "^1.7.0"
+          }
+        },
+        "https-proxy-agent": {
+          "version": "2.2.1",
+          "bundled": true,
+          "requires": {
+            "agent-base": "^4.1.0",
+            "debug": "^3.1.0"
+          }
+        },
+        "humanize-ms": {
+          "version": "1.2.1",
+          "bundled": true,
+          "requires": {
+            "ms": "^2.0.0"
+          }
+        },
+        "iconv-lite": {
+          "version": "0.4.23",
+          "bundled": true,
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3"
+          }
+        },
+        "iferr": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "ignore-walk": {
+          "version": "3.0.1",
+          "bundled": true,
+          "requires": {
+            "minimatch": "^3.0.4"
+          }
+        },
+        "import-lazy": {
+          "version": "2.1.0",
+          "bundled": true
+        },
+        "imurmurhash": {
+          "version": "0.1.4",
+          "bundled": true
+        },
+        "inflight": {
+          "version": "1.0.6",
+          "bundled": true,
+          "requires": {
+            "once": "^1.3.0",
+            "wrappy": "1"
+          }
+        },
+        "inherits": {
+          "version": "2.0.3",
+          "bundled": true
+        },
+        "ini": {
+          "version": "1.3.5",
+          "bundled": true
+        },
+        "init-package-json": {
+          "version": "1.10.3",
+          "bundled": true,
+          "requires": {
+            "glob": "^7.1.1",
+            "npm-package-arg": "^4.0.0 || ^5.0.0 || ^6.0.0",
+            "promzard": "^0.3.0",
+            "read": "~1.0.1",
+            "read-package-json": "1 || 2",
+            "semver": "2.x || 3.x || 4 || 5",
+            "validate-npm-package-license": "^3.0.1",
+            "validate-npm-package-name": "^3.0.0"
+          }
+        },
+        "invert-kv": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "ip": {
+          "version": "1.1.5",
+          "bundled": true
+        },
+        "ip-regex": {
+          "version": "2.1.0",
+          "bundled": true
+        },
+        "is-builtin-module": {
+          "version": "1.0.0",
+          "bundled": true,
+          "requires": {
+            "builtin-modules": "^1.0.0"
+          }
+        },
+        "is-ci": {
+          "version": "1.1.0",
+          "bundled": true,
+          "requires": {
+            "ci-info": "^1.0.0"
+          }
+        },
+        "is-cidr": {
+          "version": "2.0.6",
+          "bundled": true,
+          "requires": {
+            "cidr-regex": "^2.0.8"
+          }
+        },
+        "is-fullwidth-code-point": {
+          "version": "1.0.0",
+          "bundled": true,
+          "requires": {
+            "number-is-nan": "^1.0.0"
+          }
+        },
+        "is-installed-globally": {
+          "version": "0.1.0",
+          "bundled": true,
+          "requires": {
+            "global-dirs": "^0.1.0",
+            "is-path-inside": "^1.0.0"
+          }
+        },
+        "is-npm": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "is-obj": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "is-path-inside": {
+          "version": "1.0.1",
+          "bundled": true,
+          "requires": {
+            "path-is-inside": "^1.0.1"
+          }
+        },
+        "is-redirect": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "is-retry-allowed": {
+          "version": "1.1.0",
+          "bundled": true
+        },
+        "is-stream": {
+          "version": "1.1.0",
+          "bundled": true
+        },
+        "is-typedarray": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "isarray": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "isexe": {
+          "version": "2.0.0",
+          "bundled": true
+        },
+        "isstream": {
+          "version": "0.1.2",
+          "bundled": true
+        },
+        "jsbn": {
+          "version": "0.1.1",
+          "bundled": true,
+          "optional": true
+        },
+        "json-parse-better-errors": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "json-schema": {
+          "version": "0.2.3",
+          "bundled": true
+        },
+        "json-stable-stringify": {
+          "version": "1.0.1",
+          "bundled": true,
+          "requires": {
+            "jsonify": "~0.0.0"
+          }
+        },
+        "json-stringify-safe": {
+          "version": "5.0.1",
+          "bundled": true
+        },
+        "jsonify": {
+          "version": "0.0.0",
+          "bundled": true
+        },
+        "jsonparse": {
+          "version": "1.3.1",
+          "bundled": true
+        },
+        "jsprim": {
+          "version": "1.4.1",
+          "bundled": true,
+          "requires": {
+            "assert-plus": "1.0.0",
+            "extsprintf": "1.3.0",
+            "json-schema": "0.2.3",
+            "verror": "1.10.0"
+          },
+          "dependencies": {
+            "assert-plus": {
+              "version": "1.0.0",
+              "bundled": true
+            }
+          }
+        },
+        "latest-version": {
+          "version": "3.1.0",
+          "bundled": true,
+          "requires": {
+            "package-json": "^4.0.0"
+          }
+        },
+        "lazy-property": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "lcid": {
+          "version": "1.0.0",
+          "bundled": true,
+          "requires": {
+            "invert-kv": "^1.0.0"
+          }
+        },
+        "libcipm": {
+          "version": "2.0.0",
+          "bundled": true,
+          "requires": {
+            "bin-links": "^1.1.2",
+            "bluebird": "^3.5.1",
+            "find-npm-prefix": "^1.0.2",
+            "graceful-fs": "^4.1.11",
+            "lock-verify": "^2.0.2",
+            "npm-lifecycle": "^2.0.3",
+            "npm-logical-tree": "^1.2.1",
+            "npm-package-arg": "^6.1.0",
+            "pacote": "^8.1.6",
+            "protoduck": "^5.0.0",
+            "read-package-json": "^2.0.13",
+            "rimraf": "^2.6.2",
+            "worker-farm": "^1.6.0"
+          }
+        },
+        "libnpmhook": {
+          "version": "4.0.1",
+          "bundled": true,
+          "requires": {
+            "figgy-pudding": "^3.1.0",
+            "npm-registry-fetch": "^3.0.0"
+          },
+          "dependencies": {
+            "npm-registry-fetch": {
+              "version": "3.1.1",
+              "bundled": true,
+              "requires": {
+                "bluebird": "^3.5.1",
+                "figgy-pudding": "^3.1.0",
+                "lru-cache": "^4.1.2",
+                "make-fetch-happen": "^4.0.0",
+                "npm-package-arg": "^6.0.0"
+              }
+            }
+          }
+        },
+        "libnpx": {
+          "version": "10.2.0",
+          "bundled": true,
+          "requires": {
+            "dotenv": "^5.0.1",
+            "npm-package-arg": "^6.0.0",
+            "rimraf": "^2.6.2",
+            "safe-buffer": "^5.1.0",
+            "update-notifier": "^2.3.0",
+            "which": "^1.3.0",
+            "y18n": "^4.0.0",
+            "yargs": "^11.0.0"
+          }
+        },
+        "locate-path": {
+          "version": "2.0.0",
+          "bundled": true,
+          "requires": {
+            "p-locate": "^2.0.0",
+            "path-exists": "^3.0.0"
+          }
+        },
+        "lock-verify": {
+          "version": "2.0.2",
+          "bundled": true,
+          "requires": {
+            "npm-package-arg": "^5.1.2 || 6",
+            "semver": "^5.4.1"
+          }
+        },
+        "lockfile": {
+          "version": "1.0.4",
+          "bundled": true,
+          "requires": {
+            "signal-exit": "^3.0.2"
+          }
+        },
+        "lodash._baseindexof": {
+          "version": "3.1.0",
+          "bundled": true
+        },
+        "lodash._baseuniq": {
+          "version": "4.6.0",
+          "bundled": true,
+          "requires": {
+            "lodash._createset": "~4.0.0",
+            "lodash._root": "~3.0.0"
+          }
+        },
+        "lodash._bindcallback": {
+          "version": "3.0.1",
+          "bundled": true
+        },
+        "lodash._cacheindexof": {
+          "version": "3.0.2",
+          "bundled": true
+        },
+        "lodash._createcache": {
+          "version": "3.1.2",
+          "bundled": true,
+          "requires": {
+            "lodash._getnative": "^3.0.0"
+          }
+        },
+        "lodash._createset": {
+          "version": "4.0.3",
+          "bundled": true
+        },
+        "lodash._getnative": {
+          "version": "3.9.1",
+          "bundled": true
+        },
+        "lodash._root": {
+          "version": "3.0.1",
+          "bundled": true
+        },
+        "lodash.clonedeep": {
+          "version": "4.5.0",
+          "bundled": true
+        },
+        "lodash.restparam": {
+          "version": "3.6.1",
+          "bundled": true
+        },
+        "lodash.union": {
+          "version": "4.6.0",
+          "bundled": true
+        },
+        "lodash.uniq": {
+          "version": "4.5.0",
+          "bundled": true
+        },
+        "lodash.without": {
+          "version": "4.4.0",
+          "bundled": true
+        },
+        "lowercase-keys": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "lru-cache": {
+          "version": "4.1.3",
+          "bundled": true,
+          "requires": {
+            "pseudomap": "^1.0.2",
+            "yallist": "^2.1.2"
+          }
+        },
+        "make-dir": {
+          "version": "1.3.0",
+          "bundled": true,
+          "requires": {
+            "pify": "^3.0.0"
+          }
+        },
+        "make-fetch-happen": {
+          "version": "4.0.1",
+          "bundled": true,
+          "requires": {
+            "agentkeepalive": "^3.4.1",
+            "cacache": "^11.0.1",
+            "http-cache-semantics": "^3.8.1",
+            "http-proxy-agent": "^2.1.0",
+            "https-proxy-agent": "^2.2.1",
+            "lru-cache": "^4.1.2",
+            "mississippi": "^3.0.0",
+            "node-fetch-npm": "^2.0.2",
+            "promise-retry": "^1.1.1",
+            "socks-proxy-agent": "^4.0.0",
+            "ssri": "^6.0.0"
+          }
+        },
+        "meant": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "mem": {
+          "version": "1.1.0",
+          "bundled": true,
+          "requires": {
+            "mimic-fn": "^1.0.0"
+          }
+        },
+        "mime-db": {
+          "version": "1.33.0",
+          "bundled": true
+        },
+        "mime-types": {
+          "version": "2.1.18",
+          "bundled": true,
+          "requires": {
+            "mime-db": "~1.33.0"
+          }
+        },
+        "mimic-fn": {
+          "version": "1.2.0",
+          "bundled": true
+        },
+        "minimatch": {
+          "version": "3.0.4",
+          "bundled": true,
+          "requires": {
+            "brace-expansion": "^1.1.7"
+          }
+        },
+        "minimist": {
+          "version": "0.0.8",
+          "bundled": true
+        },
+        "minipass": {
+          "version": "2.3.3",
+          "bundled": true,
+          "requires": {
+            "safe-buffer": "^5.1.2",
+            "yallist": "^3.0.0"
+          },
+          "dependencies": {
+            "yallist": {
+              "version": "3.0.2",
+              "bundled": true
+            }
+          }
+        },
+        "minizlib": {
+          "version": "1.1.0",
+          "bundled": true,
+          "requires": {
+            "minipass": "^2.2.1"
+          }
+        },
+        "mississippi": {
+          "version": "3.0.0",
+          "bundled": true,
+          "requires": {
+            "concat-stream": "^1.5.0",
+            "duplexify": "^3.4.2",
+            "end-of-stream": "^1.1.0",
+            "flush-write-stream": "^1.0.0",
+            "from2": "^2.1.0",
+            "parallel-transform": "^1.1.0",
+            "pump": "^3.0.0",
+            "pumpify": "^1.3.3",
+            "stream-each": "^1.1.0",
+            "through2": "^2.0.0"
+          }
+        },
+        "mkdirp": {
+          "version": "0.5.1",
+          "bundled": true,
+          "requires": {
+            "minimist": "0.0.8"
+          }
+        },
+        "move-concurrently": {
+          "version": "1.0.1",
+          "bundled": true,
+          "requires": {
+            "aproba": "^1.1.1",
+            "copy-concurrently": "^1.0.0",
+            "fs-write-stream-atomic": "^1.0.8",
+            "mkdirp": "^0.5.1",
+            "rimraf": "^2.5.4",
+            "run-queue": "^1.0.3"
+          }
+        },
+        "ms": {
+          "version": "2.1.1",
+          "bundled": true
+        },
+        "mute-stream": {
+          "version": "0.0.7",
+          "bundled": true
+        },
+        "node-fetch-npm": {
+          "version": "2.0.2",
+          "bundled": true,
+          "requires": {
+            "encoding": "^0.1.11",
+            "json-parse-better-errors": "^1.0.0",
+            "safe-buffer": "^5.1.1"
+          }
+        },
+        "node-gyp": {
+          "version": "3.7.0",
+          "bundled": true,
+          "requires": {
+            "fstream": "^1.0.0",
+            "glob": "^7.0.3",
+            "graceful-fs": "^4.1.2",
+            "mkdirp": "^0.5.0",
+            "nopt": "2 || 3",
+            "npmlog": "0 || 1 || 2 || 3 || 4",
+            "osenv": "0",
+            "request": ">=2.9.0 <2.82.0",
+            "rimraf": "2",
+            "semver": "~5.3.0",
+            "tar": "^2.0.0",
+            "which": "1"
+          },
+          "dependencies": {
+            "nopt": {
+              "version": "3.0.6",
+              "bundled": true,
+              "requires": {
+                "abbrev": "1"
+              }
+            },
+            "semver": {
+              "version": "5.3.0",
+              "bundled": true
+            },
+            "tar": {
+              "version": "2.2.1",
+              "bundled": true,
+              "requires": {
+                "block-stream": "*",
+                "fstream": "^1.0.2",
+                "inherits": "2"
+              }
+            }
+          }
+        },
+        "nopt": {
+          "version": "4.0.1",
+          "bundled": true,
+          "requires": {
+            "abbrev": "1",
+            "osenv": "^0.1.4"
+          }
+        },
+        "normalize-package-data": {
+          "version": "2.4.0",
+          "bundled": true,
+          "requires": {
+            "hosted-git-info": "^2.1.4",
+            "is-builtin-module": "^1.0.0",
+            "semver": "2 || 3 || 4 || 5",
+            "validate-npm-package-license": "^3.0.1"
+          }
+        },
+        "npm-audit-report": {
+          "version": "1.3.1",
+          "bundled": true,
+          "requires": {
+            "cli-table3": "^0.5.0",
+            "console-control-strings": "^1.1.0"
+          }
+        },
+        "npm-bundled": {
+          "version": "1.0.3",
+          "bundled": true
+        },
+        "npm-cache-filename": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "npm-install-checks": {
+          "version": "3.0.0",
+          "bundled": true,
+          "requires": {
+            "semver": "^2.3.0 || 3.x || 4 || 5"
+          }
+        },
+        "npm-lifecycle": {
+          "version": "2.0.3",
+          "bundled": true,
+          "requires": {
+            "byline": "^5.0.0",
+            "graceful-fs": "^4.1.11",
+            "node-gyp": "^3.6.2",
+            "resolve-from": "^4.0.0",
+            "slide": "^1.1.6",
+            "uid-number": "0.0.6",
+            "umask": "^1.1.0",
+            "which": "^1.3.0"
+          }
+        },
+        "npm-logical-tree": {
+          "version": "1.2.1",
+          "bundled": true
+        },
+        "npm-package-arg": {
+          "version": "6.1.0",
+          "bundled": true,
+          "requires": {
+            "hosted-git-info": "^2.6.0",
+            "osenv": "^0.1.5",
+            "semver": "^5.5.0",
+            "validate-npm-package-name": "^3.0.0"
+          }
+        },
+        "npm-packlist": {
+          "version": "1.1.10",
+          "bundled": true,
+          "requires": {
+            "ignore-walk": "^3.0.1",
+            "npm-bundled": "^1.0.1"
+          }
+        },
+        "npm-pick-manifest": {
+          "version": "2.1.0",
+          "bundled": true,
+          "requires": {
+            "npm-package-arg": "^6.0.0",
+            "semver": "^5.4.1"
+          }
+        },
+        "npm-profile": {
+          "version": "3.0.2",
+          "bundled": true,
+          "requires": {
+            "aproba": "^1.1.2 || 2",
+            "make-fetch-happen": "^2.5.0 || 3 || 4"
+          }
+        },
+        "npm-registry-client": {
+          "version": "8.5.1",
+          "bundled": true,
+          "requires": {
+            "concat-stream": "^1.5.2",
+            "graceful-fs": "^4.1.6",
+            "normalize-package-data": "~1.0.1 || ^2.0.0",
+            "npm-package-arg": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0",
+            "npmlog": "2 || ^3.1.0 || ^4.0.0",
+            "once": "^1.3.3",
+            "request": "^2.74.0",
+            "retry": "^0.10.0",
+            "safe-buffer": "^5.1.1",
+            "semver": "2 >=2.2.1 || 3.x || 4 || 5",
+            "slide": "^1.1.3",
+            "ssri": "^5.2.4"
+          },
+          "dependencies": {
+            "retry": {
+              "version": "0.10.1",
+              "bundled": true
+            },
+            "ssri": {
+              "version": "5.3.0",
+              "bundled": true,
+              "requires": {
+                "safe-buffer": "^5.1.1"
+              }
+            }
+          }
+        },
+        "npm-registry-fetch": {
+          "version": "1.1.0",
+          "bundled": true,
+          "requires": {
+            "bluebird": "^3.5.1",
+            "figgy-pudding": "^2.0.1",
+            "lru-cache": "^4.1.2",
+            "make-fetch-happen": "^3.0.0",
+            "npm-package-arg": "^6.0.0",
+            "safe-buffer": "^5.1.1"
+          },
+          "dependencies": {
+            "cacache": {
+              "version": "10.0.4",
+              "bundled": true,
+              "requires": {
+                "bluebird": "^3.5.1",
+                "chownr": "^1.0.1",
+                "glob": "^7.1.2",
+                "graceful-fs": "^4.1.11",
+                "lru-cache": "^4.1.1",
+                "mississippi": "^2.0.0",
+                "mkdirp": "^0.5.1",
+                "move-concurrently": "^1.0.1",
+                "promise-inflight": "^1.0.1",
+                "rimraf": "^2.6.2",
+                "ssri": "^5.2.4",
+                "unique-filename": "^1.1.0",
+                "y18n": "^4.0.0"
+              },
+              "dependencies": {
+                "mississippi": {
+                  "version": "2.0.0",
+                  "bundled": true,
+                  "requires": {
+                    "concat-stream": "^1.5.0",
+                    "duplexify": "^3.4.2",
+                    "end-of-stream": "^1.1.0",
+                    "flush-write-stream": "^1.0.0",
+                    "from2": "^2.1.0",
+                    "parallel-transform": "^1.1.0",
+                    "pump": "^2.0.1",
+                    "pumpify": "^1.3.3",
+                    "stream-each": "^1.1.0",
+                    "through2": "^2.0.0"
+                  }
+                }
+              }
+            },
+            "figgy-pudding": {
+              "version": "2.0.1",
+              "bundled": true
+            },
+            "make-fetch-happen": {
+              "version": "3.0.0",
+              "bundled": true,
+              "requires": {
+                "agentkeepalive": "^3.4.1",
+                "cacache": "^10.0.4",
+                "http-cache-semantics": "^3.8.1",
+                "http-proxy-agent": "^2.1.0",
+                "https-proxy-agent": "^2.2.0",
+                "lru-cache": "^4.1.2",
+                "mississippi": "^3.0.0",
+                "node-fetch-npm": "^2.0.2",
+                "promise-retry": "^1.1.1",
+                "socks-proxy-agent": "^3.0.1",
+                "ssri": "^5.2.4"
+              }
+            },
+            "pump": {
+              "version": "2.0.1",
+              "bundled": true,
+              "requires": {
+                "end-of-stream": "^1.1.0",
+                "once": "^1.3.1"
+              }
+            },
+            "smart-buffer": {
+              "version": "1.1.15",
+              "bundled": true
+            },
+            "socks": {
+              "version": "1.1.10",
+              "bundled": true,
+              "requires": {
+                "ip": "^1.1.4",
+                "smart-buffer": "^1.0.13"
+              }
+            },
+            "socks-proxy-agent": {
+              "version": "3.0.1",
+              "bundled": true,
+              "requires": {
+                "agent-base": "^4.1.0",
+                "socks": "^1.1.10"
+              }
+            },
+            "ssri": {
+              "version": "5.3.0",
+              "bundled": true,
+              "requires": {
+                "safe-buffer": "^5.1.1"
+              }
+            }
+          }
+        },
+        "npm-run-path": {
+          "version": "2.0.2",
+          "bundled": true,
+          "requires": {
+            "path-key": "^2.0.0"
+          }
+        },
+        "npm-user-validate": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "npmlog": {
+          "version": "4.1.2",
+          "bundled": true,
+          "requires": {
+            "are-we-there-yet": "~1.1.2",
+            "console-control-strings": "~1.1.0",
+            "gauge": "~2.7.3",
+            "set-blocking": "~2.0.0"
+          }
+        },
+        "number-is-nan": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "oauth-sign": {
+          "version": "0.8.2",
+          "bundled": true
+        },
+        "object-assign": {
+          "version": "4.1.1",
+          "bundled": true
+        },
+        "once": {
+          "version": "1.4.0",
+          "bundled": true,
+          "requires": {
+            "wrappy": "1"
+          }
+        },
+        "opener": {
+          "version": "1.4.3",
+          "bundled": true
+        },
+        "os-homedir": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "os-locale": {
+          "version": "2.1.0",
+          "bundled": true,
+          "requires": {
+            "execa": "^0.7.0",
+            "lcid": "^1.0.0",
+            "mem": "^1.1.0"
+          }
+        },
+        "os-tmpdir": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "osenv": {
+          "version": "0.1.5",
+          "bundled": true,
+          "requires": {
+            "os-homedir": "^1.0.0",
+            "os-tmpdir": "^1.0.0"
+          }
+        },
+        "p-finally": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "p-limit": {
+          "version": "1.2.0",
+          "bundled": true,
+          "requires": {
+            "p-try": "^1.0.0"
+          }
+        },
+        "p-locate": {
+          "version": "2.0.0",
+          "bundled": true,
+          "requires": {
+            "p-limit": "^1.1.0"
+          }
+        },
+        "p-try": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "package-json": {
+          "version": "4.0.1",
+          "bundled": true,
+          "requires": {
+            "got": "^6.7.1",
+            "registry-auth-token": "^3.0.1",
+            "registry-url": "^3.0.3",
+            "semver": "^5.1.0"
+          }
+        },
+        "pacote": {
+          "version": "8.1.6",
+          "bundled": true,
+          "requires": {
+            "bluebird": "^3.5.1",
+            "cacache": "^11.0.2",
+            "get-stream": "^3.0.0",
+            "glob": "^7.1.2",
+            "lru-cache": "^4.1.3",
+            "make-fetch-happen": "^4.0.1",
+            "minimatch": "^3.0.4",
+            "minipass": "^2.3.3",
+            "mississippi": "^3.0.0",
+            "mkdirp": "^0.5.1",
+            "normalize-package-data": "^2.4.0",
+            "npm-package-arg": "^6.1.0",
+            "npm-packlist": "^1.1.10",
+            "npm-pick-manifest": "^2.1.0",
+            "osenv": "^0.1.5",
+            "promise-inflight": "^1.0.1",
+            "promise-retry": "^1.1.1",
+            "protoduck": "^5.0.0",
+            "rimraf": "^2.6.2",
+            "safe-buffer": "^5.1.2",
+            "semver": "^5.5.0",
+            "ssri": "^6.0.0",
+            "tar": "^4.4.3",
+            "unique-filename": "^1.1.0",
+            "which": "^1.3.0"
+          }
+        },
+        "parallel-transform": {
+          "version": "1.1.0",
+          "bundled": true,
+          "requires": {
+            "cyclist": "~0.2.2",
+            "inherits": "^2.0.3",
+            "readable-stream": "^2.1.5"
+          }
+        },
+        "path-exists": {
+          "version": "3.0.0",
+          "bundled": true
+        },
+        "path-is-absolute": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "path-is-inside": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "path-key": {
+          "version": "2.0.1",
+          "bundled": true
+        },
+        "performance-now": {
+          "version": "0.2.0",
+          "bundled": true
+        },
+        "pify": {
+          "version": "3.0.0",
+          "bundled": true
+        },
+        "prepend-http": {
+          "version": "1.0.4",
+          "bundled": true
+        },
+        "process-nextick-args": {
+          "version": "2.0.0",
+          "bundled": true
+        },
+        "promise-inflight": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "promise-retry": {
+          "version": "1.1.1",
+          "bundled": true,
+          "requires": {
+            "err-code": "^1.0.0",
+            "retry": "^0.10.0"
+          },
+          "dependencies": {
+            "retry": {
+              "version": "0.10.1",
+              "bundled": true
+            }
+          }
+        },
+        "promzard": {
+          "version": "0.3.0",
+          "bundled": true,
+          "requires": {
+            "read": "1"
+          }
+        },
+        "proto-list": {
+          "version": "1.2.4",
+          "bundled": true
+        },
+        "protoduck": {
+          "version": "5.0.0",
+          "bundled": true,
+          "requires": {
+            "genfun": "^4.0.1"
+          }
+        },
+        "prr": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "pseudomap": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "pump": {
+          "version": "3.0.0",
+          "bundled": true,
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        },
+        "pumpify": {
+          "version": "1.5.1",
+          "bundled": true,
+          "requires": {
+            "duplexify": "^3.6.0",
+            "inherits": "^2.0.3",
+            "pump": "^2.0.0"
+          },
+          "dependencies": {
+            "pump": {
+              "version": "2.0.1",
+              "bundled": true,
+              "requires": {
+                "end-of-stream": "^1.1.0",
+                "once": "^1.3.1"
+              }
+            }
+          }
+        },
+        "punycode": {
+          "version": "1.4.1",
+          "bundled": true
+        },
+        "qrcode-terminal": {
+          "version": "0.12.0",
+          "bundled": true
+        },
+        "qs": {
+          "version": "6.4.0",
+          "bundled": true
+        },
+        "query-string": {
+          "version": "6.1.0",
+          "bundled": true,
+          "requires": {
+            "decode-uri-component": "^0.2.0",
+            "strict-uri-encode": "^2.0.0"
+          }
+        },
+        "qw": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "rc": {
+          "version": "1.2.7",
+          "bundled": true,
+          "requires": {
+            "deep-extend": "^0.5.1",
+            "ini": "~1.3.0",
+            "minimist": "^1.2.0",
+            "strip-json-comments": "~2.0.1"
+          },
+          "dependencies": {
+            "minimist": {
+              "version": "1.2.0",
+              "bundled": true
+            }
+          }
+        },
+        "read": {
+          "version": "1.0.7",
+          "bundled": true,
+          "requires": {
+            "mute-stream": "~0.0.4"
+          }
+        },
+        "read-cmd-shim": {
+          "version": "1.0.1",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.2"
+          }
+        },
+        "read-installed": {
+          "version": "4.0.3",
+          "bundled": true,
+          "requires": {
+            "debuglog": "^1.0.1",
+            "graceful-fs": "^4.1.2",
+            "read-package-json": "^2.0.0",
+            "readdir-scoped-modules": "^1.0.0",
+            "semver": "2 || 3 || 4 || 5",
+            "slide": "~1.1.3",
+            "util-extend": "^1.0.1"
+          }
+        },
+        "read-package-json": {
+          "version": "2.0.13",
+          "bundled": true,
+          "requires": {
+            "glob": "^7.1.1",
+            "graceful-fs": "^4.1.2",
+            "json-parse-better-errors": "^1.0.1",
+            "normalize-package-data": "^2.0.0",
+            "slash": "^1.0.0"
+          }
+        },
+        "read-package-tree": {
+          "version": "5.2.1",
+          "bundled": true,
+          "requires": {
+            "debuglog": "^1.0.1",
+            "dezalgo": "^1.0.0",
+            "once": "^1.3.0",
+            "read-package-json": "^2.0.0",
+            "readdir-scoped-modules": "^1.0.0"
+          }
+        },
+        "readable-stream": {
+          "version": "2.3.6",
+          "bundled": true,
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "readdir-scoped-modules": {
+          "version": "1.0.2",
+          "bundled": true,
+          "requires": {
+            "debuglog": "^1.0.1",
+            "dezalgo": "^1.0.0",
+            "graceful-fs": "^4.1.2",
+            "once": "^1.3.0"
+          }
+        },
+        "registry-auth-token": {
+          "version": "3.3.2",
+          "bundled": true,
+          "requires": {
+            "rc": "^1.1.6",
+            "safe-buffer": "^5.0.1"
+          }
+        },
+        "registry-url": {
+          "version": "3.1.0",
+          "bundled": true,
+          "requires": {
+            "rc": "^1.0.1"
+          }
+        },
+        "request": {
+          "version": "2.81.0",
+          "bundled": true,
+          "requires": {
+            "aws-sign2": "~0.6.0",
+            "aws4": "^1.2.1",
+            "caseless": "~0.12.0",
+            "combined-stream": "~1.0.5",
+            "extend": "~3.0.0",
+            "forever-agent": "~0.6.1",
+            "form-data": "~2.1.1",
+            "har-validator": "~4.2.1",
+            "hawk": "~3.1.3",
+            "http-signature": "~1.1.0",
+            "is-typedarray": "~1.0.0",
+            "isstream": "~0.1.2",
+            "json-stringify-safe": "~5.0.1",
+            "mime-types": "~2.1.7",
+            "oauth-sign": "~0.8.1",
+            "performance-now": "^0.2.0",
+            "qs": "~6.4.0",
+            "safe-buffer": "^5.0.1",
+            "stringstream": "~0.0.4",
+            "tough-cookie": "~2.3.0",
+            "tunnel-agent": "^0.6.0",
+            "uuid": "^3.0.0"
+          }
+        },
+        "require-directory": {
+          "version": "2.1.1",
+          "bundled": true
+        },
+        "require-main-filename": {
+          "version": "1.0.1",
+          "bundled": true
+        },
+        "resolve-from": {
+          "version": "4.0.0",
+          "bundled": true
+        },
+        "retry": {
+          "version": "0.12.0",
+          "bundled": true
+        },
+        "rimraf": {
+          "version": "2.6.2",
+          "bundled": true,
+          "requires": {
+            "glob": "^7.0.5"
+          }
+        },
+        "run-queue": {
+          "version": "1.0.3",
+          "bundled": true,
+          "requires": {
+            "aproba": "^1.1.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "bundled": true
+        },
+        "safer-buffer": {
+          "version": "2.1.2",
+          "bundled": true
+        },
+        "semver": {
+          "version": "5.5.0",
+          "bundled": true
+        },
+        "semver-diff": {
+          "version": "2.1.0",
+          "bundled": true,
+          "requires": {
+            "semver": "^5.0.3"
+          }
+        },
+        "set-blocking": {
+          "version": "2.0.0",
+          "bundled": true
+        },
+        "sha": {
+          "version": "2.0.1",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.2",
+            "readable-stream": "^2.0.2"
+          }
+        },
+        "shebang-command": {
+          "version": "1.2.0",
+          "bundled": true,
+          "requires": {
+            "shebang-regex": "^1.0.0"
+          }
+        },
+        "shebang-regex": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "signal-exit": {
+          "version": "3.0.2",
+          "bundled": true
+        },
+        "slash": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "slide": {
+          "version": "1.1.6",
+          "bundled": true
+        },
+        "smart-buffer": {
+          "version": "4.0.1",
+          "bundled": true
+        },
+        "sntp": {
+          "version": "1.0.9",
+          "bundled": true,
+          "requires": {
+            "hoek": "2.x.x"
+          }
+        },
+        "socks": {
+          "version": "2.2.0",
+          "bundled": true,
+          "requires": {
+            "ip": "^1.1.5",
+            "smart-buffer": "^4.0.1"
+          }
+        },
+        "socks-proxy-agent": {
+          "version": "4.0.1",
+          "bundled": true,
+          "requires": {
+            "agent-base": "~4.2.0",
+            "socks": "~2.2.0"
+          }
+        },
+        "sorted-object": {
+          "version": "2.0.1",
+          "bundled": true
+        },
+        "sorted-union-stream": {
+          "version": "2.1.3",
+          "bundled": true,
+          "requires": {
+            "from2": "^1.3.0",
+            "stream-iterate": "^1.1.0"
+          },
+          "dependencies": {
+            "from2": {
+              "version": "1.3.0",
+              "bundled": true,
+              "requires": {
+                "inherits": "~2.0.1",
+                "readable-stream": "~1.1.10"
+              }
+            },
+            "isarray": {
+              "version": "0.0.1",
+              "bundled": true
+            },
+            "readable-stream": {
+              "version": "1.1.14",
+              "bundled": true,
+              "requires": {
+                "core-util-is": "~1.0.0",
+                "inherits": "~2.0.1",
+                "isarray": "0.0.1",
+                "string_decoder": "~0.10.x"
+              }
+            },
+            "string_decoder": {
+              "version": "0.10.31",
+              "bundled": true
+            }
+          }
+        },
+        "spdx-correct": {
+          "version": "3.0.0",
+          "bundled": true,
+          "requires": {
+            "spdx-expression-parse": "^3.0.0",
+            "spdx-license-ids": "^3.0.0"
+          }
+        },
+        "spdx-exceptions": {
+          "version": "2.1.0",
+          "bundled": true
+        },
+        "spdx-expression-parse": {
+          "version": "3.0.0",
+          "bundled": true,
+          "requires": {
+            "spdx-exceptions": "^2.1.0",
+            "spdx-license-ids": "^3.0.0"
+          }
+        },
+        "spdx-license-ids": {
+          "version": "3.0.0",
+          "bundled": true
+        },
+        "sshpk": {
+          "version": "1.14.2",
+          "bundled": true,
+          "requires": {
+            "asn1": "~0.2.3",
+            "assert-plus": "^1.0.0",
+            "bcrypt-pbkdf": "^1.0.0",
+            "dashdash": "^1.12.0",
+            "ecc-jsbn": "~0.1.1",
+            "getpass": "^0.1.1",
+            "jsbn": "~0.1.0",
+            "safer-buffer": "^2.0.2",
+            "tweetnacl": "~0.14.0"
+          },
+          "dependencies": {
+            "assert-plus": {
+              "version": "1.0.0",
+              "bundled": true
+            }
+          }
+        },
+        "ssri": {
+          "version": "6.0.0",
+          "bundled": true
+        },
+        "stream-each": {
+          "version": "1.2.2",
+          "bundled": true,
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "stream-shift": "^1.0.0"
+          }
+        },
+        "stream-iterate": {
+          "version": "1.2.0",
+          "bundled": true,
+          "requires": {
+            "readable-stream": "^2.1.5",
+            "stream-shift": "^1.0.0"
+          }
+        },
+        "stream-shift": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "strict-uri-encode": {
+          "version": "2.0.0",
+          "bundled": true
+        },
+        "string-width": {
+          "version": "2.1.1",
+          "bundled": true,
+          "requires": {
+            "is-fullwidth-code-point": "^2.0.0",
+            "strip-ansi": "^4.0.0"
+          },
+          "dependencies": {
+            "ansi-regex": {
+              "version": "3.0.0",
+              "bundled": true
+            },
+            "is-fullwidth-code-point": {
+              "version": "2.0.0",
+              "bundled": true
+            },
+            "strip-ansi": {
+              "version": "4.0.0",
+              "bundled": true,
+              "requires": {
+                "ansi-regex": "^3.0.0"
+              }
+            }
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "bundled": true,
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        },
+        "stringify-package": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "stringstream": {
+          "version": "0.0.6",
+          "bundled": true
+        },
+        "strip-ansi": {
+          "version": "3.0.1",
+          "bundled": true,
+          "requires": {
+            "ansi-regex": "^2.0.0"
+          }
+        },
+        "strip-eof": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "strip-json-comments": {
+          "version": "2.0.1",
+          "bundled": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "bundled": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        },
+        "tar": {
+          "version": "4.4.4",
+          "bundled": true,
+          "requires": {
+            "chownr": "^1.0.1",
+            "fs-minipass": "^1.2.5",
+            "minipass": "^2.3.3",
+            "minizlib": "^1.1.0",
+            "mkdirp": "^0.5.0",
+            "safe-buffer": "^5.1.2",
+            "yallist": "^3.0.2"
+          },
+          "dependencies": {
+            "yallist": {
+              "version": "3.0.2",
+              "bundled": true
+            }
+          }
+        },
+        "term-size": {
+          "version": "1.2.0",
+          "bundled": true,
+          "requires": {
+            "execa": "^0.7.0"
+          }
+        },
+        "text-table": {
+          "version": "0.2.0",
+          "bundled": true
+        },
+        "through": {
+          "version": "2.3.8",
+          "bundled": true
+        },
+        "through2": {
+          "version": "2.0.3",
+          "bundled": true,
+          "requires": {
+            "readable-stream": "^2.1.5",
+            "xtend": "~4.0.1"
+          }
+        },
+        "timed-out": {
+          "version": "4.0.1",
+          "bundled": true
+        },
+        "tiny-relative-date": {
+          "version": "1.3.0",
+          "bundled": true
+        },
+        "tough-cookie": {
+          "version": "2.3.4",
+          "bundled": true,
+          "requires": {
+            "punycode": "^1.4.1"
+          }
+        },
+        "tunnel-agent": {
+          "version": "0.6.0",
+          "bundled": true,
+          "requires": {
+            "safe-buffer": "^5.0.1"
+          }
+        },
+        "tweetnacl": {
+          "version": "0.14.5",
+          "bundled": true,
+          "optional": true
+        },
+        "typedarray": {
+          "version": "0.0.6",
+          "bundled": true
+        },
+        "uid-number": {
+          "version": "0.0.6",
+          "bundled": true
+        },
+        "umask": {
+          "version": "1.1.0",
+          "bundled": true
+        },
+        "unique-filename": {
+          "version": "1.1.0",
+          "bundled": true,
+          "requires": {
+            "unique-slug": "^2.0.0"
+          }
+        },
+        "unique-slug": {
+          "version": "2.0.0",
+          "bundled": true,
+          "requires": {
+            "imurmurhash": "^0.1.4"
+          }
+        },
+        "unique-string": {
+          "version": "1.0.0",
+          "bundled": true,
+          "requires": {
+            "crypto-random-string": "^1.0.0"
+          }
+        },
+        "unpipe": {
+          "version": "1.0.0",
+          "bundled": true
+        },
+        "unzip-response": {
+          "version": "2.0.1",
+          "bundled": true
+        },
+        "update-notifier": {
+          "version": "2.5.0",
+          "bundled": true,
+          "requires": {
+            "boxen": "^1.2.1",
+            "chalk": "^2.0.1",
+            "configstore": "^3.0.0",
+            "import-lazy": "^2.1.0",
+            "is-ci": "^1.0.10",
+            "is-installed-globally": "^0.1.0",
+            "is-npm": "^1.0.0",
+            "latest-version": "^3.0.0",
+            "semver-diff": "^2.0.0",
+            "xdg-basedir": "^3.0.0"
+          }
+        },
+        "url-parse-lax": {
+          "version": "1.0.0",
+          "bundled": true,
+          "requires": {
+            "prepend-http": "^1.0.1"
+          }
+        },
+        "util-deprecate": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "util-extend": {
+          "version": "1.0.3",
+          "bundled": true
+        },
+        "uuid": {
+          "version": "3.3.2",
+          "bundled": true
+        },
+        "validate-npm-package-license": {
+          "version": "3.0.3",
+          "bundled": true,
+          "requires": {
+            "spdx-correct": "^3.0.0",
+            "spdx-expression-parse": "^3.0.0"
+          }
+        },
+        "validate-npm-package-name": {
+          "version": "3.0.0",
+          "bundled": true,
+          "requires": {
+            "builtins": "^1.0.3"
+          }
+        },
+        "verror": {
+          "version": "1.10.0",
+          "bundled": true,
+          "requires": {
+            "assert-plus": "^1.0.0",
+            "core-util-is": "1.0.2",
+            "extsprintf": "^1.2.0"
+          },
+          "dependencies": {
+            "assert-plus": {
+              "version": "1.0.0",
+              "bundled": true
+            }
+          }
+        },
+        "wcwidth": {
+          "version": "1.0.1",
+          "bundled": true,
+          "requires": {
+            "defaults": "^1.0.3"
+          }
+        },
+        "which": {
+          "version": "1.3.1",
+          "bundled": true,
+          "requires": {
+            "isexe": "^2.0.0"
+          }
+        },
+        "which-module": {
+          "version": "2.0.0",
+          "bundled": true
+        },
+        "wide-align": {
+          "version": "1.1.2",
+          "bundled": true,
+          "requires": {
+            "string-width": "^1.0.2"
+          },
+          "dependencies": {
+            "string-width": {
+              "version": "1.0.2",
+              "bundled": true,
+              "requires": {
+                "code-point-at": "^1.0.0",
+                "is-fullwidth-code-point": "^1.0.0",
+                "strip-ansi": "^3.0.0"
+              }
+            }
+          }
+        },
+        "widest-line": {
+          "version": "2.0.0",
+          "bundled": true,
+          "requires": {
+            "string-width": "^2.1.1"
+          }
+        },
+        "worker-farm": {
+          "version": "1.6.0",
+          "bundled": true,
+          "requires": {
+            "errno": "~0.1.7"
+          }
+        },
+        "wrap-ansi": {
+          "version": "2.1.0",
+          "bundled": true,
+          "requires": {
+            "string-width": "^1.0.1",
+            "strip-ansi": "^3.0.1"
+          },
+          "dependencies": {
+            "string-width": {
+              "version": "1.0.2",
+              "bundled": true,
+              "requires": {
+                "code-point-at": "^1.0.0",
+                "is-fullwidth-code-point": "^1.0.0",
+                "strip-ansi": "^3.0.0"
+              }
+            }
+          }
+        },
+        "wrappy": {
+          "version": "1.0.2",
+          "bundled": true
+        },
+        "write-file-atomic": {
+          "version": "2.3.0",
+          "bundled": true,
+          "requires": {
+            "graceful-fs": "^4.1.11",
+            "imurmurhash": "^0.1.4",
+            "signal-exit": "^3.0.2"
+          }
+        },
+        "xdg-basedir": {
+          "version": "3.0.0",
+          "bundled": true
+        },
+        "xtend": {
+          "version": "4.0.1",
+          "bundled": true
+        },
+        "y18n": {
+          "version": "4.0.0",
+          "bundled": true
+        },
+        "yallist": {
+          "version": "2.1.2",
+          "bundled": true
+        },
+        "yargs": {
+          "version": "11.0.0",
+          "bundled": true,
+          "requires": {
+            "cliui": "^4.0.0",
+            "decamelize": "^1.1.1",
+            "find-up": "^2.1.0",
+            "get-caller-file": "^1.0.1",
+            "os-locale": "^2.0.0",
+            "require-directory": "^2.1.1",
+            "require-main-filename": "^1.0.1",
+            "set-blocking": "^2.0.0",
+            "string-width": "^2.0.0",
+            "which-module": "^2.0.0",
+            "y18n": "^3.2.1",
+            "yargs-parser": "^9.0.2"
+          },
+          "dependencies": {
+            "y18n": {
+              "version": "3.2.1",
+              "bundled": true
+            }
+          }
+        },
+        "yargs-parser": {
+          "version": "9.0.2",
+          "bundled": true,
+          "requires": {
+            "camelcase": "^4.1.0"
+          }
+        }
+      }
+    },
+    "npm-run-path": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
+      "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=",
+      "dev": true,
+      "requires": {
+        "path-key": "^2.0.0"
+      }
+    },
+    "num2fraction": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/num2fraction/-/num2fraction-1.2.2.tgz",
+      "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=",
+      "dev": true
+    },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
+      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
+      "dev": true
+    },
+    "nvd3": {
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/nvd3/-/nvd3-1.8.6.tgz",
+      "integrity": "sha1-LT66dL8zNjtRAevx0JPFmlOuc8Q="
+    },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=",
+      "dev": true
+    },
+    "object-copy": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
+      "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=",
+      "dev": true,
+      "requires": {
+        "copy-descriptor": "^0.1.0",
+        "define-property": "^0.2.5",
+        "kind-of": "^3.0.3"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        }
+      }
+    },
+    "object-keys": {
+      "version": "1.0.12",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz",
+      "integrity": "sha512-FTMyFUm2wBcGHnH2eXmz7tC6IwlqQZ6mVZ+6dm6vZ4IQIHjs6FdNsQBuKGPuUUUY6NfJw2PshC08Tn6LzLDOag==",
+      "dev": true
+    },
+    "object-visit": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
+      "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=",
+      "dev": true,
+      "requires": {
+        "isobject": "^3.0.0"
+      }
+    },
+    "object.assign": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz",
+      "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==",
+      "dev": true,
+      "requires": {
+        "define-properties": "^1.1.2",
+        "function-bind": "^1.1.1",
+        "has-symbols": "^1.0.0",
+        "object-keys": "^1.0.11"
+      }
+    },
+    "object.entries": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.0.4.tgz",
+      "integrity": "sha1-G/mk3SKI9bM/Opk9JXZh8F0WGl8=",
+      "dev": true,
+      "requires": {
+        "define-properties": "^1.1.2",
+        "es-abstract": "^1.6.1",
+        "function-bind": "^1.1.0",
+        "has": "^1.0.1"
+      }
+    },
+    "object.pick": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz",
+      "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=",
+      "dev": true,
+      "requires": {
+        "isobject": "^3.0.1"
+      }
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "onetime": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
+      "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=",
+      "dev": true,
+      "requires": {
+        "mimic-fn": "^1.0.0"
+      }
+    },
+    "optimist": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
+      "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
+      "dev": true,
+      "requires": {
+        "minimist": "~0.0.1",
+        "wordwrap": "~0.0.2"
+      },
+      "dependencies": {
+        "wordwrap": {
+          "version": "0.0.3",
+          "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
+          "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
+          "dev": true
+        }
+      }
+    },
+    "optionator": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
+      "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=",
+      "dev": true,
+      "requires": {
+        "deep-is": "~0.1.3",
+        "fast-levenshtein": "~2.0.4",
+        "levn": "~0.3.0",
+        "prelude-ls": "~1.1.2",
+        "type-check": "~0.3.2",
+        "wordwrap": "~1.0.0"
+      }
+    },
+    "os-browserify": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz",
+      "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=",
+      "dev": true
+    },
+    "os-homedir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+      "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=",
+      "dev": true
+    },
+    "os-locale": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz",
+      "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==",
+      "dev": true,
+      "requires": {
+        "execa": "^0.7.0",
+        "lcid": "^1.0.0",
+        "mem": "^1.1.0"
+      }
+    },
+    "os-tmpdir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
+      "dev": true
+    },
+    "p-finally": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+      "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=",
+      "dev": true
+    },
+    "p-limit": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz",
+      "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==",
+      "dev": true,
+      "requires": {
+        "p-try": "^1.0.0"
+      }
+    },
+    "p-locate": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
+      "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
+      "dev": true,
+      "requires": {
+        "p-limit": "^1.1.0"
+      }
+    },
+    "p-try": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
+      "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=",
+      "dev": true
+    },
+    "pako": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz",
+      "integrity": "sha512-lQe48YPsMJAig+yngZ87Lus+NF+3mtu7DVOBu6b/gHO1YpKwIj5AWjZ/TOS7i46HD/UixzWb1zeWDZfGZ3iYcg==",
+      "dev": true
+    },
+    "parallel-transform": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz",
+      "integrity": "sha1-1BDwZbBdojCB/NEPKIVMKb2jOwY=",
+      "dev": true,
+      "requires": {
+        "cyclist": "~0.2.2",
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.1.5"
+      }
+    },
+    "parse-asn1": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz",
+      "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==",
+      "dev": true,
+      "requires": {
+        "asn1.js": "^4.0.0",
+        "browserify-aes": "^1.0.0",
+        "create-hash": "^1.1.0",
+        "evp_bytestokey": "^1.0.0",
+        "pbkdf2": "^3.0.3"
+      }
+    },
+    "parse-json": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
+      "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=",
+      "dev": true,
+      "requires": {
+        "error-ex": "^1.2.0"
+      }
+    },
+    "pascalcase": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz",
+      "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=",
+      "dev": true
+    },
+    "path-browserify": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz",
+      "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo=",
+      "dev": true
+    },
+    "path-dirname": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz",
+      "integrity": "sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=",
+      "dev": true
+    },
+    "path-exists": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
+      "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
+      "dev": true
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true
+    },
+    "path-is-inside": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz",
+      "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=",
+      "dev": true
+    },
+    "path-key": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
+      "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=",
+      "dev": true
+    },
+    "path-parse": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz",
+      "integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME=",
+      "dev": true
+    },
+    "path-type": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
+      "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
+      "dev": true,
+      "requires": {
+        "pify": "^3.0.0"
+      }
+    },
+    "pbkdf2": {
+      "version": "3.0.16",
+      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.16.tgz",
+      "integrity": "sha512-y4CXP3thSxqf7c0qmOF+9UeOTrifiVTIM+u7NWlq+PRsHbr7r7dpCmvzrZxa96JJUNi0Y5w9VqG5ZNeCVMoDcA==",
+      "dev": true,
+      "requires": {
+        "create-hash": "^1.1.2",
+        "create-hmac": "^1.1.4",
+        "ripemd160": "^2.0.1",
+        "safe-buffer": "^5.0.1",
+        "sha.js": "^2.4.8"
+      }
+    },
+    "pify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
+      "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
+      "dev": true
+    },
+    "pinkie": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz",
+      "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=",
+      "dev": true
+    },
+    "pinkie-promise": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
+      "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=",
+      "dev": true,
+      "requires": {
+        "pinkie": "^2.0.0"
+      }
+    },
+    "pkg-dir": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz",
+      "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=",
+      "dev": true,
+      "requires": {
+        "find-up": "^2.1.0"
+      }
+    },
+    "pluralize": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-7.0.0.tgz",
+      "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==",
+      "dev": true
+    },
+    "posix-character-classes": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
+      "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=",
+      "dev": true
+    },
+    "postcss": {
+      "version": "6.0.23",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.23.tgz",
+      "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==",
+      "dev": true,
+      "requires": {
+        "chalk": "^2.4.1",
+        "source-map": "^0.6.1",
+        "supports-color": "^5.4.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "postcss-calc": {
+      "version": "5.3.1",
+      "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-5.3.1.tgz",
+      "integrity": "sha1-d7rnypKK2FcW4v2kLyYb98HWW14=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.2",
+        "postcss-message-helpers": "^2.0.0",
+        "reduce-css-calc": "^1.2.6"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-colormin": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-2.2.2.tgz",
+      "integrity": "sha1-ZjFBfV8OkJo9fsJrJMio0eT5bks=",
+      "dev": true,
+      "requires": {
+        "colormin": "^1.0.5",
+        "postcss": "^5.0.13",
+        "postcss-value-parser": "^3.2.3"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-convert-values": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-2.6.1.tgz",
+      "integrity": "sha1-u9hZPFwf0uPRwyK7kl3K6Nrk1i0=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.11",
+        "postcss-value-parser": "^3.1.2"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-discard-comments": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-2.0.4.tgz",
+      "integrity": "sha1-vv6J+v1bPazlzM5Rt2uBUUvgDj0=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.14"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-discard-duplicates": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-2.1.0.tgz",
+      "integrity": "sha1-uavye4isGIFYpesSq8riAmO5GTI=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.4"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-discard-empty": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-2.1.0.tgz",
+      "integrity": "sha1-0rS9nVztXr2Nyt52QMfXzX9PkrU=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.14"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-discard-overridden": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-0.1.1.tgz",
+      "integrity": "sha1-ix6vVU9ob7KIzYdMVWZ7CqNmjVg=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.16"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-discard-unused": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/postcss-discard-unused/-/postcss-discard-unused-2.2.3.tgz",
+      "integrity": "sha1-vOMLLMWR/8Y0Mitfs0ZLbZNPRDM=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.14",
+        "uniqs": "^2.0.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-filter-plugins": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/postcss-filter-plugins/-/postcss-filter-plugins-2.0.3.tgz",
+      "integrity": "sha512-T53GVFsdinJhgwm7rg1BzbeBRomOg9y5MBVhGcsV0CxurUdVj1UlPdKtn7aqYA/c/QVkzKMjq2bSV5dKG5+AwQ==",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.4"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-merge-idents": {
+      "version": "2.1.7",
+      "resolved": "https://registry.npmjs.org/postcss-merge-idents/-/postcss-merge-idents-2.1.7.tgz",
+      "integrity": "sha1-TFUwMTwI4dWzu/PSu8dH4njuonA=",
+      "dev": true,
+      "requires": {
+        "has": "^1.0.1",
+        "postcss": "^5.0.10",
+        "postcss-value-parser": "^3.1.1"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-merge-longhand": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-2.0.2.tgz",
+      "integrity": "sha1-I9kM0Sewp3mUkVMyc5A0oaTz1lg=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.4"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-merge-rules": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-2.1.2.tgz",
+      "integrity": "sha1-0d9d+qexrMO+VT8OnhDofGG19yE=",
+      "dev": true,
+      "requires": {
+        "browserslist": "^1.5.2",
+        "caniuse-api": "^1.5.2",
+        "postcss": "^5.0.4",
+        "postcss-selector-parser": "^2.2.2",
+        "vendors": "^1.0.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-message-helpers": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/postcss-message-helpers/-/postcss-message-helpers-2.0.0.tgz",
+      "integrity": "sha1-pPL0+rbk/gAvCu0ABHjN9S+bpg4=",
+      "dev": true
+    },
+    "postcss-minify-font-values": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-1.0.5.tgz",
+      "integrity": "sha1-S1jttWZB66fIR0qzUmyv17vey2k=",
+      "dev": true,
+      "requires": {
+        "object-assign": "^4.0.1",
+        "postcss": "^5.0.4",
+        "postcss-value-parser": "^3.0.2"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-minify-gradients": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-1.0.5.tgz",
+      "integrity": "sha1-Xb2hE3NwP4PPtKPqOIHY11/15uE=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.12",
+        "postcss-value-parser": "^3.3.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-minify-params": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-1.2.2.tgz",
+      "integrity": "sha1-rSzgcTc7lDs9kwo/pZo1jCjW8fM=",
+      "dev": true,
+      "requires": {
+        "alphanum-sort": "^1.0.1",
+        "postcss": "^5.0.2",
+        "postcss-value-parser": "^3.0.2",
+        "uniqs": "^2.0.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-minify-selectors": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-2.1.1.tgz",
+      "integrity": "sha1-ssapjAByz5G5MtGkllCBFDEXNb8=",
+      "dev": true,
+      "requires": {
+        "alphanum-sort": "^1.0.2",
+        "has": "^1.0.1",
+        "postcss": "^5.0.14",
+        "postcss-selector-parser": "^2.0.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-modules-extract-imports": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-1.1.0.tgz",
+      "integrity": "sha1-thTJcgvmgW6u41+zpfqh26agXds=",
+      "dev": true,
+      "requires": {
+        "postcss": "^6.0.1"
+      }
+    },
+    "postcss-modules-local-by-default": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-1.2.0.tgz",
+      "integrity": "sha1-99gMOYxaOT+nlkRmvRlQCn1hwGk=",
+      "dev": true,
+      "requires": {
+        "css-selector-tokenizer": "^0.7.0",
+        "postcss": "^6.0.1"
+      }
+    },
+    "postcss-modules-resolve-imports": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/postcss-modules-resolve-imports/-/postcss-modules-resolve-imports-1.3.0.tgz",
+      "integrity": "sha1-OY0wALla6WlCDN9M2D+oBn8cXq4=",
+      "dev": true,
+      "requires": {
+        "css-selector-tokenizer": "^0.7.0",
+        "icss-utils": "^3.0.1",
+        "minimist": "^1.2.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
+          "dev": true
+        }
+      }
+    },
+    "postcss-modules-scope": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-1.1.0.tgz",
+      "integrity": "sha1-1upkmUx5+XtipytCb75gVqGUu5A=",
+      "dev": true,
+      "requires": {
+        "css-selector-tokenizer": "^0.7.0",
+        "postcss": "^6.0.1"
+      }
+    },
+    "postcss-modules-values": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-1.3.0.tgz",
+      "integrity": "sha1-7P+p1+GSUYOJ9CrQ6D9yrsRW6iA=",
+      "dev": true,
+      "requires": {
+        "icss-replace-symbols": "^1.1.0",
+        "postcss": "^6.0.1"
+      }
+    },
+    "postcss-normalize-charset": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-1.1.1.tgz",
+      "integrity": "sha1-757nEhLX/nWceO0WL2HtYrXLk/E=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.5"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-normalize-url": {
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-3.0.8.tgz",
+      "integrity": "sha1-EI90s/L82viRov+j6kWSJ5/HgiI=",
+      "dev": true,
+      "requires": {
+        "is-absolute-url": "^2.0.0",
+        "normalize-url": "^1.4.0",
+        "postcss": "^5.0.14",
+        "postcss-value-parser": "^3.2.3"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-ordered-values": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-2.2.3.tgz",
+      "integrity": "sha1-7sbCpntsQSqNsgQud/6NpD+VwR0=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.4",
+        "postcss-value-parser": "^3.0.1"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-reduce-idents": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/postcss-reduce-idents/-/postcss-reduce-idents-2.4.0.tgz",
+      "integrity": "sha1-wsbSDMlYKE9qv75j92Cb9AkFmtM=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.4",
+        "postcss-value-parser": "^3.0.2"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-reduce-initial": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/postcss-reduce-initial/-/postcss-reduce-initial-1.0.1.tgz",
+      "integrity": "sha1-aPgGlfBF0IJjqHmtJA343WT2ROo=",
+      "dev": true,
+      "requires": {
+        "postcss": "^5.0.4"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-reduce-transforms": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-1.0.4.tgz",
+      "integrity": "sha1-/3b02CEkN7McKYpC0uFEQCV3GuE=",
+      "dev": true,
+      "requires": {
+        "has": "^1.0.1",
+        "postcss": "^5.0.8",
+        "postcss-value-parser": "^3.0.1"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-selector-parser": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-2.2.3.tgz",
+      "integrity": "sha1-+UN3iGBsPJrO4W/+jYsWKX8nu5A=",
+      "dev": true,
+      "requires": {
+        "flatten": "^1.0.2",
+        "indexes-of": "^1.0.1",
+        "uniq": "^1.0.1"
+      }
+    },
+    "postcss-svgo": {
+      "version": "2.1.6",
+      "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-2.1.6.tgz",
+      "integrity": "sha1-tt8YqmE7Zm4TPwittSGcJoSsEI0=",
+      "dev": true,
+      "requires": {
+        "is-svg": "^2.0.0",
+        "postcss": "^5.0.14",
+        "postcss-value-parser": "^3.2.3",
+        "svgo": "^0.7.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-unique-selectors": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-2.0.2.tgz",
+      "integrity": "sha1-mB1X0p3csz57Hf4f1DuGSfkzyh0=",
+      "dev": true,
+      "requires": {
+        "alphanum-sort": "^1.0.1",
+        "postcss": "^5.0.4",
+        "uniqs": "^2.0.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "postcss-value-parser": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.0.tgz",
+      "integrity": "sha1-h/OPnxj3dKSrTIojL1xc6IcqnRU=",
+      "dev": true
+    },
+    "postcss-zindex": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-2.2.0.tgz",
+      "integrity": "sha1-0hCd3AVbka9n/EyzsCWUZjnSryI=",
+      "dev": true,
+      "requires": {
+        "has": "^1.0.1",
+        "postcss": "^5.0.4",
+        "uniqs": "^2.0.0"
+      },
+      "dependencies": {
+        "postcss": {
+          "version": "5.2.18",
+          "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
+          "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
+          "dev": true,
+          "requires": {
+            "chalk": "^1.1.3",
+            "js-base64": "^2.1.9",
+            "source-map": "^0.5.6",
+            "supports-color": "^3.2.3"
+          }
+        },
+        "supports-color": {
+          "version": "3.2.3",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
+          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
+          "dev": true,
+          "requires": {
+            "has-flag": "^1.0.0"
+          }
+        }
+      }
+    },
+    "prelude-ls": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
+      "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
+      "dev": true
+    },
+    "prepend-http": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz",
+      "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=",
+      "dev": true
+    },
+    "private": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz",
+      "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==",
+      "dev": true
+    },
+    "process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=",
+      "dev": true
+    },
+    "process-nextick-args": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+      "dev": true
+    },
+    "progress": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz",
+      "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=",
+      "dev": true
+    },
+    "promise-inflight": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
+      "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=",
+      "dev": true
+    },
+    "prr": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
+      "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
+      "dev": true
+    },
+    "pseudomap": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=",
+      "dev": true
+    },
+    "public-encrypt": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.2.tgz",
+      "integrity": "sha512-4kJ5Esocg8X3h8YgJsKAuoesBgB7mqH3eowiDzMUPKiRDDE7E/BqqZD1hnTByIaAFiwAw246YEltSq7tdrOH0Q==",
+      "dev": true,
+      "requires": {
+        "bn.js": "^4.1.0",
+        "browserify-rsa": "^4.0.0",
+        "create-hash": "^1.1.0",
+        "parse-asn1": "^5.0.0",
+        "randombytes": "^2.0.1"
+      }
+    },
+    "pump": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
+      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+      "dev": true,
+      "requires": {
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.1"
+      }
+    },
+    "pumpify": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz",
+      "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==",
+      "dev": true,
+      "requires": {
+        "duplexify": "^3.6.0",
+        "inherits": "^2.0.3",
+        "pump": "^2.0.0"
+      }
+    },
+    "punycode": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+      "dev": true
+    },
+    "q": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
+      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=",
+      "dev": true
+    },
+    "query-string": {
+      "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz",
+      "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=",
+      "dev": true,
+      "requires": {
+        "object-assign": "^4.1.0",
+        "strict-uri-encode": "^1.0.0"
+      }
+    },
+    "querystring": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
+      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
+      "dev": true
+    },
+    "querystring-es3": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz",
+      "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=",
+      "dev": true
+    },
+    "randombytes": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.0.6.tgz",
+      "integrity": "sha512-CIQ5OFxf4Jou6uOKe9t1AOgqpeU5fd70A8NPdHSGeYXqXsPe6peOwI0cUl88RWZ6sP1vPMV3avd/R6cZ5/sP1A==",
+      "dev": true,
+      "requires": {
+        "safe-buffer": "^5.1.0"
+      }
+    },
+    "randomfill": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz",
+      "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==",
+      "dev": true,
+      "requires": {
+        "randombytes": "^2.0.5",
+        "safe-buffer": "^5.1.0"
+      }
+    },
+    "read-pkg": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz",
+      "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=",
+      "dev": true,
+      "requires": {
+        "load-json-file": "^2.0.0",
+        "normalize-package-data": "^2.3.2",
+        "path-type": "^2.0.0"
+      },
+      "dependencies": {
+        "path-type": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz",
+          "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=",
+          "dev": true,
+          "requires": {
+            "pify": "^2.0.0"
+          }
+        },
+        "pify": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "dev": true
+        }
+      }
+    },
+    "read-pkg-up": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz",
+      "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=",
+      "dev": true,
+      "requires": {
+        "find-up": "^2.0.0",
+        "read-pkg": "^2.0.0"
+      }
+    },
+    "readable-stream": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+      "dev": true,
+      "requires": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "readdirp": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz",
+      "integrity": "sha1-TtCtBg3zBzMAxIRANz9y0cxkLXg=",
+      "dev": true,
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "minimatch": "^3.0.2",
+        "readable-stream": "^2.0.2",
+        "set-immediate-shim": "^1.0.1"
+      }
+    },
+    "reduce-css-calc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/reduce-css-calc/-/reduce-css-calc-1.3.0.tgz",
+      "integrity": "sha1-dHyRTgSWFKTJz7umKYca0dKSdxY=",
+      "dev": true,
+      "requires": {
+        "balanced-match": "^0.4.2",
+        "math-expression-evaluator": "^1.2.14",
+        "reduce-function-call": "^1.0.1"
+      },
+      "dependencies": {
+        "balanced-match": {
+          "version": "0.4.2",
+          "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz",
+          "integrity": "sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg=",
+          "dev": true
+        }
+      }
+    },
+    "reduce-function-call": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/reduce-function-call/-/reduce-function-call-1.0.2.tgz",
+      "integrity": "sha1-WiAL+S4ON3UXUv5FsKszD9S2vpk=",
+      "dev": true,
+      "requires": {
+        "balanced-match": "^0.4.2"
+      },
+      "dependencies": {
+        "balanced-match": {
+          "version": "0.4.2",
+          "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz",
+          "integrity": "sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg=",
+          "dev": true
+        }
+      }
+    },
+    "regenerate": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.0.tgz",
+      "integrity": "sha512-1G6jJVDWrt0rK99kBjvEtziZNCICAuvIPkSiUFIQxVP06RCVpq3dmDo2oi6ABpYaDYaTRr67BEhL8r1wgEZZKg==",
+      "dev": true
+    },
+    "regenerator-runtime": {
+      "version": "0.11.1",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
+      "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==",
+      "dev": true
+    },
+    "regex-not": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
+      "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==",
+      "dev": true,
+      "requires": {
+        "extend-shallow": "^3.0.2",
+        "safe-regex": "^1.1.0"
+      }
+    },
+    "regexpp": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-1.1.0.tgz",
+      "integrity": "sha512-LOPw8FpgdQF9etWMaAfG/WRthIdXJGYp4mJ2Jgn/2lpkbod9jPn0t9UqN7AxBOKNfzRbYyVfgc7Vk4t/MpnXgw==",
+      "dev": true
+    },
+    "regexpu-core": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz",
+      "integrity": "sha1-hqdj9Y7k18L2sQLkdkBQ3n7ZDGs=",
+      "dev": true,
+      "requires": {
+        "regenerate": "^1.2.1",
+        "regjsgen": "^0.2.0",
+        "regjsparser": "^0.1.4"
+      }
+    },
+    "regjsgen": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz",
+      "integrity": "sha1-bAFq3qxVT3WCP+N6wFuS1aTtsfc=",
+      "dev": true
+    },
+    "regjsparser": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz",
+      "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=",
+      "dev": true,
+      "requires": {
+        "jsesc": "~0.5.0"
+      },
+      "dependencies": {
+        "jsesc": {
+          "version": "0.5.0",
+          "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz",
+          "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=",
+          "dev": true
+        }
+      }
+    },
+    "remove-trailing-separator": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz",
+      "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=",
+      "dev": true
+    },
+    "repeat-element": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.2.tgz",
+      "integrity": "sha1-7wiaF40Ug7quTZPrmLT55OEdmQo=",
+      "dev": true
+    },
+    "repeat-string": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
+      "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=",
+      "dev": true
+    },
+    "repeating": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz",
+      "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=",
+      "dev": true,
+      "requires": {
+        "is-finite": "^1.0.0"
+      }
+    },
+    "require-directory": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
+      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
+      "dev": true
+    },
+    "require-main-filename": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz",
+      "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=",
+      "dev": true
+    },
+    "require-uncached": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz",
+      "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=",
+      "dev": true,
+      "requires": {
+        "caller-path": "^0.1.0",
+        "resolve-from": "^1.0.0"
+      },
+      "dependencies": {
+        "resolve-from": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz",
+          "integrity": "sha1-Jsv+k10a7uq7Kbw/5a6wHpPUQiY=",
+          "dev": true
+        }
+      }
+    },
+    "resolve": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz",
+      "integrity": "sha512-AicPrAC7Qu1JxPCZ9ZgCZlY35QgFnNqc+0LtbRNxnVw4TXvjQ72wnuL9JQcEBgXkI9JM8MsT9kaQoHcpCRJOYA==",
+      "dev": true,
+      "requires": {
+        "path-parse": "^1.0.5"
+      }
+    },
+    "resolve-cwd": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz",
+      "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=",
+      "dev": true,
+      "requires": {
+        "resolve-from": "^3.0.0"
+      }
+    },
+    "resolve-from": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
+      "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=",
+      "dev": true
+    },
+    "resolve-url": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
+      "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=",
+      "dev": true
+    },
+    "restore-cursor": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz",
+      "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=",
+      "dev": true,
+      "requires": {
+        "onetime": "^2.0.0",
+        "signal-exit": "^3.0.2"
+      }
+    },
+    "ret": {
+      "version": "0.1.15",
+      "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
+      "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==",
+      "dev": true
+    },
+    "right-align": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz",
+      "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "align-text": "^0.1.1"
+      }
+    },
+    "rimraf": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz",
+      "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==",
+      "dev": true,
+      "requires": {
+        "glob": "^7.0.5"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "7.1.2",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
+          "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
+          "dev": true,
+          "requires": {
+            "fs.realpath": "^1.0.0",
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "^3.0.4",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        }
+      }
+    },
+    "ripemd160": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz",
+      "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==",
+      "dev": true,
+      "requires": {
+        "hash-base": "^3.0.0",
+        "inherits": "^2.0.1"
+      }
+    },
+    "run-async": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz",
+      "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=",
+      "dev": true,
+      "requires": {
+        "is-promise": "^2.1.0"
+      }
+    },
+    "run-queue": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",
+      "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=",
+      "dev": true,
+      "requires": {
+        "aproba": "^1.1.1"
+      }
+    },
+    "rx-lite": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz",
+      "integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ=",
+      "dev": true
+    },
+    "rx-lite-aggregates": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz",
+      "integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=",
+      "dev": true,
+      "requires": {
+        "rx-lite": "*"
+      }
+    },
+    "rxjs": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.2.2.tgz",
+      "integrity": "sha512-0MI8+mkKAXZUF9vMrEoPnaoHkfzBPP4IGwUYRJhIRJF6/w3uByO1e91bEHn8zd43RdkTMKiooYKmwz7RH6zfOQ==",
+      "dev": true,
+      "requires": {
+        "tslib": "^1.9.0"
+      }
+    },
+    "safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "safe-regex": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
+      "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=",
+      "dev": true,
+      "requires": {
+        "ret": "~0.1.10"
+      }
+    },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "dev": true
+    },
+    "sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==",
+      "dev": true
+    },
+    "schema-utils": {
+      "version": "0.4.5",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.5.tgz",
+      "integrity": "sha512-yYrjb9TX2k/J1Y5UNy3KYdZq10xhYcF8nMpAW6o3hy6Q8WSIEf9lJHG/ePnOBfziPM3fvQwfOwa13U/Fh8qTfA==",
+      "dev": true,
+      "requires": {
+        "ajv": "^6.1.0",
+        "ajv-keywords": "^3.1.0"
+      }
+    },
+    "seekout": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/seekout/-/seekout-1.0.2.tgz",
+      "integrity": "sha1-CbqfG9W0b7sTRxjrGaaDgsuxuck=",
+      "dev": true
+    },
+    "semver": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz",
+      "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==",
+      "dev": true
+    },
+    "serialize-javascript": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.5.0.tgz",
+      "integrity": "sha512-Ga8c8NjAAp46Br4+0oZ2WxJCwIzwP60Gq1YPgU+39PiTVxyed/iKE/zyZI6+UlVYH5Q4PaQdHhcegIFPZTUfoQ==",
+      "dev": true
+    },
+    "set-blocking": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
+      "dev": true
+    },
+    "set-immediate-shim": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz",
+      "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=",
+      "dev": true
+    },
+    "set-value": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
+      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "dev": true,
+      "requires": {
+        "extend-shallow": "^2.0.1",
+        "is-extendable": "^0.1.1",
+        "is-plain-object": "^2.0.3",
+        "split-string": "^3.0.1"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "setimmediate": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
+      "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=",
+      "dev": true
+    },
+    "sha.js": {
+      "version": "2.4.11",
+      "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz",
+      "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==",
+      "dev": true,
+      "requires": {
+        "inherits": "^2.0.1",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "shebang-command": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
+      "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
+      "dev": true,
+      "requires": {
+        "shebang-regex": "^1.0.0"
+      }
+    },
+    "shebang-regex": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
+      "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=",
+      "dev": true
+    },
+    "signal-exit": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
+      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
+      "dev": true
+    },
+    "slash": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
+      "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=",
+      "dev": true
+    },
+    "slice-ansi": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz",
+      "integrity": "sha512-POqxBK6Lb3q6s047D/XsDVNPnF9Dl8JSaqe9h9lURl0OdNqy/ujDrOiIHtsqXMGbWWTIomRzAMaTyawAU//Reg==",
+      "dev": true,
+      "requires": {
+        "is-fullwidth-code-point": "^2.0.0"
+      }
+    },
+    "snapdragon": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz",
+      "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==",
+      "dev": true,
+      "requires": {
+        "base": "^0.11.1",
+        "debug": "^2.2.0",
+        "define-property": "^0.2.5",
+        "extend-shallow": "^2.0.1",
+        "map-cache": "^0.2.2",
+        "source-map": "^0.5.6",
+        "source-map-resolve": "^0.5.0",
+        "use": "^3.1.0"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        },
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        }
+      }
+    },
+    "snapdragon-node": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz",
+      "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==",
+      "dev": true,
+      "requires": {
+        "define-property": "^1.0.0",
+        "isobject": "^3.0.0",
+        "snapdragon-util": "^3.0.1"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
+          "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^1.0.0"
+          }
+        },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "dev": true,
+          "requires": {
+            "kind-of": "^6.0.0"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "^1.0.0",
+            "is-data-descriptor": "^1.0.0",
+            "kind-of": "^6.0.2"
+          }
+        },
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
+          "dev": true
+        }
+      }
+    },
+    "snapdragon-util": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz",
+      "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==",
+      "dev": true,
+      "requires": {
+        "kind-of": "^3.2.0"
+      }
+    },
+    "sort-keys": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz",
+      "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=",
+      "dev": true,
+      "requires": {
+        "is-plain-obj": "^1.0.0"
+      }
+    },
+    "source-list-map": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.0.tgz",
+      "integrity": "sha512-I2UmuJSRr/T8jisiROLU3A3ltr+swpniSmNPI4Ml3ZCX6tVnDsuZzK7F2hl5jTqbZBWCEKlj5HRQiPExXLgE8A==",
+      "dev": true
+    },
+    "source-map": {
+      "version": "0.5.7",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
+      "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=",
+      "dev": true
+    },
+    "source-map-resolve": {
+      "version": "0.5.2",
+      "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz",
+      "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==",
+      "dev": true,
+      "requires": {
+        "atob": "^2.1.1",
+        "decode-uri-component": "^0.2.0",
+        "resolve-url": "^0.2.1",
+        "source-map-url": "^0.4.0",
+        "urix": "^0.1.0"
+      }
+    },
+    "source-map-support": {
+      "version": "0.4.18",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.4.18.tgz",
+      "integrity": "sha512-try0/JqxPLF9nOjvSta7tVondkP5dwgyLDjVoyMDlmjugT2lRZ1OfsrYTkCd2hkDnJTKRbO/Rl3orm8vlsUzbA==",
+      "dev": true,
+      "requires": {
+        "source-map": "^0.5.6"
+      }
+    },
+    "source-map-url": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",
+      "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=",
+      "dev": true
+    },
+    "spdx-correct": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.0.tgz",
+      "integrity": "sha512-N19o9z5cEyc8yQQPukRCZ9EUmb4HUpnrmaL/fxS2pBo2jbfcFRVuFZ/oFC+vZz0MNNk0h80iMn5/S6qGZOL5+g==",
+      "dev": true,
+      "requires": {
+        "spdx-expression-parse": "^3.0.0",
+        "spdx-license-ids": "^3.0.0"
+      }
+    },
+    "spdx-exceptions": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz",
+      "integrity": "sha512-4K1NsmrlCU1JJgUrtgEeTVyfx8VaYea9J9LvARxhbHtVtohPs/gFGG5yy49beySjlIMhhXZ4QqujIZEfS4l6Cg==",
+      "dev": true
+    },
+    "spdx-expression-parse": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
+      "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
+      "dev": true,
+      "requires": {
+        "spdx-exceptions": "^2.1.0",
+        "spdx-license-ids": "^3.0.0"
+      }
+    },
+    "spdx-license-ids": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.0.tgz",
+      "integrity": "sha512-2+EPwgbnmOIl8HjGBXXMd9NAu02vLjOO1nWw4kmeRDFyHn+M/ETfHxQUK0oXg8ctgVnl9t3rosNVsZ1jG61nDA==",
+      "dev": true
+    },
+    "split-string": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
+      "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==",
+      "dev": true,
+      "requires": {
+        "extend-shallow": "^3.0.0"
+      }
+    },
+    "sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+      "dev": true
+    },
+    "ssri": {
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/ssri/-/ssri-5.3.0.tgz",
+      "integrity": "sha512-XRSIPqLij52MtgoQavH/x/dU1qVKtWUAAZeOHsR9c2Ddi4XerFy3mc1alf+dLJKl9EUIm/Ht+EowFkTUOA6GAQ==",
+      "dev": true,
+      "requires": {
+        "safe-buffer": "^5.1.1"
+      }
+    },
+    "static-extend": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
+      "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=",
+      "dev": true,
+      "requires": {
+        "define-property": "^0.2.5",
+        "object-copy": "^0.1.0"
+      },
+      "dependencies": {
+        "define-property": {
+          "version": "0.2.5",
+          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
+          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+          "dev": true,
+          "requires": {
+            "is-descriptor": "^0.1.0"
+          }
+        }
+      }
+    },
+    "stream-browserify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
+      "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=",
+      "dev": true,
+      "requires": {
+        "inherits": "~2.0.1",
+        "readable-stream": "^2.0.2"
+      }
+    },
+    "stream-each": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.2.tgz",
+      "integrity": "sha512-mc1dbFhGBxvTM3bIWmAAINbqiuAk9TATcfIQC8P+/+HJefgaiTlMn2dHvkX8qlI12KeYKSQ1Ua9RrIqrn1VPoA==",
+      "dev": true,
+      "requires": {
+        "end-of-stream": "^1.1.0",
+        "stream-shift": "^1.0.0"
+      }
+    },
+    "stream-http": {
+      "version": "2.8.3",
+      "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz",
+      "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==",
+      "dev": true,
+      "requires": {
+        "builtin-status-codes": "^3.0.0",
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.3.6",
+        "to-arraybuffer": "^1.0.0",
+        "xtend": "^4.0.0"
+      }
+    },
+    "stream-shift": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
+      "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=",
+      "dev": true
+    },
+    "strict-uri-encode": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
+      "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=",
+      "dev": true
+    },
+    "string-width": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
+      "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
+      "dev": true,
+      "requires": {
+        "is-fullwidth-code-point": "^2.0.0",
+        "strip-ansi": "^4.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        }
+      }
+    },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "requires": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "strip-ansi": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+      "dev": true,
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
+    "strip-bom": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
+      "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+      "dev": true
+    },
+    "strip-eof": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
+      "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
+      "dev": true
+    },
+    "strip-json-comments": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
+      "dev": true
+    },
+    "style-loader": {
+      "version": "0.21.0",
+      "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-0.21.0.tgz",
+      "integrity": "sha512-T+UNsAcl3Yg+BsPKs1vd22Fr8sVT+CJMtzqc6LEw9bbJZb43lm9GoeIfUcDEefBSWC0BhYbcdupV1GtI4DGzxg==",
+      "dev": true,
+      "requires": {
+        "loader-utils": "^1.1.0",
+        "schema-utils": "^0.4.5"
+      }
+    },
+    "supports-color": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+      "dev": true
+    },
+    "svgo": {
+      "version": "0.7.2",
+      "resolved": "https://registry.npmjs.org/svgo/-/svgo-0.7.2.tgz",
+      "integrity": "sha1-n1dyQTlSE1xv779Ar+ak+qiLS7U=",
+      "dev": true,
+      "requires": {
+        "coa": "~1.0.1",
+        "colors": "~1.1.2",
+        "csso": "~2.3.1",
+        "js-yaml": "~3.7.0",
+        "mkdirp": "~0.5.1",
+        "sax": "~1.2.1",
+        "whet.extend": "~0.9.9"
+      },
+      "dependencies": {
+        "js-yaml": {
+          "version": "3.7.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
+          "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+          "dev": true,
+          "requires": {
+            "argparse": "^1.0.7",
+            "esprima": "^2.6.0"
+          }
+        }
+      }
+    },
+    "table": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/table/-/table-4.0.2.tgz",
+      "integrity": "sha512-UUkEAPdSGxtRpiV9ozJ5cMTtYiqz7Ni1OGqLXRCynrvzdtR1p+cfOWe2RJLwvUG8hNanaSRjecIqwOjqeatDsA==",
+      "dev": true,
+      "requires": {
+        "ajv": "^5.2.3",
+        "ajv-keywords": "^2.1.0",
+        "chalk": "^2.1.0",
+        "lodash": "^4.17.4",
+        "slice-ansi": "1.0.0",
+        "string-width": "^2.1.1"
+      },
+      "dependencies": {
+        "ajv": {
+          "version": "5.5.2",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
+          "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
+          "dev": true,
+          "requires": {
+            "co": "^4.6.0",
+            "fast-deep-equal": "^1.0.0",
+            "fast-json-stable-stringify": "^2.0.0",
+            "json-schema-traverse": "^0.3.0"
+          }
+        },
+        "ajv-keywords": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-2.1.1.tgz",
+          "integrity": "sha1-YXmX/F9gV2iUxDX5QNgZ4TW4B2I=",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "fast-deep-equal": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
+          "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "json-schema-traverse": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
+          "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "tapable": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.0.0.tgz",
+      "integrity": "sha512-dQRhbNQkRnaqauC7WqSJ21EEksgT0fYZX2lqXzGkpo8JNig9zGZTYoMGvyI2nWmXlE2VSVXVDu7wLVGu/mQEsg==",
+      "dev": true
+    },
+    "text-table": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+      "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
+      "dev": true
+    },
+    "through": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
+      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
+      "dev": true
+    },
+    "through2": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.3.tgz",
+      "integrity": "sha1-AARWmzfHx0ujnEPzzteNGtlBQL4=",
+      "dev": true,
+      "requires": {
+        "readable-stream": "^2.1.5",
+        "xtend": "~4.0.1"
+      }
+    },
+    "timers-browserify": {
+      "version": "2.0.10",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.10.tgz",
+      "integrity": "sha512-YvC1SV1XdOUaL6gx5CoGroT3Gu49pK9+TZ38ErPldOWW4j49GI1HKs9DV+KGq/w6y+LZ72W1c8cKz2vzY+qpzg==",
+      "dev": true,
+      "requires": {
+        "setimmediate": "^1.0.4"
+      }
+    },
+    "tmp": {
+      "version": "0.0.33",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
+      "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
+      "dev": true,
+      "requires": {
+        "os-tmpdir": "~1.0.2"
+      }
+    },
+    "to-arraybuffer": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz",
+      "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=",
+      "dev": true
+    },
+    "to-fast-properties": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
+      "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=",
+      "dev": true
+    },
+    "to-object-path": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
+      "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=",
+      "dev": true,
+      "requires": {
+        "kind-of": "^3.0.2"
+      }
+    },
+    "to-regex": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz",
+      "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==",
+      "dev": true,
+      "requires": {
+        "define-property": "^2.0.2",
+        "extend-shallow": "^3.0.2",
+        "regex-not": "^1.0.2",
+        "safe-regex": "^1.1.0"
+      }
+    },
+    "to-regex-range": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz",
+      "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=",
+      "dev": true,
+      "requires": {
+        "is-number": "^3.0.0",
+        "repeat-string": "^1.6.1"
+      }
+    },
+    "trim-right": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz",
+      "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=",
+      "dev": true
+    },
+    "tslib": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
+      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==",
+      "dev": true
+    },
+    "tty-browserify": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
+      "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=",
+      "dev": true
+    },
+    "type-check": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
+      "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
+      "dev": true,
+      "requires": {
+        "prelude-ls": "~1.1.2"
+      }
+    },
+    "typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=",
+      "dev": true
+    },
+    "uglify-js": {
+      "version": "2.8.29",
+      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz",
+      "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "source-map": "~0.5.1",
+        "uglify-to-browserify": "~1.0.0",
+        "yargs": "~3.10.0"
+      }
+    },
+    "uglify-to-browserify": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz",
+      "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=",
+      "dev": true,
+      "optional": true
+    },
+    "uglifyjs-webpack-plugin": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-1.2.7.tgz",
+      "integrity": "sha512-1VicfKhCYHLS8m1DCApqBhoulnASsEoJ/BvpUpP4zoNAPpKzdH+ghk0olGJMmwX2/jprK2j3hAHdUbczBSy2FA==",
+      "dev": true,
+      "requires": {
+        "cacache": "^10.0.4",
+        "find-cache-dir": "^1.0.0",
+        "schema-utils": "^0.4.5",
+        "serialize-javascript": "^1.4.0",
+        "source-map": "^0.6.1",
+        "uglify-es": "^3.3.4",
+        "webpack-sources": "^1.1.0",
+        "worker-farm": "^1.5.2"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        },
+        "uglify-es": {
+          "version": "3.3.9",
+          "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz",
+          "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==",
+          "dev": true,
+          "requires": {
+            "commander": "~2.13.0",
+            "source-map": "~0.6.1"
+          }
+        }
+      }
+    },
+    "union-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
+      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "dev": true,
+      "requires": {
+        "arr-union": "^3.1.0",
+        "get-value": "^2.0.6",
+        "is-extendable": "^0.1.1",
+        "set-value": "^0.4.3"
+      },
+      "dependencies": {
+        "extend-shallow": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+          "dev": true,
+          "requires": {
+            "is-extendable": "^0.1.0"
+          }
+        },
+        "set-value": {
+          "version": "0.4.3",
+          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
+          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
+          "dev": true,
+          "requires": {
+            "extend-shallow": "^2.0.1",
+            "is-extendable": "^0.1.1",
+            "is-plain-object": "^2.0.1",
+            "to-object-path": "^0.3.0"
+          }
+        }
+      }
+    },
+    "uniq": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz",
+      "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=",
+      "dev": true
+    },
+    "uniqs": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz",
+      "integrity": "sha1-/+3ks2slKQaW5uFl1KWe25mOawI=",
+      "dev": true
+    },
+    "unique-filename": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.0.tgz",
+      "integrity": "sha1-0F8v5AMlYIcfMOk8vnNe6iAVFPM=",
+      "dev": true,
+      "requires": {
+        "unique-slug": "^2.0.0"
+      }
+    },
+    "unique-slug": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.0.tgz",
+      "integrity": "sha1-22Z258fMBimHj/GWCXx4hVrp9Ks=",
+      "dev": true,
+      "requires": {
+        "imurmurhash": "^0.1.4"
+      }
+    },
+    "unset-value": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz",
+      "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=",
+      "dev": true,
+      "requires": {
+        "has-value": "^0.3.1",
+        "isobject": "^3.0.0"
+      },
+      "dependencies": {
+        "has-value": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz",
+          "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=",
+          "dev": true,
+          "requires": {
+            "get-value": "^2.0.3",
+            "has-values": "^0.1.4",
+            "isobject": "^2.0.0"
+          },
+          "dependencies": {
+            "isobject": {
+              "version": "2.1.0",
+              "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz",
+              "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=",
+              "dev": true,
+              "requires": {
+                "isarray": "1.0.0"
+              }
+            }
+          }
+        },
+        "has-values": {
+          "version": "0.1.4",
+          "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz",
+          "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=",
+          "dev": true
+        }
+      }
+    },
+    "upath": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/upath/-/upath-1.1.0.tgz",
+      "integrity": "sha512-bzpH/oBhoS/QI/YtbkqCg6VEiPYjSZtrHQM6/QnJS6OL9pKUFLqb3aFh4Scvwm45+7iAgiMkLhSbaZxUqmrprw==",
+      "dev": true
+    },
+    "uri-js": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
+      "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
+      "dev": true,
+      "requires": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "urix": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
+      "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=",
+      "dev": true
+    },
+    "url": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
+      "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
+      "dev": true,
+      "requires": {
+        "punycode": "1.3.2",
+        "querystring": "0.2.0"
+      },
+      "dependencies": {
+        "punycode": {
+          "version": "1.3.2",
+          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
+          "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
+          "dev": true
+        }
+      }
+    },
+    "url-loader": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-1.0.1.tgz",
+      "integrity": "sha512-rAonpHy7231fmweBKUFe0bYnlGDty77E+fm53NZdij7j/YOpyGzc7ttqG1nAXl3aRs0k41o0PC3TvGXQiw2Zvw==",
+      "dev": true,
+      "requires": {
+        "loader-utils": "^1.1.0",
+        "mime": "^2.0.3",
+        "schema-utils": "^0.4.3"
+      }
+    },
+    "use": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz",
+      "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==",
+      "dev": true
+    },
+    "util": {
+      "version": "0.10.4",
+      "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
+      "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
+      "dev": true,
+      "requires": {
+        "inherits": "2.0.3"
+      }
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
+      "dev": true
+    },
+    "uuid": {
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
+      "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
+      "dev": true
+    },
+    "v8-compile-cache": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.0.0.tgz",
+      "integrity": "sha512-qNdTUMaCjPs4eEnM3W9H94R3sU70YCuT+/ST7nUf+id1bVOrdjrpUaeZLqPBPRph3hsgn4a4BvwpxhHZx+oSDg==",
+      "dev": true
+    },
+    "validate-npm-package-license": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.3.tgz",
+      "integrity": "sha512-63ZOUnL4SIXj4L0NixR3L1lcjO38crAbgrTpl28t8jjrfuiOBL5Iygm+60qPs/KsZGzPNg6Smnc/oY16QTjF0g==",
+      "dev": true,
+      "requires": {
+        "spdx-correct": "^3.0.0",
+        "spdx-expression-parse": "^3.0.0"
+      }
+    },
+    "vendors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.2.tgz",
+      "integrity": "sha512-w/hry/368nO21AN9QljsaIhb9ZiZtZARoVH5f3CsFbawdLdayCgKRPup7CggujvySMxx0I91NOyxdVENohprLQ==",
+      "dev": true
+    },
+    "vm-browserify": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz",
+      "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=",
+      "dev": true,
+      "requires": {
+        "indexof": "0.0.1"
+      }
+    },
+    "watchpack": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.6.0.tgz",
+      "integrity": "sha512-i6dHe3EyLjMmDlU1/bGQpEw25XSjkJULPuAVKCbNRefQVq48yXKUpwg538F7AZTf9kyr57zj++pQFltUa5H7yA==",
+      "dev": true,
+      "requires": {
+        "chokidar": "^2.0.2",
+        "graceful-fs": "^4.1.2",
+        "neo-async": "^2.5.0"
+      }
+    },
+    "webpack": {
+      "version": "4.16.3",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.16.3.tgz",
+      "integrity": "sha512-3VcrVoFgzSz1IYgga71YpU3HO89Al5bSnDOj9RJQPsy+FNyI1sFsUyJITn3pktNuaRBlQT0usvKZE3GgkPGAIw==",
+      "dev": true,
+      "requires": {
+        "@webassemblyjs/ast": "1.5.13",
+        "@webassemblyjs/helper-module-context": "1.5.13",
+        "@webassemblyjs/wasm-edit": "1.5.13",
+        "@webassemblyjs/wasm-opt": "1.5.13",
+        "@webassemblyjs/wasm-parser": "1.5.13",
+        "acorn": "^5.6.2",
+        "acorn-dynamic-import": "^3.0.0",
+        "ajv": "^6.1.0",
+        "ajv-keywords": "^3.1.0",
+        "chrome-trace-event": "^1.0.0",
+        "enhanced-resolve": "^4.1.0",
+        "eslint-scope": "^4.0.0",
+        "json-parse-better-errors": "^1.0.2",
+        "loader-runner": "^2.3.0",
+        "loader-utils": "^1.1.0",
+        "memory-fs": "~0.4.1",
+        "micromatch": "^3.1.8",
+        "mkdirp": "~0.5.0",
+        "neo-async": "^2.5.0",
+        "node-libs-browser": "^2.0.0",
+        "schema-utils": "^0.4.4",
+        "tapable": "^1.0.0",
+        "uglifyjs-webpack-plugin": "^1.2.4",
+        "watchpack": "^1.5.0",
+        "webpack-sources": "^1.0.1"
+      },
+      "dependencies": {
+        "eslint-scope": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.0.tgz",
+          "integrity": "sha512-1G6UTDi7Jc1ELFwnR58HV4fK9OQK4S6N985f166xqXxpjU6plxFISJa2Ba9KCQuFa8RCnj/lSFJbHo7UFDBnUA==",
+          "dev": true,
+          "requires": {
+            "esrecurse": "^4.1.0",
+            "estraverse": "^4.1.1"
+          }
+        },
+        "estraverse": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
+          "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=",
+          "dev": true
+        }
+      }
+    },
+    "webpack-cli": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-3.1.0.tgz",
+      "integrity": "sha512-p5NeKDtYwjZozUWq6kGNs9w+Gtw/CPvyuXjXn2HMdz8Tie+krjEg8oAtonvIyITZdvpF7XG9xDHwscLr2c+ugQ==",
+      "dev": true,
+      "requires": {
+        "chalk": "^2.4.1",
+        "cross-spawn": "^6.0.5",
+        "enhanced-resolve": "^4.0.0",
+        "global-modules-path": "^2.1.0",
+        "import-local": "^1.0.0",
+        "inquirer": "^6.0.0",
+        "interpret": "^1.1.0",
+        "loader-utils": "^1.1.0",
+        "supports-color": "^5.4.0",
+        "v8-compile-cache": "^2.0.0",
+        "yargs": "^12.0.1"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "cliui": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
+          "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
+          "dev": true,
+          "requires": {
+            "string-width": "^2.1.1",
+            "strip-ansi": "^4.0.0",
+            "wrap-ansi": "^2.0.0"
+          }
+        },
+        "decamelize": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-2.0.0.tgz",
+          "integrity": "sha512-Ikpp5scV3MSYxY39ymh45ZLEecsTdv/Xj2CaQfI8RLMuwi7XvjX9H/fhraiSuU+C5w5NTDu4ZU72xNiZnurBPg==",
+          "dev": true,
+          "requires": {
+            "xregexp": "4.0.0"
+          }
+        },
+        "find-up": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
+          "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
+          "dev": true,
+          "requires": {
+            "locate-path": "^3.0.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "locate-path": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
+          "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
+          "dev": true,
+          "requires": {
+            "p-locate": "^3.0.0",
+            "path-exists": "^3.0.0"
+          }
+        },
+        "p-limit": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz",
+          "integrity": "sha512-fl5s52lI5ahKCernzzIyAP0QAZbGIovtVHGwpcu1Jr/EpzLVDI2myISHwGqK7m8uQFugVWSrbxH7XnhGtvEc+A==",
+          "dev": true,
+          "requires": {
+            "p-try": "^2.0.0"
+          }
+        },
+        "p-locate": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
+          "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
+          "dev": true,
+          "requires": {
+            "p-limit": "^2.0.0"
+          }
+        },
+        "p-try": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz",
+          "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==",
+          "dev": true
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        },
+        "yargs": {
+          "version": "12.0.1",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.1.tgz",
+          "integrity": "sha512-B0vRAp1hRX4jgIOWFtjfNjd9OA9RWYZ6tqGA9/I/IrTMsxmKvtWy+ersM+jzpQqbC3YfLzeABPdeTgcJ9eu1qQ==",
+          "dev": true,
+          "requires": {
+            "cliui": "^4.0.0",
+            "decamelize": "^2.0.0",
+            "find-up": "^3.0.0",
+            "get-caller-file": "^1.0.1",
+            "os-locale": "^2.0.0",
+            "require-directory": "^2.1.1",
+            "require-main-filename": "^1.0.1",
+            "set-blocking": "^2.0.0",
+            "string-width": "^2.0.0",
+            "which-module": "^2.0.0",
+            "y18n": "^3.2.1 || ^4.0.0",
+            "yargs-parser": "^10.1.0"
+          }
+        }
+      }
+    },
+    "webpack-log": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/webpack-log/-/webpack-log-1.2.0.tgz",
+      "integrity": "sha512-U9AnICnu50HXtiqiDxuli5gLB5PGBo7VvcHx36jRZHwK4vzOYLbImqT4lwWwoMHdQWwEKw736fCHEekokTEKHA==",
+      "dev": true,
+      "requires": {
+        "chalk": "^2.1.0",
+        "log-symbols": "^2.1.0",
+        "loglevelnext": "^1.0.1",
+        "uuid": "^3.1.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "chalk": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
+          "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^3.2.1",
+            "escape-string-regexp": "^1.0.5",
+            "supports-color": "^5.3.0"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "webpack-manifest-plugin": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/webpack-manifest-plugin/-/webpack-manifest-plugin-2.0.3.tgz",
+      "integrity": "sha512-FZcnB3MMQ0CT0aU1+LItwywXWAixLTGUEAtN0fw15dScf2LudQwheLPUCj+QMhDlwZT+9ysfKqUFTcfUGc8bXg==",
+      "dev": true,
+      "requires": {
+        "fs-extra": "^0.30.0",
+        "lodash": ">=3.5 <5",
+        "tapable": "^1.0.0"
+      }
+    },
+    "webpack-sources": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.1.0.tgz",
+      "integrity": "sha512-aqYp18kPphgoO5c/+NaUvEeACtZjMESmDChuD3NBciVpah3XpMEU9VAAtIaB1BsfJWWTSdv8Vv1m3T0aRk2dUw==",
+      "dev": true,
+      "requires": {
+        "source-list-map": "^2.0.0",
+        "source-map": "~0.6.1"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
+      }
+    },
+    "whet.extend": {
+      "version": "0.9.9",
+      "resolved": "https://registry.npmjs.org/whet.extend/-/whet.extend-0.9.9.tgz",
+      "integrity": "sha1-+HfVv2SMl+WqVC+twW1qJZucEaE=",
+      "dev": true
+    },
+    "which": {
+      "version": "1.2.14",
+      "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz",
+      "integrity": "sha1-mofEN48D6CfOyvGs31bHNsAcFOU=",
+      "dev": true,
+      "requires": {
+        "isexe": "^2.0.0"
+      }
+    },
+    "which-module": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
+      "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
+      "dev": true
+    },
+    "window-size": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz",
+      "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=",
+      "dev": true,
+      "optional": true
+    },
+    "wordwrap": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
+      "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=",
+      "dev": true
+    },
+    "worker-farm": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.6.0.tgz",
+      "integrity": "sha512-6w+3tHbM87WnSWnENBUvA2pxJPLhQUg5LKwUQHq3r+XPhIM+Gh2R5ycbwPCyuGbNg+lPgdcnQUhuC02kJCvffQ==",
+      "dev": true,
+      "requires": {
+        "errno": "~0.1.7"
+      }
+    },
+    "wrap-ansi": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
+      "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
+      "dev": true,
+      "requires": {
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1"
+      },
+      "dependencies": {
+        "is-fullwidth-code-point": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+          "dev": true,
+          "requires": {
+            "number-is-nan": "^1.0.0"
+          }
+        },
+        "string-width": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+          "dev": true,
+          "requires": {
+            "code-point-at": "^1.0.0",
+            "is-fullwidth-code-point": "^1.0.0",
+            "strip-ansi": "^3.0.0"
+          }
+        }
+      }
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    },
+    "write": {
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz",
+      "integrity": "sha1-X8A4KOJkzqP+kUVUdvejxWbLB1c=",
+      "dev": true,
+      "requires": {
+        "mkdirp": "^0.5.1"
+      }
+    },
+    "xregexp": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.0.0.tgz",
+      "integrity": "sha512-PHyM+sQouu7xspQQwELlGwwd05mXUFqwFYfqPO0cC7x4fxyHnnuetmQr6CjJiafIDoH4MogHb9dOoJzR/Y4rFg==",
+      "dev": true
+    },
+    "xtend": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
+      "dev": true
+    },
+    "y18n": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
+      "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
+      "dev": true
+    },
+    "yallist": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
+      "dev": true
+    },
+    "yargs": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
+      "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "camelcase": "^1.0.2",
+        "cliui": "^2.1.0",
+        "decamelize": "^1.0.0",
+        "window-size": "0.1.0"
+      }
+    },
+    "yargs-parser": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz",
+      "integrity": "sha512-VCIyR1wJoEBZUqk5PA+oOBF6ypbwh5aNB3I50guxAL/quggdfs4TtNHQrSazFA3fYZ+tEqfs0zIGlv0c/rgjbQ==",
+      "dev": true,
+      "requires": {
+        "camelcase": "^4.1.0"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
+          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=",
+          "dev": true
+        }
+      }
+    }
+  }
+}
diff --git a/airflow/www_rbac/package.json b/airflow/www_rbac/package.json
new file mode 100644
index 0000000000000..5c431895a7666
--- /dev/null
+++ b/airflow/www_rbac/package.json
@@ -0,0 +1,69 @@
+{
+  "description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development",
+    "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress",
+    "build": "NODE_ENV=production webpack --colors --progress",
+    "lint": "eslint --ignore-path=.eslintignore --ext .js,.html .",
+    "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ."
+  },
+  "author": "Apache",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apache/airflow.git"
+  },
+  "homepage": "http://airflow.apache.org/",
+  "keywords": [
+    "big",
+    "data",
+    "workflow",
+    "airflow",
+    "d3",
+    "nerds",
+    "database",
+    "flask"
+  ],
+  "devDependencies": {
+    "babel": "^6.23.0",
+    "babel-core": "^6.26.3",
+    "babel-eslint": "^8.2.6",
+    "babel-istanbul": "^0.12.2",
+    "babel-loader": "^7.1.4",
+    "babel-plugin-css-modules-transform": "^1.6.1",
+    "babel-polyfill": "^6.26.0",
+    "clean-webpack-plugin": "^0.1.19",
+    "copy-webpack-plugin": "^4.5.2",
+    "css-loader": "^0.28.11",
+    "eslint": "^4.19.1",
+    "eslint-config-airbnb-base": "^13.0.0",
+    "eslint-plugin-html": "^4.0.5",
+    "eslint-plugin-import": "^2.13.0",
+    "eslint-plugin-node": "^7.0.1",
+    "eslint-plugin-promise": "^3.8.0",
+    "eslint-plugin-standard": "^3.1.0",
+    "file-loader": "^1.1.11",
+    "imports-loader": "^0.8.0",
+    "lodash": "^4.17.11",
+    "mini-css-extract-plugin": "^0.4.1",
+    "style-loader": "^0.21.0",
+    "url-loader": "^1.0.1",
+    "webpack": "^4.16.3",
+    "webpack-cli": "^3.1.0",
+    "webpack-manifest-plugin": "^2.0.3"
+  },
+  "dependencies": {
+    "bootstrap-3-typeahead": "^4.0.2",
+    "bootstrap-toggle": "^2.2.2",
+    "d3": "^3.4.4",
+    "d3-tip": "^0.9.1",
+    "dagre-d3": "^0.4.18",
+    "datatables.net": "^1.10.19",
+    "datatables.net-bs": "^1.10.19",
+    "lodash": "^4.17.11",
+    "moment-timezone": "^0.5.21",
+    "npm": "^6.3.0",
+    "nvd3": "^1.8.6"
+  }
+}
diff --git a/airflow/www_rbac/security.py b/airflow/www_rbac/security.py
index d2271f822a47e..18a9c447b66a4 100644
--- a/airflow/www_rbac/security.py
+++ b/airflow/www_rbac/security.py
@@ -7,22 +7,31 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+#
 
+from flask import g
 from flask_appbuilder.security.sqla import models as sqla_models
+from flask_appbuilder.security.sqla.manager import SecurityManager
+from sqlalchemy import or_
+
+from airflow import models
+from airflow.www_rbac.app import appbuilder
+from airflow.utils.db import provide_session
+from airflow.utils.log.logging_mixin import LoggingMixin
 
 ###########################################################################
 #                               VIEW MENUS
 ###########################################################################
-viewer_vms = [
+VIEWER_VMS = {
     'Airflow',
     'DagModelView',
     'Browse',
@@ -42,11 +51,11 @@
     'About',
     'Version',
     'VersionView',
-]
+}
 
-user_vms = viewer_vms
+USER_VMS = VIEWER_VMS
 
-op_vms = [
+OP_VMS = {
     'Admin',
     'Configurations',
     'ConfigurationView',
@@ -58,13 +67,13 @@
     'VariableModelView',
     'XComs',
     'XComModelView',
-]
+}
 
 ###########################################################################
 #                               PERMISSIONS
 ###########################################################################
 
-viewer_perms = [
+VIEWER_PERMS = {
     'menu_access',
     'can_index',
     'can_list',
@@ -75,6 +84,7 @@
     'can_task_stats',
     'can_code',
     'can_log',
+    'can_get_logs_with_metadata',
     'can_tries',
     'can_graph',
     'can_tree',
@@ -88,9 +98,9 @@
     'can_rendered',
     'can_pickle_info',
     'can_version',
-]
+}
 
-user_perms = [
+USER_PERMS = {
     'can_dagrun_clear',
     'can_run',
     'can_trigger',
@@ -105,12 +115,23 @@
     'set_running',
     'set_success',
     'clear',
-]
+    'can_clear',
+}
 
-op_perms = [
+OP_PERMS = {
     'can_conf',
     'can_varimport',
-]
+}
+
+# global view-menu for dag-level access
+DAG_VMS = {
+    'all_dags'
+}
+
+DAG_PERMS = {
+    'can_dag_read',
+    'can_dag_edit',
+}
 
 ###########################################################################
 #                     DEFAULT ROLE CONFIGURATIONS
@@ -119,61 +140,344 @@
 ROLE_CONFIGS = [
     {
         'role': 'Viewer',
-        'perms': viewer_perms,
-        'vms': viewer_vms,
+        'perms': VIEWER_PERMS,
+        'vms': VIEWER_VMS | DAG_VMS,
     },
     {
         'role': 'User',
-        'perms': viewer_perms + user_perms,
-        'vms': viewer_vms + user_vms,
+        'perms': VIEWER_PERMS | USER_PERMS | DAG_PERMS,
+        'vms': VIEWER_VMS | DAG_VMS | USER_VMS,
     },
     {
         'role': 'Op',
-        'perms': viewer_perms + user_perms + op_perms,
-        'vms': viewer_vms + user_vms + op_vms,
+        'perms': VIEWER_PERMS | USER_PERMS | OP_PERMS | DAG_PERMS,
+        'vms': VIEWER_VMS | DAG_VMS | USER_VMS | OP_VMS,
     },
 ]
 
+EXISTING_ROLES = {
+    'Admin',
+    'Viewer',
+    'User',
+    'Op',
+    'Public',
+}
+
+
+class AirflowSecurityManager(SecurityManager, LoggingMixin):
+
+    def init_role(self, role_name, role_vms, role_perms):
+        """
+        Initialize the role with the permissions and related view-menus.
+
+        :param role_name:
+        :param role_vms:
+        :param role_perms:
+        :return:
+        """
+        pvms = self.get_session.query(sqla_models.PermissionView).all()
+        pvms = [p for p in pvms if p.permission and p.view_menu]
+
+        role = self.find_role(role_name)
+        if not role:
+            role = self.add_role(role_name)
+
+        if len(role.permissions) == 0:
+            self.log.info('Initializing permissions for role:%s in the database.', role_name)
+            role_pvms = set()
+            for pvm in pvms:
+                if pvm.view_menu.name in role_vms and pvm.permission.name in role_perms:
+                    role_pvms.add(pvm)
+            role.permissions = list(role_pvms)
+            self.get_session.merge(role)
+            self.get_session.commit()
+        else:
+            self.log.debug('Existing permissions for the role:%s '
+                           'within the database will persist.', role_name)
+
+    def get_user_roles(self, user=None):
+        """
+        Get all the roles associated with the user.
+
+        :param user: the ab_user in FAB model.
+        :return: a list of roles associated with the user.
+        """
+        if user is None:
+            user = g.user
+        if user.is_anonymous:
+            public_role = appbuilder.config.get('AUTH_ROLE_PUBLIC')
+            return [appbuilder.security_manager.find_role(public_role)] \
+                if public_role else []
+        return user.roles
+
+    def get_all_permissions_views(self):
+        """
+        Returns a set of tuples with the perm name and view menu name
+        """
+        perms_views = set()
+        for role in self.get_user_roles():
+            perms_views.update({(perm_view.permission.name, perm_view.view_menu.name)
+                                for perm_view in role.permissions})
+        return perms_views
+
+    def get_accessible_dag_ids(self, username=None):
+        """
+        Return a set of dags that user has access to(either read or write).
+
+        :param username: Name of the user.
+        :return: A set of dag ids that the user could access.
+        """
+        if not username:
+            username = g.user
+
+        if username.is_anonymous or 'Public' in username.roles:
+            # return an empty set if the role is public
+            return set()
+
+        roles = {role.name for role in username.roles}
+        if {'Admin', 'Viewer', 'User', 'Op'} & roles:
+            return DAG_VMS
+
+        user_perms_views = self.get_all_permissions_views()
+        # return a set of all dags that the user could access
+        return set([view for perm, view in user_perms_views if perm in DAG_PERMS])
+
+    def has_access(self, permission, view_name, user=None):
+        """
+        Verify whether a given user could perform certain permission
+        (e.g can_read, can_write) on the given dag_id.
+
+        :param permission: permission on dag_id(e.g can_read, can_edit).
+        :type permission: str
+        :param view_name: name of view-menu(e.g dag id is a view-menu as well).
+        :type permission: str
+        :param user: user name
+        :type permission: str
+        :return: a bool whether user could perform certain permission on the dag_id.
+        :rtype bool
+        """
+        if not user:
+            user = g.user
+        if user.is_anonymous:
+            return self.is_item_public(permission, view_name)
+        return self._has_view_access(user, permission, view_name)
+
+    def _get_and_cache_perms(self):
+        """
+        Cache permissions-views
+        """
+        self.perms = self.get_all_permissions_views()
+
+    def _has_role(self, role_name_or_list):
+        """
+        Whether the user has this role name
+        """
+        if not isinstance(role_name_or_list, list):
+            role_name_or_list = [role_name_or_list]
+        return any(
+            [r.name in role_name_or_list for r in self.get_user_roles()])
+
+    def _has_perm(self, permission_name, view_menu_name):
+        """
+        Whether the user has this perm
+        """
+        if hasattr(self, 'perms'):
+            if (permission_name, view_menu_name) in self.perms:
+                return True
+        # rebuild the permissions set
+        self._get_and_cache_perms()
+        return (permission_name, view_menu_name) in self.perms
+
+    def has_all_dags_access(self):
+        """
+        Has all the dag access in any of the 3 cases:
+        1. Role needs to be in (Admin, Viewer, User, Op).
+        2. Has can_dag_read permission on all_dags view.
+        3. Has can_dag_edit permission on all_dags view.
+        """
+        return (
+            self._has_role(['Admin', 'Viewer', 'Op', 'User']) or
+            self._has_perm('can_dag_read', 'all_dags') or
+            self._has_perm('can_dag_edit', 'all_dags'))
+
+    def clean_perms(self):
+        """
+        FAB leaves faulty permissions that need to be cleaned up
+        """
+        self.log.info('Cleaning faulty perms')
+        sesh = self.get_session
+        pvms = (
+            sesh.query(sqla_models.PermissionView)
+            .filter(or_(
+                sqla_models.PermissionView.permission == None,  # NOQA
+                sqla_models.PermissionView.view_menu == None,  # NOQA
+            ))
+        )
+        deleted_count = pvms.delete()
+        sesh.commit()
+        if deleted_count:
+            self.log.info('Deleted %s faulty permissions', deleted_count)
+
+    def _merge_perm(self, permission_name, view_menu_name):
+        """
+        Add the new permission , view_menu to ab_permission_view_role if not exists.
+        It will add the related entry to ab_permission
+        and ab_view_menu two meta tables as well.
+
+        :param permission_name: Name of the permission.
+        :type permission_name: str
+        :param view_menu_name: Name of the view-menu
+        :type view_menu_name: str
+        :return:
+        """
+        permission = self.find_permission(permission_name)
+        view_menu = self.find_view_menu(view_menu_name)
+        pv = None
+        if permission and view_menu:
+            pv = self.get_session.query(self.permissionview_model).filter_by(
+                permission=permission, view_menu=view_menu).first()
+        if not pv and permission_name and view_menu_name:
+            self.add_permission_view_menu(permission_name, view_menu_name)
+
+    @provide_session
+    def create_custom_dag_permission_view(self, session=None):
+        """
+        Workflow:
+        1. Fetch all the existing (permissions, view-menu) from Airflow DB.
+        2. Fetch all the existing dag models that are either active or paused. Exclude the subdags.
+        3. Create both read and write permission view-menus relation for every dags from step 2
+        4. Find out all the dag specific roles(excluded pubic, admin, viewer, op, user)
+        5. Get all the permission-vm owned by the user role.
+        6. Grant all the user role's permission-vm except the all-dag view-menus to the dag roles.
+        7. Commit the updated permission-vm-role into db
+
+        :return: None.
+        """
+        # todo(Tao): should we put this function here or in scheduler loop?
+        self.log.info('Fetching a set of all permission, view_menu from FAB meta-table')
+
+        def merge_pv(perm, view_menu):
+            """Create permission view menu only if it doesn't exist"""
+            if view_menu and perm and (view_menu, perm) not in all_pvs:
+                self._merge_perm(perm, view_menu)
+
+        all_pvs = set()
+        for pv in self.get_session.query(self.permissionview_model).all():
+            if pv.permission and pv.view_menu:
+                all_pvs.add((pv.permission.name, pv.view_menu.name))
+
+        # create perm for global logical dag
+        for dag in DAG_VMS:
+            for perm in DAG_PERMS:
+                merge_pv(perm, dag)
+
+        # Get all the active / paused dags and insert them into a set
+        all_dags_models = session.query(models.DagModel)\
+            .filter(or_(models.DagModel.is_active, models.DagModel.is_paused))\
+            .filter(~models.DagModel.is_subdag).all()
+
+        # create can_dag_edit and can_dag_read permissions for every dag(vm)
+        for dag in all_dags_models:
+            for perm in DAG_PERMS:
+                merge_pv(perm, dag.dag_id)
+
+        # for all the dag-level role, add the permission of viewer
+        # with the dag view to ab_permission_view
+        all_roles = self.get_all_roles()
+        user_role = self.find_role('User')
+
+        dag_role = [role for role in all_roles if role.name not in EXISTING_ROLES]
+        update_perm_views = []
+
+        # need to remove all_dag vm from all the existing view-menus
+        dag_vm = self.find_view_menu('all_dags')
+        ab_perm_view_role = sqla_models.assoc_permissionview_role
+        perm_view = self.permissionview_model
+        view_menu = self.viewmenu_model
+
+        all_perm_view_by_user = session.query(ab_perm_view_role)\
+            .join(perm_view, perm_view.id == ab_perm_view_role
+                  .columns.permission_view_id)\
+            .filter(ab_perm_view_role.columns.role_id == user_role.id)\
+            .join(view_menu)\
+            .filter(perm_view.view_menu_id != dag_vm.id)
+        all_perm_views = set([role.permission_view_id for role in all_perm_view_by_user])
+
+        for role in dag_role:
+            # Get all the perm-view of the role
+            existing_perm_view_by_user = self.get_session.query(ab_perm_view_role)\
+                .filter(ab_perm_view_role.columns.role_id == role.id)
+
+            existing_perms_views = set([pv.permission_view_id
+                                        for pv in existing_perm_view_by_user])
+            missing_perm_views = all_perm_views - existing_perms_views
+
+            for perm_view_id in missing_perm_views:
+                update_perm_views.append({'permission_view_id': perm_view_id,
+                                          'role_id': role.id})
+
+        if update_perm_views:
+            self.get_session.execute(ab_perm_view_role.insert(), update_perm_views)
+        self.get_session.commit()
+
+    def update_admin_perm_view(self):
+        """
+        Admin should have all the permission-views.
+        Add the missing ones to the table for admin.
+
+        :return: None.
+        """
+        pvms = self.get_session.query(sqla_models.PermissionView).all()
+        pvms = [p for p in pvms if p.permission and p.view_menu]
+
+        admin = self.find_role('Admin')
+        admin.permissions = list(set(admin.permissions) | set(pvms))
+
+        self.get_session.commit()
+
+    def sync_roles(self):
+        """
+        1. Init the default role(Admin, Viewer, User, Op, public)
+           with related permissions.
+        2. Init the custom role(dag-user) with related permissions.
+
+        :return: None.
+        """
+        self.log.info('Start syncing user roles.')
+        # Create global all-dag VM
+        self.create_perm_vm_for_all_dag()
+
+        # Create default user role.
+        for config in ROLE_CONFIGS:
+            role = config['role']
+            vms = config['vms']
+            perms = config['perms']
+            self.init_role(role, vms, perms)
+        self.create_custom_dag_permission_view()
+
+        # init existing roles, the rest role could be created through UI.
+        self.update_admin_perm_view()
+        self.clean_perms()
+
+    def sync_perm_for_dag(self, dag_id):
+        """
+        Sync permissions for given dag id. The dag id surely exists in our dag bag
+        as only / refresh button will call this function
+
+        :param dag_id:
+        :return:
+        """
+        for dag_perm in DAG_PERMS:
+            perm_on_dag = self.find_permission_view_menu(dag_perm, dag_id)
+            if perm_on_dag is None:
+                self.add_permission_view_menu(dag_perm, dag_id)
 
-def init_role(sm, role_name, role_vms, role_perms):
-    sm_session = sm.get_session
-    pvms = sm_session.query(sqla_models.PermissionView).all()
-    pvms = [p for p in pvms if p.permission and p.view_menu]
-
-    valid_perms = [p.permission.name for p in pvms]
-    valid_vms = [p.view_menu.name for p in pvms]
-    invalid_perms = [p for p in role_perms if p not in valid_perms]
-    if invalid_perms:
-        raise Exception('The following permissions are not valid: {}'
-                        .format(invalid_perms))
-    invalid_vms = [v for v in role_vms if v not in valid_vms]
-    if invalid_vms:
-        raise Exception('The following view menus are not valid: {}'
-                        .format(invalid_vms))
-
-    role = sm.add_role(role_name)
-    role_pvms = []
-    for pvm in pvms:
-        if pvm.view_menu.name in role_vms and pvm.permission.name in role_perms:
-            role_pvms.append(pvm)
-    role_pvms = list(set(role_pvms))
-    role.permissions = role_pvms
-    sm_session.merge(role)
-    sm_session.commit()
-
-
-def init_roles(appbuilder):
-    for config in ROLE_CONFIGS:
-        name = config['role']
-        vms = config['vms']
-        perms = config['perms']
-        init_role(appbuilder.sm, name, vms, perms)
-
-
-def is_view_only(user, appbuilder):
-    if user.is_anonymous():
-        anonymous_role = appbuilder.sm.auth_role_public
-        return anonymous_role == 'Viewer'
-
-    user_roles = user.roles
-    return len(user_roles) == 1 and user_roles[0].name == 'Viewer'
+    def create_perm_vm_for_all_dag(self):
+        """
+        Create perm-vm if not exist and insert into FAB security model for all-dags.
+        """
+        # create perm for global logical dag
+        for dag_vm in DAG_VMS:
+            for perm in DAG_PERMS:
+                self._merge_perm(permission_name=perm,
+                                 view_menu_name=dag_vm)
diff --git a/airflow/www_rbac/static/bootstrap-toggle.min.css b/airflow/www_rbac/static/bootstrap-toggle.min.css
deleted file mode 100644
index 0d42ed09cdaa3..0000000000000
--- a/airflow/www_rbac/static/bootstrap-toggle.min.css
+++ /dev/null
@@ -1,28 +0,0 @@
-/*! ========================================================================
- * Bootstrap Toggle: bootstrap-toggle.css v2.2.0
- * http://www.bootstraptoggle.com
- * ========================================================================
- * Copyright 2014 Min Hur, The New York Times Company
- * Licensed under MIT
- * ======================================================================== */
-.checkbox label .toggle,.checkbox-inline .toggle{margin-left:-20px;margin-right:5px}
-.toggle{position:relative;overflow:hidden}
-.toggle input[type=checkbox]{display:none}
-.toggle-group{position:absolute;width:200%;top:0;bottom:0;left:0;transition:left .35s;-webkit-transition:left .35s;-moz-user-select:none;-webkit-user-select:none}
-.toggle.off .toggle-group{left:-100%}
-.toggle-on{position:absolute;top:0;bottom:0;left:0;right:50%;margin:0;border:0;border-radius:0}
-.toggle-off{position:absolute;top:0;bottom:0;left:50%;right:0;margin:0;border:0;border-radius:0}
-.toggle-handle{position:relative;margin:0 auto;padding-top:0;padding-bottom:0;height:100%;width:0;border-width:0 1px}
-.toggle.btn{min-width:59px;min-height:34px}
-.toggle-on.btn{padding-right:24px}
-.toggle-off.btn{padding-left:24px}
-.toggle.btn-lg{min-width:79px;min-height:45px}
-.toggle-on.btn-lg{padding-right:31px}
-.toggle-off.btn-lg{padding-left:31px}
-.toggle-handle.btn-lg{width:40px}
-.toggle.btn-sm{min-width:50px;min-height:30px}
-.toggle-on.btn-sm{padding-right:20px}
-.toggle-off.btn-sm{padding-left:20px}
-.toggle.btn-xs{min-width:35px;min-height:22px}
-.toggle-on.btn-xs{padding-right:12px}
-.toggle-off.btn-xs{padding-left:12px}
\ No newline at end of file
diff --git a/airflow/www_rbac/static/bootstrap-toggle.min.js b/airflow/www_rbac/static/bootstrap-toggle.min.js
deleted file mode 100644
index 37113200bff5c..0000000000000
--- a/airflow/www_rbac/static/bootstrap-toggle.min.js
+++ /dev/null
@@ -1,9 +0,0 @@
-/*! ========================================================================
- * Bootstrap Toggle: bootstrap-toggle.js v2.2.0
- * http://www.bootstraptoggle.com
- * ========================================================================
- * Copyright 2014 Min Hur, The New York Times Company
- * Licensed under MIT
- * ======================================================================== */
-+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.toggle"),f="object"==typeof b&&b;e||d.data("bs.toggle",e=new c(this,f)),"string"==typeof b&&e[b]&&e[b]()})}var c=function(b,c){this.$element=a(b),this.options=a.extend({},this.defaults(),c),this.render()};c.VERSION="2.2.0",c.DEFAULTS={on:"On",off:"Off",onstyle:"primary",offstyle:"default",size:"normal",style:"",width:null,height:null},c.prototype.defaults=function(){return{on:this.$element.attr("data-on")||c.DEFAULTS.on,off:this.$element.attr("data-off")||c.DEFAULTS.off,onstyle:this.$element.attr("data-onstyle")||c.DEFAULTS.onstyle,offstyle:this.$element.attr("data-offstyle")||c.DEFAULTS.offstyle,size:this.$element.attr("data-size")||c.DEFAULTS.size,style:this.$element.attr("data-style")||c.DEFAULTS.style,width:this.$element.attr("data-width")||c.DEFAULTS.width,height:this.$element.attr("data-height")||c.DEFAULTS.height}},c.prototype.render=function(){this._onstyle="btn-"+this.options.onstyle,this._offstyle="btn-"+this.options.offstyle;var b="large"===this.options.size?"btn-lg":"small"===this.options.size?"btn-sm":"mini"===this.options.size?"btn-xs":"",c=a('<label class="btn">').html(this.options.on).addClass(this._onstyle+" "+b),d=a('<label class="btn">').html(this.options.off).addClass(this._offstyle+" "+b+" active"),e=a('<span class="toggle-handle btn btn-default">').addClass(b),f=a('<div class="toggle-group">').append(c,d,e),g=a('<div class="toggle btn" data-toggle="toggle">').addClass(this.$element.prop("checked")?this._onstyle:this._offstyle+" off").addClass(b).addClass(this.options.style);this.$element.wrap(g),a.extend(this,{$toggle:this.$element.parent(),$toggleOn:c,$toggleOff:d,$toggleGroup:f}),this.$toggle.append(f);var h=this.options.width||Math.max(c.outerWidth(),d.outerWidth())+e.outerWidth()/2,i=this.options.height||Math.max(c.outerHeight(),d.outerHeight());c.addClass("toggle-on"),d.addClass("toggle-off"),this.$toggle.css({width:h,height:i}),this.options.height&&(c.css("line-height",c.height()+"px"),d.css("line-height",d.height()+"px")),this.update(!0),this.trigger(!0)},c.prototype.toggle=function(){this.$element.prop("checked")?this.off():this.on()},c.prototype.on=function(a){return this.$element.prop("disabled")?!1:(this.$toggle.removeClass(this._offstyle+" off").addClass(this._onstyle),this.$element.prop("checked",!0),void(a||this.trigger()))},c.prototype.off=function(a){return this.$element.prop("disabled")?!1:(this.$toggle.removeClass(this._onstyle).addClass(this._offstyle+" off"),this.$element.prop("checked",!1),void(a||this.trigger()))},c.prototype.enable=function(){this.$toggle.removeAttr("disabled"),this.$element.prop("disabled",!1)},c.prototype.disable=function(){this.$toggle.attr("disabled","disabled"),this.$element.prop("disabled",!0)},c.prototype.update=function(a){this.$element.prop("disabled")?this.disable():this.enable(),this.$element.prop("checked")?this.on(a):this.off(a)},c.prototype.trigger=function(b){this.$element.off("change.bs.toggle"),b||this.$element.change(),this.$element.on("change.bs.toggle",a.proxy(function(){this.update()},this))},c.prototype.destroy=function(){this.$element.off("change.bs.toggle"),this.$toggleGroup.remove(),this.$element.removeData("bs.toggle"),this.$element.unwrap()};var d=a.fn.bootstrapToggle;a.fn.bootstrapToggle=b,a.fn.bootstrapToggle.Constructor=c,a.fn.toggle.noConflict=function(){return a.fn.bootstrapToggle=d,this},a(function(){a("input[type=checkbox][data-toggle^=toggle]").bootstrapToggle()}),a(document).on("click.bs.toggle","div[data-toggle^=toggle]",function(b){var c=a(this).find("input[type=checkbox]");c.bootstrapToggle("toggle"),b.preventDefault()})}(jQuery);
-//# sourceMappingURL=bootstrap-toggle.min.js.map
\ No newline at end of file
diff --git a/airflow/www_rbac/static/bootstrap3-typeahead.min.js b/airflow/www_rbac/static/bootstrap3-typeahead.min.js
deleted file mode 100644
index be0703db33581..0000000000000
--- a/airflow/www_rbac/static/bootstrap3-typeahead.min.js
+++ /dev/null
@@ -1,26 +0,0 @@
-/* =============================================================
- * bootstrap3-typeahead.js v4.0.2
- * https://github.com/bassjobsen/Bootstrap-3-Typeahead
- * =============================================================
- * Original written by @mdo and @fat
- * =============================================================
- * Copyright 2014 Bass Jobsen @bassjobsen
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * ============================================================ */
-!function(a,b){"use strict";"undefined"!=typeof module&&module.exports?module.exports=b(require("jquery")):"function"==typeof define&&define.amd?define(["jquery"],function(a){return b(a)}):b(a.jQuery)}(this,function(a){"use strict";var b=function(c,d){this.$element=a(c),this.options=a.extend({},b.defaults,d),this.matcher=this.options.matcher||this.matcher,this.sorter=this.options.sorter||this.sorter,this.select=this.options.select||this.select,this.autoSelect="boolean"!=typeof this.options.autoSelect||this.options.autoSelect,this.highlighter=this.options.highlighter||this.highlighter,this.render=this.options.render||this.render,this.updater=this.options.updater||this.updater,this.displayText=this.options.displayText||this.displayText,this.source=this.options.source,this.delay=this.options.delay,this.$menu=a(this.options.menu),this.$appendTo=this.options.appendTo?a(this.options.appendTo):null,this.fitToElement="boolean"==typeof this.options.fitToElement&&this.options.fitToElement,this.shown=!1,this.listen(),this.showHintOnFocus=("boolean"==typeof this.options.showHintOnFocus||"all"===this.options.showHintOnFocus)&&this.options.showHintOnFocus,this.afterSelect=this.options.afterSelect,this.addItem=!1,this.value=this.$element.val()||this.$element.text()};b.prototype={constructor:b,select:function(){var a=this.$menu.find(".active").data("value");if(this.$element.data("active",a),this.autoSelect||a){var b=this.updater(a);b||(b=""),this.$element.val(this.displayText(b)||b).text(this.displayText(b)||b).change(),this.afterSelect(b)}return this.hide()},updater:function(a){return a},setSource:function(a){this.source=a},show:function(){var d,b=a.extend({},this.$element.position(),{height:this.$element[0].offsetHeight}),c="function"==typeof this.options.scrollHeight?this.options.scrollHeight.call():this.options.scrollHeight;if(this.shown?d=this.$menu:this.$appendTo?(d=this.$menu.appendTo(this.$appendTo),this.hasSameParent=this.$appendTo.is(this.$element.parent())):(d=this.$menu.insertAfter(this.$element),this.hasSameParent=!0),!this.hasSameParent){d.css("position","fixed");var e=this.$element.offset();b.top=e.top,b.left=e.left}var f=a(d).parent().hasClass("dropup"),g=f?"auto":b.top+b.height+c,h=a(d).hasClass("dropdown-menu-right"),i=h?"auto":b.left;return d.css({top:g,left:i}).show(),this.options.fitToElement===!0&&d.css("width",this.$element.outerWidth()+"px"),this.shown=!0,this},hide:function(){return this.$menu.hide(),this.shown=!1,this},lookup:function(b){if("undefined"!=typeof b&&null!==b?this.query=b:this.query=this.$element.val()||this.$element.text()||"",this.query.length<this.options.minLength&&!this.options.showHintOnFocus)return this.shown?this.hide():this;var d=a.proxy(function(){a.isFunction(this.source)?this.source(this.query,a.proxy(this.process,this)):this.source&&this.process(this.source)},this);clearTimeout(this.lookupWorker),this.lookupWorker=setTimeout(d,this.delay)},process:function(b){var c=this;return b=a.grep(b,function(a){return c.matcher(a)}),b=this.sorter(b),b.length||this.options.addItem?(b.length>0?this.$element.data("active",b[0]):this.$element.data("active",null),this.options.addItem&&b.push(this.options.addItem),"all"==this.options.items?this.render(b).show():this.render(b.slice(0,this.options.items)).show()):this.shown?this.hide():this},matcher:function(a){var b=this.displayText(a);return~b.toLowerCase().indexOf(this.query.toLowerCase())},sorter:function(a){for(var e,b=[],c=[],d=[];e=a.shift();){var f=this.displayText(e);f.toLowerCase().indexOf(this.query.toLowerCase())?~f.indexOf(this.query)?c.push(e):d.push(e):b.push(e)}return b.concat(c,d)},highlighter:function(a){var b=this.query;if(""===b)return a;var f,c=a.match(/(>)([^<]*)(<)/g),d=[],e=[];if(c&&c.length)for(f=0;f<c.length;++f)c[f].length>2&&d.push(c[f]);else d=[],d.push(a);b = b.replace((/[\(\)\/\.\*\+\?\[\]]/g), function(m) {return '\\'+m;});var h,g=new RegExp(b,"g");for(f=0;f<d.length;++f)h=d[f].match(g),h&&h.length>0&&e.push(d[f]);for(f=0;f<e.length;++f)a=a.replace(e[f],e[f].replace(g,"<strong>$&</strong>"));return a},render:function(b){var c=this,d=this,e=!1,f=[],g=c.options.separator;return a.each(b,function(a,c){a>0&&c[g]!==b[a-1][g]&&f.push({__type:"divider"}),!c[g]||0!==a&&c[g]===b[a-1][g]||f.push({__type:"category",name:c[g]}),f.push(c)}),b=a(f).map(function(b,f){if("category"==(f.__type||!1))return a(c.options.headerHtml).text(f.name)[0];if("divider"==(f.__type||!1))return a(c.options.headerDivider)[0];var g=d.displayText(f);return b=a(c.options.item).data("value",f),b.find("a").html(c.highlighter(g,f)),g==d.$element.val()&&(b.addClass("active"),d.$element.data("active",f),e=!0),b[0]}),this.autoSelect&&!e&&(b.filter(":not(.dropdown-header)").first().addClass("active"),this.$element.data("active",b.first().data("value"))),this.$menu.html(b),this},displayText:function(a){return"undefined"!=typeof a&&"undefined"!=typeof a.name?a.name:a},next:function(b){var c=this.$menu.find(".active").removeClass("active"),d=c.next();d.length||(d=a(this.$menu.find("li")[0])),d.addClass("active")},prev:function(a){var b=this.$menu.find(".active").removeClass("active"),c=b.prev();c.length||(c=this.$menu.find("li").last()),c.addClass("active")},listen:function(){this.$element.on("focus",a.proxy(this.focus,this)).on("blur",a.proxy(this.blur,this)).on("keypress",a.proxy(this.keypress,this)).on("input",a.proxy(this.input,this)).on("keyup",a.proxy(this.keyup,this)),this.eventSupported("keydown")&&this.$element.on("keydown",a.proxy(this.keydown,this)),this.$menu.on("click",a.proxy(this.click,this)).on("mouseenter","li",a.proxy(this.mouseenter,this)).on("mouseleave","li",a.proxy(this.mouseleave,this)).on("mousedown",a.proxy(this.mousedown,this))},destroy:function(){this.$element.data("typeahead",null),this.$element.data("active",null),this.$element.off("focus").off("blur").off("keypress").off("input").off("keyup"),this.eventSupported("keydown")&&this.$element.off("keydown"),this.$menu.remove(),this.destroyed=!0},eventSupported:function(a){var b=a in this.$element;return b||(this.$element.setAttribute(a,"return;"),b="function"==typeof this.$element[a]),b},move:function(a){if(this.shown)switch(a.keyCode){case 9:case 13:case 27:a.preventDefault();break;case 38:if(a.shiftKey)return;a.preventDefault(),this.prev();break;case 40:if(a.shiftKey)return;a.preventDefault(),this.next()}},keydown:function(b){this.suppressKeyPressRepeat=~a.inArray(b.keyCode,[40,38,9,13,27]),this.shown||40!=b.keyCode?this.move(b):this.lookup()},keypress:function(a){this.suppressKeyPressRepeat||this.move(a)},input:function(a){var b=this.$element.val()||this.$element.text();this.value!==b&&(this.value=b,this.lookup())},keyup:function(a){if(!this.destroyed)switch(a.keyCode){case 40:case 38:case 16:case 17:case 18:break;case 9:case 13:if(!this.shown)return;this.select();break;case 27:if(!this.shown)return;this.hide()}},focus:function(a){this.focused||(this.focused=!0,this.options.showHintOnFocus&&this.skipShowHintOnFocus!==!0&&("all"===this.options.showHintOnFocus?this.lookup(""):this.lookup())),this.skipShowHintOnFocus&&(this.skipShowHintOnFocus=!1)},blur:function(a){this.mousedover||this.mouseddown||!this.shown?this.mouseddown&&(this.skipShowHintOnFocus=!0,this.$element.focus(),this.mouseddown=!1):(this.hide(),this.focused=!1)},click:function(a){a.preventDefault(),this.skipShowHintOnFocus=!0,this.select(),this.$element.focus(),this.hide()},mouseenter:function(b){this.mousedover=!0,this.$menu.find(".active").removeClass("active"),a(b.currentTarget).addClass("active")},mouseleave:function(a){this.mousedover=!1,!this.focused&&this.shown&&this.hide()},mousedown:function(a){this.mouseddown=!0,this.$menu.one("mouseup",function(a){this.mouseddown=!1}.bind(this))}};var c=a.fn.typeahead;a.fn.typeahead=function(c){var d=arguments;return"string"==typeof c&&"getActive"==c?this.data("active"):this.each(function(){var e=a(this),f=e.data("typeahead"),g="object"==typeof c&&c;f||e.data("typeahead",f=new b(this,g)),"string"==typeof c&&f[c]&&(d.length>1?f[c].apply(f,Array.prototype.slice.call(d,1)):f[c]())})},b.defaults={source:[],items:8,menu:'<ul class="typeahead dropdown-menu" role="listbox"></ul>',item:'<li><a class="dropdown-item" href="#" role="option"></a></li>',minLength:1,scrollHeight:0,autoSelect:!0,afterSelect:a.noop,addItem:!1,delay:0,separator:"category",headerHtml:'<li class="dropdown-header"></li>',headerDivider:'<li class="divider" role="separator"></li>'},a.fn.typeahead.Constructor=b,a.fn.typeahead.noConflict=function(){return a.fn.typeahead=c,this},a(document).on("focus.typeahead.data-api",'[data-provide="typeahead"]',function(b){var c=a(this);c.data("typeahead")||c.typeahead(c.data())})});
diff --git a/airflow/www_rbac/static/connection_form.js b/airflow/www_rbac/static/connection_form.js
deleted file mode 100644
index b78d1d60eeb86..0000000000000
--- a/airflow/www_rbac/static/connection_form.js
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-/**
- * Created by janomar on 23/07/15.
- */
-
-  $(document).ready(function() {
-      var config = {
-        jdbc: {
-            hidden_fields: ['port', 'schema', 'extra'],
-            relabeling: {'host': 'Connection URL'},
-        },
-        google_cloud_platform: {
-            hidden_fields: ['host', 'schema', 'login', 'password', 'port', 'extra'],
-            relabeling: {},
-        },
-        cloudant: {
-            hidden_fields: ['port', 'extra'],
-            relabeling: {
-                'host': 'Account',
-                'login': 'Username (or API Key)',
-                'schema': 'Database'
-            }
-        },
-        docker: {
-            hidden_fields: ['port', 'schema'],
-            relabeling: {
-                'host': 'Registry URL',
-                'login': 'Username',
-            },
-        },
-      }
-      function connTypeChange(connectionType) {
-        $(".hide").removeClass("hide");
-        $.each($("[id^='extra__']"), function() {
-            $(this).parent().parent().addClass('hide')
-        });
-        $.each($("[id^='extra__"+connectionType+"']"), function() {
-            $(this).parent().parent().removeClass('hide')
-        });
-        $("label[orig_text]").each(function(){
-            $(this).text($(this).attr("orig_text"));
-        });
-        if (config[connectionType] != undefined){
-          $.each(config[connectionType].hidden_fields, function(i, field){
-            $("#" + field).parent().parent().addClass('hide')
-          });
-          $.each(config[connectionType].relabeling, function(k, v){
-            lbl = $("label[for='" + k + "']")
-            lbl.attr("orig_text", lbl.text());
-            $("label[for='" + k + "']").text(v);
-          });
-        }
-      }
-      var connectionType=$("#conn_type").val();
-      $("#conn_type").on('change', function(e) {
-        connectionType = $("#conn_type").val();
-        connTypeChange(connectionType);
-      });
-      connTypeChange(connectionType);
-});
diff --git a/airflow/www_rbac/static/bootstrap-theme.css b/airflow/www_rbac/static/css/bootstrap-theme.css
similarity index 99%
rename from airflow/www_rbac/static/bootstrap-theme.css
rename to airflow/www_rbac/static/css/bootstrap-theme.css
index 734f940feed61..3adaec1d3b80d 100644
--- a/airflow/www_rbac/static/bootstrap-theme.css
+++ b/airflow/www_rbac/static/css/bootstrap-theme.css
@@ -4949,6 +4949,28 @@ a.thumbnail.active {
 .alert-danger .alert-link {
   color: #843534;
 }
+.alert-message {
+  background-color: #d9edf7;
+  border-color: #bce8f1;
+  color: #31708f;
+}
+.alert-message hr {
+  border-top-color: #a6e1ec;
+}
+.alert-message .alert-link {
+  color: #245269;
+}
+.alert-error {
+  background-color: #f2dede;
+  border-color: #ebccd1;
+  color: #a94442;
+}
+.alert-error hr {
+  border-top-color: #e4b9c0;
+}
+.alert-error .alert-link {
+  color: #843534;
+}
 @-webkit-keyframes progress-bar-stripes {
   from {
     background-position: 40px 0;
diff --git a/airflow/www_rbac/static/gantt.css b/airflow/www_rbac/static/css/gantt.css
similarity index 100%
rename from airflow/www_rbac/static/gantt.css
rename to airflow/www_rbac/static/css/gantt.css
diff --git a/airflow/www_rbac/static/graph.css b/airflow/www_rbac/static/css/graph.css
similarity index 81%
rename from airflow/www_rbac/static/graph.css
rename to airflow/www_rbac/static/css/graph.css
index f1d34800612fa..12de132d38ab6 100644
--- a/airflow/www_rbac/static/graph.css
+++ b/airflow/www_rbac/static/css/graph.css
@@ -17,11 +17,34 @@
 * under the License.
 */
 
+svg {
+    overflow: hidden;
+}
+
+.node rect {
+    stroke: #333;
+    stroke-width: 1px;
+    fill: #fff;
+}
+
+.edgeLabel rect {
+    fill: #fff;
+}
+
+.edgePath {
+    stroke: #333;
+    stroke-width: 1px;
+}
+
 g.node rect {
-    stroke-width: 3;
+    stroke-width: 3px;
     stroke: white;
     cursor: pointer;
 }
+g.node .label {
+  font-size: inherit;
+  font-weight: normal;
+}
 g.node text {
     cursor: pointer;
 }
@@ -31,11 +54,12 @@ g.node.success rect {
 g.node.up_for_retry rect {
     stroke: gold;
 }
-
+g.node.up_for_reschedule rect{
+    stroke: turquoise;
+}
 g.node.queued rect {
     stroke: grey;
 }
-
 g.node.running rect{
     stroke: lime;
 }
diff --git a/airflow/www_rbac/static/main.css b/airflow/www_rbac/static/css/main.css
similarity index 94%
rename from airflow/www_rbac/static/main.css
rename to airflow/www_rbac/static/css/main.css
index 5a10bfd34dc62..b3119e56247f2 100644
--- a/airflow/www_rbac/static/main.css
+++ b/airflow/www_rbac/static/css/main.css
@@ -47,10 +47,14 @@ td>span.glyphicon{
     padding-left: 3px;
     padding-top: 3px;
 }
-button.btn {
+button.btn, label.btn {
     border: 1px solid black;
 }
 
+.btn-group label.btn {
+    background-color: #f0f0f0;
+}
+
 div.rich_doc {
     padding: 5px 10px;
     border: 1px solid #dddddd;
@@ -78,6 +82,9 @@ span.success{
 span.up_for_retry{
     background-color: gold;
 }
+span.up_for_reschedule{
+    background-color: turquoise;
+}
 span.started{
     background-color: lime;
 }
@@ -162,13 +169,13 @@ table.dataframe.dataTable thead > tr > th {
     padding: 10px 20px 10px 10px;
 }
 table.dataTable.dataframe thead .sorting {
-    background: url('sort_both.png') no-repeat center right
+    background: url('./../sort_both.png') no-repeat center right
 }
 table.dataTable.dataframe thead .sorting_desc {
-    background: url('sort_desc.png') no-repeat center right
+    background: url('./../sort_desc.png') no-repeat center right
 }
 table.dataTable.dataframe thead .sorting_asc {
-    background: url('sort_asc.png') no-repeat center right
+    background: url('./../sort_asc.png') no-repeat center right
 }
 .no-wrap {
     white-space: nowrap;
@@ -265,3 +272,4 @@ div.square {
 .sc { color: #BA2121 } /* Literal.String.Char */
 .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
 .s2 { color: #BA2121 } /* Literal.String.Double */
+.s1 { color: #BA2121 } /* Literal.String.Single */
diff --git a/airflow/www_rbac/static/tree.css b/airflow/www_rbac/static/css/tree.css
similarity index 97%
rename from airflow/www_rbac/static/tree.css
rename to airflow/www_rbac/static/css/tree.css
index 9304bb1c55f78..17ff748b59a94 100644
--- a/airflow/www_rbac/static/tree.css
+++ b/airflow/www_rbac/static/css/tree.css
@@ -62,6 +62,9 @@ rect.upstream_failed {
 rect.up_for_retry {
     fill: gold;
 }
+rect.up_for_reschedule {
+    fill: turquoise;
+}
 rect.skipped {
     fill: pink;
 }
diff --git a/airflow/www_rbac/static/d3.tip.v0.6.3.js b/airflow/www_rbac/static/d3.tip.v0.6.3.js
deleted file mode 100644
index b2b48fbfeb0c0..0000000000000
--- a/airflow/www_rbac/static/d3.tip.v0.6.3.js
+++ /dev/null
@@ -1,302 +0,0 @@
-/**
- * The MIT License (MIT)
- * Copyright (c) 2013 Justin Palmer
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal in
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- * of the Software, and to permit persons to whom the Software is furnished to do
- * so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
- * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
- * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-// d3.tip
-// Copyright (c) 2013 Justin Palmer
-//
-// Tooltips for d3.js SVG visualizations
-
-// Public - contructs a new tooltip
-//
-// Returns a tip
-d3.tip = function() {
-  var direction = d3_tip_direction,
-      offset    = d3_tip_offset,
-      html      = d3_tip_html,
-      node      = initNode(),
-      svg       = null,
-      point     = null,
-      target    = null
-
-  function tip(vis) {
-    svg = getSVGNode(vis)
-    point = svg.createSVGPoint()
-    document.body.appendChild(node)
-  }
-
-  // Public - show the tooltip on the screen
-  //
-  // Returns a tip
-  tip.show = function() {
-    var args = Array.prototype.slice.call(arguments)
-    if(args[args.length - 1] instanceof SVGElement) target = args.pop()
-
-    var content = html.apply(this, args),
-        poffset = offset.apply(this, args),
-        dir     = direction.apply(this, args),
-        nodel   = d3.select(node), i = 0,
-        coords
-
-    nodel.html(content)
-      .style({ opacity: 1, 'pointer-events': 'all' })
-
-    while(i--) nodel.classed(directions[i], false)
-    coords = direction_callbacks.get(dir).apply(this)
-    nodel.classed(dir, true).style({
-      top: (coords.top +  poffset[0]) + 'px',
-      left: (coords.left + poffset[1]) + 'px'
-    })
-
-    return tip
-  }
-
-  // Public - hide the tooltip
-  //
-  // Returns a tip
-  tip.hide = function() {
-    nodel = d3.select(node)
-    nodel.style({ opacity: 0, 'pointer-events': 'none' })
-    return tip
-  }
-
-  // Public: Proxy attr calls to the d3 tip container.  Sets or gets attribute value.
-  //
-  // n - name of the attribute
-  // v - value of the attribute
-  //
-  // Returns tip or attribute value
-  tip.attr = function(n, v) {
-    if (arguments.length < 2 && typeof n === 'string') {
-      return d3.select(node).attr(n)
-    } else {
-      var args =  Array.prototype.slice.call(arguments)
-      d3.selection.prototype.attr.apply(d3.select(node), args)
-    }
-
-    return tip
-  }
-
-  // Public: Proxy style calls to the d3 tip container.  Sets or gets a style value.
-  //
-  // n - name of the property
-  // v - value of the property
-  //
-  // Returns tip or style property value
-  tip.style = function(n, v) {
-    if (arguments.length < 2 && typeof n === 'string') {
-      return d3.select(node).style(n)
-    } else {
-      var args =  Array.prototype.slice.call(arguments)
-      d3.selection.prototype.style.apply(d3.select(node), args)
-    }
-
-    return tip
-  }
-
-  // Public: Set or get the direction of the tooltip
-  //
-  // v - One of n(north), s(south), e(east), or w(west), nw(northwest),
-  //     sw(southwest), ne(northeast) or se(southeast)
-  //
-  // Returns tip or direction
-  tip.direction = function(v) {
-    if (!arguments.length) return direction
-    direction = v == null ? v : d3.functor(v)
-
-    return tip
-  }
-
-  // Public: Sets or gets the offset of the tip
-  //
-  // v - Array of [x, y] offset
-  //
-  // Returns offset or
-  tip.offset = function(v) {
-    if (!arguments.length) return offset
-    offset = v == null ? v : d3.functor(v)
-
-    return tip
-  }
-
-  // Public: sets or gets the html value of the tooltip
-  //
-  // v - String value of the tip
-  //
-  // Returns html value or tip
-  tip.html = function(v) {
-    if (!arguments.length) return html
-    html = v == null ? v : d3.functor(v)
-
-    return tip
-  }
-
-  function d3_tip_direction() { return 'n' }
-  function d3_tip_offset() { return [0, 0] }
-  function d3_tip_html() { return ' ' }
-
-  var direction_callbacks = d3.map({
-    n:  direction_n,
-    s:  direction_s,
-    e:  direction_e,
-    w:  direction_w,
-    nw: direction_nw,
-    ne: direction_ne,
-    sw: direction_sw,
-    se: direction_se
-  }),
-
-  directions = direction_callbacks.keys()
-
-  function direction_n() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.n.y - node.offsetHeight,
-      left: bbox.n.x - node.offsetWidth / 2
-    }
-  }
-
-  function direction_s() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.s.y,
-      left: bbox.s.x - node.offsetWidth / 2
-    }
-  }
-
-  function direction_e() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.e.y - node.offsetHeight / 2,
-      left: bbox.e.x
-    }
-  }
-
-  function direction_w() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.w.y - node.offsetHeight / 2,
-      left: bbox.w.x - node.offsetWidth
-    }
-  }
-
-  function direction_nw() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.nw.y - node.offsetHeight,
-      left: bbox.nw.x - node.offsetWidth
-    }
-  }
-
-  function direction_ne() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.ne.y - node.offsetHeight,
-      left: bbox.ne.x
-    }
-  }
-
-  function direction_sw() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.sw.y,
-      left: bbox.sw.x - node.offsetWidth
-    }
-  }
-
-  function direction_se() {
-    var bbox = getScreenBBox()
-    return {
-      top:  bbox.se.y,
-      left: bbox.e.x
-    }
-  }
-
-  function initNode() {
-    var node = d3.select(document.createElement('div'))
-    node.style({
-      position: 'absolute',
-      opacity: 0,
-      pointerEvents: 'none',
-      boxSizing: 'border-box'
-    })
-
-    return node.node()
-  }
-
-  function getSVGNode(el) {
-    el = el.node()
-    if(el.tagName.toLowerCase() == 'svg')
-      return el
-
-    return el.ownerSVGElement
-  }
-
-  // Private - gets the screen coordinates of a shape
-  //
-  // Given a shape on the screen, will return an SVGPoint for the directions
-  // n(north), s(south), e(east), w(west), ne(northeast), se(southeast), nw(northwest),
-  // sw(southwest).
-  //
-  //    +-+-+
-  //    |   |
-  //    +   +
-  //    |   |
-  //    +-+-+
-  //
-  // Returns an Object {n, s, e, w, nw, sw, ne, se}
-  function getScreenBBox() {
-    var targetel   = target || d3.event.target,
-        bbox       = {},
-        matrix     = targetel.getScreenCTM(),
-        tbbox      = targetel.getBBox(),
-        width      = tbbox.width,
-        height     = tbbox.height,
-        x          = tbbox.x,
-        y          = tbbox.y,
-        scrollTop  = document.documentElement.scrollTop || document.body.scrollTop,
-        scrollLeft = document.documentElement.scrollLeft || document.body.scrollLeft
-
-
-    point.x = x + scrollLeft
-    point.y = y + scrollTop
-    bbox.nw = point.matrixTransform(matrix)
-    point.x += width
-    bbox.ne = point.matrixTransform(matrix)
-    point.y += height
-    bbox.se = point.matrixTransform(matrix)
-    point.x -= width
-    bbox.sw = point.matrixTransform(matrix)
-    point.y -= height / 2
-    bbox.w  = point.matrixTransform(matrix)
-    point.x += width
-    bbox.e = point.matrixTransform(matrix)
-    point.x -= width / 2
-    point.y -= height / 2
-    bbox.n = point.matrixTransform(matrix)
-    point.y += height
-    bbox.s = point.matrixTransform(matrix)
-
-    return bbox
-  }
-
-  return tip
-};
diff --git a/airflow/www_rbac/static/d3.v3.min.js b/airflow/www_rbac/static/d3.v3.min.js
deleted file mode 100644
index 77615c0cf1c4c..0000000000000
--- a/airflow/www_rbac/static/d3.v3.min.js
+++ /dev/null
@@ -1,5 +0,0 @@
-d3=function(){function n(n){return null!=n&&!isNaN(n)}function t(n){return n.length}function e(n){for(var t=1;n*t%1;)t*=10;return t}function r(n,t){try{for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}catch(r){n.prototype=t}}function u(){}function i(){}function o(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function a(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.substring(1);for(var e=0,r=Do.length;r>e;++e){var u=Do[e]+t;if(u in n)return u}}function c(){}function l(){}function s(n){function t(){for(var t,r=e,u=-1,i=r.length;++u<i;)(t=r[u].on)&&t.apply(this,arguments);return n}var e=[],r=new u;return t.on=function(t,u){var i,o=r.get(t);return arguments.length<2?o&&o.on:(o&&(o.on=null,e=e.slice(0,i=e.indexOf(o)).concat(e.slice(i+1)),r.remove(t)),u&&e.push(r.set(t,{on:u})),n)},t}function f(){mo.event.preventDefault()}function h(){for(var n,t=mo.event;n=t.sourceEvent;)t=n;return t}function g(n){for(var t=new l,e=0,r=arguments.length;++e<r;)t[arguments[e]]=s(t);return t.of=function(e,r){return function(u){try{var i=u.sourceEvent=mo.event;u.target=n,mo.event=u,t[u.type].apply(e,r)}finally{mo.event=i}}},t}function p(n){return Lo(n,Ro),n}function d(n){return"function"==typeof n?n:function(){return Ho(n,this)}}function v(n){return"function"==typeof n?n:function(){return Fo(n,this)}}function m(n,t){function e(){this.removeAttribute(n)}function r(){this.removeAttributeNS(n.space,n.local)}function u(){this.setAttribute(n,t)}function i(){this.setAttributeNS(n.space,n.local,t)}function o(){var e=t.apply(this,arguments);null==e?this.removeAttribute(n):this.setAttribute(n,e)}function a(){var e=t.apply(this,arguments);null==e?this.removeAttributeNS(n.space,n.local):this.setAttributeNS(n.space,n.local,e)}return n=mo.ns.qualify(n),null==t?n.local?r:e:"function"==typeof t?n.local?a:o:n.local?i:u}function y(n){return n.trim().replace(/\s+/g," ")}function M(n){return new RegExp("(?:^|\\s+)"+mo.requote(n)+"(?:\\s+|$)","g")}function x(n,t){function e(){for(var e=-1;++e<u;)n[e](this,t)}function r(){for(var e=-1,r=t.apply(this,arguments);++e<u;)n[e](this,r)}n=n.trim().split(/\s+/).map(b);var u=n.length;return"function"==typeof t?r:e}function b(n){var t=M(n);return function(e,r){if(u=e.classList)return r?u.add(n):u.remove(n);var u=e.getAttribute("class")||"";r?(t.lastIndex=0,t.test(u)||e.setAttribute("class",y(u+" "+n))):e.setAttribute("class",y(u.replace(t," ")))}}function _(n,t,e){function r(){this.style.removeProperty(n)}function u(){this.style.setProperty(n,t,e)}function i(){var r=t.apply(this,arguments);null==r?this.style.removeProperty(n):this.style.setProperty(n,r,e)}return null==t?r:"function"==typeof t?i:u}function w(n,t){function e(){delete this[n]}function r(){this[n]=t}function u(){var e=t.apply(this,arguments);null==e?delete this[n]:this[n]=e}return null==t?e:"function"==typeof t?u:r}function S(n){return"function"==typeof n?n:(n=mo.ns.qualify(n)).local?function(){return xo.createElementNS(n.space,n.local)}:function(){return xo.createElementNS(this.namespaceURI,n)}}function E(n){return{__data__:n}}function k(n){return function(){return Oo(this,n)}}function A(n){return arguments.length||(n=mo.ascending),function(t,e){return t&&e?n(t.__data__,e.__data__):!t-!e}}function N(n,t){for(var e=0,r=n.length;r>e;e++)for(var u,i=n[e],o=0,a=i.length;a>o;o++)(u=i[o])&&t(u,o,e);return n}function T(n){return Lo(n,Io),n}function q(n){var t,e;return function(r,u,i){var o,a=n[i].update,c=a.length;for(i!=e&&(e=i,t=0),u>=t&&(t=u+1);!(o=a[t])&&++t<c;);return o}}function z(){var n=this.__transition__;n&&++n.active}function C(n,t,e){function r(){var t=this[o];t&&(this.removeEventListener(n,t,t.$),delete this[o])}function u(){var u=l(t,Mo(arguments));r.call(this),this.addEventListener(n,this[o]=u,u.$=e),u._=t}function i(){var t,e=new RegExp("^__on([^.]+)"+mo.requote(n)+"$");for(var r in this)if(t=r.match(e)){var u=this[r];this.removeEventListener(t[1],u,u.$),delete this[r]}}var o="__on"+n,a=n.indexOf("."),l=D;a>0&&(n=n.substring(0,a));var s=Zo.get(n);return s&&(n=s,l=j),a?t?u:r:t?c:i}function D(n,t){return function(e){var r=mo.event;mo.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{mo.event=r}}}function j(n,t){var e=D(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function L(){var n=".dragsuppress-"+ ++Xo,t="touchmove"+n,e="selectstart"+n,r="dragstart"+n,u="click"+n,i=mo.select(_o).on(t,f).on(e,f).on(r,f),o=bo.style,a=o[Vo];return o[Vo]="none",function(t){function e(){i.on(u,null)}i.on(n,null),o[Vo]=a,t&&(i.on(u,function(){f(),e()},!0),setTimeout(e,0))}}function H(n,t){t.changedTouches&&(t=t.changedTouches[0]);var e=n.ownerSVGElement||n;if(e.createSVGPoint){var r=e.createSVGPoint();if(0>$o&&(_o.scrollX||_o.scrollY)){e=mo.select("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var u=e[0][0].getScreenCTM();$o=!(u.f||u.e),e.remove()}return $o?(r.x=t.pageX,r.y=t.pageY):(r.x=t.clientX,r.y=t.clientY),r=r.matrixTransform(n.getScreenCTM().inverse()),[r.x,r.y]}var i=n.getBoundingClientRect();return[t.clientX-i.left-n.clientLeft,t.clientY-i.top-n.clientTop]}function F(n){return n>0?1:0>n?-1:0}function P(n){return n>1?0:-1>n?Bo:Math.acos(n)}function O(n){return n>1?Jo:-1>n?-Jo:Math.asin(n)}function R(n){return((n=Math.exp(n))-1/n)/2}function Y(n){return((n=Math.exp(n))+1/n)/2}function I(n){return((n=Math.exp(2*n))-1)/(n+1)}function U(n){return(n=Math.sin(n/2))*n}function Z(){}function V(n,t,e){return new X(n,t,e)}function X(n,t,e){this.h=n,this.s=t,this.l=e}function $(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?i+(o-i)*n/60:180>n?o:240>n?i+(o-i)*(240-n)/60:i}function u(n){return Math.round(255*r(n))}var i,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,i=2*e-o,ot(u(n+120),u(n),u(n-120))}function B(n,t,e){return new W(n,t,e)}function W(n,t,e){this.h=n,this.c=t,this.l=e}function J(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),G(e,Math.cos(n*=Qo)*t,Math.sin(n)*t)}function G(n,t,e){return new K(n,t,e)}function K(n,t,e){this.l=n,this.a=t,this.b=e}function Q(n,t,e){var r=(n+16)/116,u=r+t/500,i=r-e/200;return u=tt(u)*sa,r=tt(r)*fa,i=tt(i)*ha,ot(rt(3.2404542*u-1.5371385*r-.4985314*i),rt(-.969266*u+1.8760108*r+.041556*i),rt(.0556434*u-.2040259*r+1.0572252*i))}function nt(n,t,e){return n>0?B(Math.atan2(e,t)*na,Math.sqrt(t*t+e*e),n):B(0/0,0/0,n)}function tt(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function et(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function rt(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function ut(n){return ot(n>>16,255&n>>8,255&n)}function it(n){return ut(n)+""}function ot(n,t,e){return new at(n,t,e)}function at(n,t,e){this.r=n,this.g=t,this.b=e}function ct(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function lt(n,t,e){var r,u,i,o=0,a=0,c=0;if(r=/([a-z]+)\((.*)\)/i.exec(n))switch(u=r[2].split(","),r[1]){case"hsl":return e(parseFloat(u[0]),parseFloat(u[1])/100,parseFloat(u[2])/100);case"rgb":return t(gt(u[0]),gt(u[1]),gt(u[2]))}return(i=da.get(n))?t(i.r,i.g,i.b):(null!=n&&"#"===n.charAt(0)&&(4===n.length?(o=n.charAt(1),o+=o,a=n.charAt(2),a+=a,c=n.charAt(3),c+=c):7===n.length&&(o=n.substring(1,3),a=n.substring(3,5),c=n.substring(5,7)),o=parseInt(o,16),a=parseInt(a,16),c=parseInt(c,16)),t(o,a,c))}function st(n,t,e){var r,u,i=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-i,c=(o+i)/2;return a?(u=.5>c?a/(o+i):a/(2-o-i),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=0/0,u=c>0&&1>c?0:r),V(r,u,c)}function ft(n,t,e){n=ht(n),t=ht(t),e=ht(e);var r=et((.4124564*n+.3575761*t+.1804375*e)/sa),u=et((.2126729*n+.7151522*t+.072175*e)/fa),i=et((.0193339*n+.119192*t+.9503041*e)/ha);return G(116*u-16,500*(r-u),200*(u-i))}function ht(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function gt(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function pt(n){return"function"==typeof n?n:function(){return n}}function dt(n){return n}function vt(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),mt(t,e,n,r)}}function mt(n,t,e,r){function u(){var n,t=c.status;if(!t&&c.responseText||t>=200&&300>t||304===t){try{n=e.call(i,c)}catch(r){return o.error.call(i,r),void 0}o.load.call(i,n)}else o.error.call(i,c)}var i={},o=mo.dispatch("beforesend","progress","load","error"),a={},c=new XMLHttpRequest,l=null;return!_o.XDomainRequest||"withCredentials"in c||!/^(http(s)?:)?\/\//.test(n)||(c=new XDomainRequest),"onload"in c?c.onload=c.onerror=u:c.onreadystatechange=function(){c.readyState>3&&u()},c.onprogress=function(n){var t=mo.event;mo.event=n;try{o.progress.call(i,c)}finally{mo.event=t}},i.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",i)},i.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",i):t},i.responseType=function(n){return arguments.length?(l=n,i):l},i.response=function(n){return e=n,i},["get","post"].forEach(function(n){i[n]=function(){return i.send.apply(i,[n].concat(Mo(arguments)))}}),i.send=function(e,r,u){if(2===arguments.length&&"function"==typeof r&&(u=r,r=null),c.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),c.setRequestHeader)for(var s in a)c.setRequestHeader(s,a[s]);return null!=t&&c.overrideMimeType&&c.overrideMimeType(t),null!=l&&(c.responseType=l),null!=u&&i.on("error",u).on("load",function(n){u(null,n)}),o.beforesend.call(i,c),c.send(null==r?null:r),i},i.abort=function(){return c.abort(),i},mo.rebind(i,o,"on"),null==r?i:i.get(yt(r))}function yt(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function Mt(){var n=bt(),t=_t()-n;t>24?(isFinite(t)&&(clearTimeout(Ma),Ma=setTimeout(Mt,t)),ya=0):(ya=1,ba(Mt))}function xt(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now()),xa.callback=n,xa.time=e+t}function bt(){var n=Date.now();for(xa=va;xa;)n>=xa.time&&(xa.flush=xa.callback(n-xa.time)),xa=xa.next;return n}function _t(){for(var n,t=va,e=1/0;t;)t.flush?t=n?n.next=t.next:va=t.next:(t.time<e&&(e=t.time),t=(n=t).next);return ma=n,e}function wt(n,t){var e=Math.pow(10,3*Math.abs(8-t));return{scale:t>8?function(n){return n/e}:function(n){return n*e},symbol:n}}function St(n,t){return t-(n?Math.ceil(Math.log(n)/Math.LN10):1)}function Et(n){return n+""}function kt(){}function At(n,t,e){var r=e.s=n+t,u=r-n,i=r-u;e.t=n-i+(t-u)}function Nt(n,t){n&&Da.hasOwnProperty(n.type)&&Da[n.type](n,t)}function Tt(n,t,e){var r,u=-1,i=n.length-e;for(t.lineStart();++u<i;)r=n[u],t.point(r[0],r[1],r[2]);t.lineEnd()}function qt(n,t){var e=-1,r=n.length;for(t.polygonStart();++e<r;)Tt(n[e],t,1);t.polygonEnd()}function zt(){function n(n,t){n*=Qo,t=t*Qo/2+Bo/4;var e=n-r,o=Math.cos(t),a=Math.sin(t),c=i*a,l=u*o+c*Math.cos(e),s=c*Math.sin(e);La.add(Math.atan2(s,l)),r=n,u=o,i=a}var t,e,r,u,i;Ha.point=function(o,a){Ha.point=n,r=(t=o)*Qo,u=Math.cos(a=(e=a)*Qo/2+Bo/4),i=Math.sin(a)},Ha.lineEnd=function(){n(t,e)}}function Ct(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function Dt(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function jt(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function Lt(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function Ht(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function Ft(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function Pt(n){return[Math.atan2(n[1],n[0]),O(n[2])]}function Ot(n,t){return Math.abs(n[0]-t[0])<Go&&Math.abs(n[1]-t[1])<Go}function Rt(n,t){n*=Qo;var e=Math.cos(t*=Qo);Yt(e*Math.cos(n),e*Math.sin(n),Math.sin(t))}function Yt(n,t,e){++Fa,Oa+=(n-Oa)/Fa,Ra+=(t-Ra)/Fa,Ya+=(e-Ya)/Fa}function It(){function n(n,u){n*=Qo;var i=Math.cos(u*=Qo),o=i*Math.cos(n),a=i*Math.sin(n),c=Math.sin(u),l=Math.atan2(Math.sqrt((l=e*c-r*a)*l+(l=r*o-t*c)*l+(l=t*a-e*o)*l),t*o+e*a+r*c);Pa+=l,Ia+=l*(t+(t=o)),Ua+=l*(e+(e=a)),Za+=l*(r+(r=c)),Yt(t,e,r)}var t,e,r;Ba.point=function(u,i){u*=Qo;var o=Math.cos(i*=Qo);t=o*Math.cos(u),e=o*Math.sin(u),r=Math.sin(i),Ba.point=n,Yt(t,e,r)}}function Ut(){Ba.point=Rt}function Zt(){function n(n,t){n*=Qo;var e=Math.cos(t*=Qo),o=e*Math.cos(n),a=e*Math.sin(n),c=Math.sin(t),l=u*c-i*a,s=i*o-r*c,f=r*a-u*o,h=Math.sqrt(l*l+s*s+f*f),g=r*o+u*a+i*c,p=h&&-P(g)/h,d=Math.atan2(h,g);Va+=p*l,Xa+=p*s,$a+=p*f,Pa+=d,Ia+=d*(r+(r=o)),Ua+=d*(u+(u=a)),Za+=d*(i+(i=c)),Yt(r,u,i)}var t,e,r,u,i;Ba.point=function(o,a){t=o,e=a,Ba.point=n,o*=Qo;var c=Math.cos(a*=Qo);r=c*Math.cos(o),u=c*Math.sin(o),i=Math.sin(a),Yt(r,u,i)},Ba.lineEnd=function(){n(t,e),Ba.lineEnd=Ut,Ba.point=Rt}}function Vt(){return!0}function Xt(n,t,e,r,u){var i=[],o=[];if(n.forEach(function(n){if(!((t=n.length-1)<=0)){var t,e=n[0],r=n[t];if(Ot(e,r)){u.lineStart();for(var a=0;t>a;++a)u.point((e=n[a])[0],e[1]);return u.lineEnd(),void 0}var c={point:e,points:n,other:null,visited:!1,entry:!0,subject:!0},l={point:e,points:[e],other:c,visited:!1,entry:!1,subject:!1};c.other=l,i.push(c),o.push(l),c={point:r,points:[r],other:null,visited:!1,entry:!1,subject:!0},l={point:r,points:[r],other:c,visited:!1,entry:!0,subject:!1},c.other=l,i.push(c),o.push(l)}}),o.sort(t),$t(i),$t(o),i.length){for(var a=0,c=e,l=o.length;l>a;++a)o[a].entry=c=!c;for(var s,f,h,g=i[0];;){for(s=g;s.visited;)if((s=s.next)===g)return;f=s.points,u.lineStart();do{if(s.visited=s.other.visited=!0,s.entry){if(s.subject)for(var a=0;a<f.length;a++)u.point((h=f[a])[0],h[1]);else r(s.point,s.next.point,1,u);s=s.next}else{if(s.subject){f=s.prev.points;for(var a=f.length;--a>=0;)u.point((h=f[a])[0],h[1])}else r(s.point,s.prev.point,-1,u);s=s.prev}s=s.other,f=s.points}while(!s.visited);u.lineEnd()}}}function $t(n){if(t=n.length){for(var t,e,r=0,u=n[0];++r<t;)u.next=e=n[r],e.prev=u,u=e;u.next=e=n[0],e.prev=u}}function Bt(n,t,e,r){return function(u,i){function o(t,e){var r=u(t,e);n(t=r[0],e=r[1])&&i.point(t,e)}function a(n,t){var e=u(n,t);v.point(e[0],e[1])}function c(){y.point=a,v.lineStart()}function l(){y.point=o,v.lineEnd()}function s(n,t){d.push([n,t]);var e=u(n,t);x.point(e[0],e[1])}function f(){x.lineStart(),d=[]}function h(){s(d[0][0],d[0][1]),x.lineEnd();var n,t=x.clean(),e=M.buffer(),r=e.length;if(d.pop(),p.push(d),d=null,r){if(1&t){n=e[0];var u,r=n.length-1,o=-1;for(i.lineStart();++o<r;)i.point((u=n[o])[0],u[1]);return i.lineEnd(),void 0}r>1&&2&t&&e.push(e.pop().concat(e.shift())),g.push(e.filter(Wt))}}var g,p,d,v=t(i),m=u.invert(r[0],r[1]),y={point:o,lineStart:c,lineEnd:l,polygonStart:function(){y.point=s,y.lineStart=f,y.lineEnd=h,g=[],p=[],i.polygonStart()},polygonEnd:function(){y.point=o,y.lineStart=c,y.lineEnd=l,g=mo.merge(g);var n=Kt(m,p);g.length?Xt(g,Gt,n,e,i):n&&(i.lineStart(),e(null,null,1,i),i.lineEnd()),i.polygonEnd(),g=p=null},sphere:function(){i.polygonStart(),i.lineStart(),e(null,null,1,i),i.lineEnd(),i.polygonEnd()}},M=Jt(),x=t(M);return y}}function Wt(n){return n.length>1}function Jt(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:c,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function Gt(n,t){return((n=n.point)[0]<0?n[1]-Jo-Go:Jo-n[1])-((t=t.point)[0]<0?t[1]-Jo-Go:Jo-t[1])}function Kt(n,t){var e=n[0],r=n[1],u=[Math.sin(e),-Math.cos(e),0],i=0,o=0;La.reset();for(var a=0,c=t.length;c>a;++a){var l=t[a],s=l.length;if(s)for(var f=l[0],h=f[0],g=f[1]/2+Bo/4,p=Math.sin(g),d=Math.cos(g),v=1;;){v===s&&(v=0),n=l[v];var m=n[0],y=n[1]/2+Bo/4,M=Math.sin(y),x=Math.cos(y),b=m-h,_=Math.abs(b)>Bo,w=p*M;if(La.add(Math.atan2(w*Math.sin(b),d*x+w*Math.cos(b))),i+=_?b+(b>=0?2:-2)*Bo:b,_^h>=e^m>=e){var S=jt(Ct(f),Ct(n));Ft(S);var E=jt(u,S);Ft(E);var k=(_^b>=0?-1:1)*O(E[2]);(r>k||r===k&&(S[0]||S[1]))&&(o+=_^b>=0?1:-1)}if(!v++)break;h=m,p=M,d=x,f=n}}return(-Go>i||Go>i&&0>La)^1&o}function Qt(n){var t,e=0/0,r=0/0,u=0/0;return{lineStart:function(){n.lineStart(),t=1},point:function(i,o){var a=i>0?Bo:-Bo,c=Math.abs(i-e);Math.abs(c-Bo)<Go?(n.point(e,r=(r+o)/2>0?Jo:-Jo),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(i,r),t=0):u!==a&&c>=Bo&&(Math.abs(e-u)<Go&&(e-=u*Go),Math.abs(i-a)<Go&&(i-=a*Go),r=ne(e,r,i,o),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),t=0),n.point(e=i,r=o),u=a},lineEnd:function(){n.lineEnd(),e=r=0/0},clean:function(){return 2-t}}}function ne(n,t,e,r){var u,i,o=Math.sin(n-e);return Math.abs(o)>Go?Math.atan((Math.sin(t)*(i=Math.cos(r))*Math.sin(e)-Math.sin(r)*(u=Math.cos(t))*Math.sin(n))/(u*i*o)):(t+r)/2}function te(n,t,e,r){var u;if(null==n)u=e*Jo,r.point(-Bo,u),r.point(0,u),r.point(Bo,u),r.point(Bo,0),r.point(Bo,-u),r.point(0,-u),r.point(-Bo,-u),r.point(-Bo,0),r.point(-Bo,u);else if(Math.abs(n[0]-t[0])>Go){var i=(n[0]<t[0]?1:-1)*Bo;u=e*i/2,r.point(-i,u),r.point(0,u),r.point(i,u)}else r.point(t[0],t[1])}function ee(n){function t(n,t){return Math.cos(n)*Math.cos(t)>i}function e(n){var e,i,c,l,s;return{lineStart:function(){l=c=!1,s=1},point:function(f,h){var g,p=[f,h],d=t(f,h),v=o?d?0:u(f,h):d?u(f+(0>f?Bo:-Bo),h):0;if(!e&&(l=c=d)&&n.lineStart(),d!==c&&(g=r(e,p),(Ot(e,g)||Ot(p,g))&&(p[0]+=Go,p[1]+=Go,d=t(p[0],p[1]))),d!==c)s=0,d?(n.lineStart(),g=r(p,e),n.point(g[0],g[1])):(g=r(e,p),n.point(g[0],g[1]),n.lineEnd()),e=g;else if(a&&e&&o^d){var m;v&i||!(m=r(p,e,!0))||(s=0,o?(n.lineStart(),n.point(m[0][0],m[0][1]),n.point(m[1][0],m[1][1]),n.lineEnd()):(n.point(m[1][0],m[1][1]),n.lineEnd(),n.lineStart(),n.point(m[0][0],m[0][1])))}!d||e&&Ot(e,p)||n.point(p[0],p[1]),e=p,c=d,i=v},lineEnd:function(){c&&n.lineEnd(),e=null},clean:function(){return s|(l&&c)<<1}}}function r(n,t,e){var r=Ct(n),u=Ct(t),o=[1,0,0],a=jt(r,u),c=Dt(a,a),l=a[0],s=c-l*l;if(!s)return!e&&n;var f=i*c/s,h=-i*l/s,g=jt(o,a),p=Ht(o,f),d=Ht(a,h);Lt(p,d);var v=g,m=Dt(p,v),y=Dt(v,v),M=m*m-y*(Dt(p,p)-1);if(!(0>M)){var x=Math.sqrt(M),b=Ht(v,(-m-x)/y);if(Lt(b,p),b=Pt(b),!e)return b;var _,w=n[0],S=t[0],E=n[1],k=t[1];w>S&&(_=w,w=S,S=_);var A=S-w,N=Math.abs(A-Bo)<Go,T=N||Go>A;if(!N&&E>k&&(_=E,E=k,k=_),T?N?E+k>0^b[1]<(Math.abs(b[0]-w)<Go?E:k):E<=b[1]&&b[1]<=k:A>Bo^(w<=b[0]&&b[0]<=S)){var q=Ht(v,(-m+x)/y);return Lt(q,p),[b,Pt(q)]}}}function u(t,e){var r=o?n:Bo-n,u=0;return-r>t?u|=1:t>r&&(u|=2),-r>e?u|=4:e>r&&(u|=8),u}var i=Math.cos(n),o=i>0,a=Math.abs(i)>Go,c=Te(n,6*Qo);return Bt(t,e,c,o?[0,-n]:[-Bo,n-Bo])}function re(n,t,e,r){function u(r,u){return Math.abs(r[0]-n)<Go?u>0?0:3:Math.abs(r[0]-e)<Go?u>0?2:1:Math.abs(r[1]-t)<Go?u>0?1:0:u>0?3:2}function i(n,t){return o(n.point,t.point)}function o(n,t){var e=u(n,1),r=u(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}function a(u,i){var o=i[0]-u[0],a=i[1]-u[1],c=[0,1];return Math.abs(o)<Go&&Math.abs(a)<Go?n<=u[0]&&u[0]<=e&&t<=u[1]&&u[1]<=r:ue(n-u[0],o,c)&&ue(u[0]-e,-o,c)&&ue(t-u[1],a,c)&&ue(u[1]-r,-a,c)?(c[1]<1&&(i[0]=u[0]+c[1]*o,i[1]=u[1]+c[1]*a),c[0]>0&&(u[0]+=c[0]*o,u[1]+=c[0]*a),!0):!1}return function(c){function l(n){for(var t=0,e=y.length,r=n[1],u=0;e>u;++u)for(var i,o=1,a=y[u],c=a.length,l=a[0];c>o;++o)i=a[o],l[1]<=r?i[1]>r&&s(l,i,n)>0&&++t:i[1]<=r&&s(l,i,n)<0&&--t,l=i;return 0!==t}function s(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(e[0]-n[0])*(t[1]-n[1])}function f(i,a,c,l){var s=0,f=0;if(null==i||(s=u(i,c))!==(f=u(a,c))||o(i,a)<0^c>0){do l.point(0===s||3===s?n:e,s>1?r:t);while((s=(s+c+4)%4)!==f)}else l.point(a[0],a[1])}function h(u,i){return u>=n&&e>=u&&i>=t&&r>=i}function g(n,t){h(n,t)&&c.point(n,t)}function p(){q.point=v,y&&y.push(M=[]),k=!0,E=!1,w=S=0/0}function d(){m&&(v(x,b),_&&E&&T.rejoin(),m.push(T.buffer())),q.point=g,E&&c.lineEnd()}function v(n,t){n=Math.max(-Ja,Math.min(Ja,n)),t=Math.max(-Ja,Math.min(Ja,t));var e=h(n,t);if(y&&M.push([n,t]),k)x=n,b=t,_=e,k=!1,e&&(c.lineStart(),c.point(n,t));else if(e&&E)c.point(n,t);else{var r=[w,S],u=[n,t];a(r,u)?(E||(c.lineStart(),c.point(r[0],r[1])),c.point(u[0],u[1]),e||c.lineEnd(),A=!1):e&&(c.lineStart(),c.point(n,t),A=!1)}w=n,S=t,E=e}var m,y,M,x,b,_,w,S,E,k,A,N=c,T=Jt(),q={point:g,lineStart:p,lineEnd:d,polygonStart:function(){c=T,m=[],y=[],A=!0},polygonEnd:function(){c=N,m=mo.merge(m);var t=l([n,r]),e=A&&t,u=m.length;(e||u)&&(c.polygonStart(),e&&(c.lineStart(),f(null,null,1,c),c.lineEnd()),u&&Xt(m,i,t,f,c),c.polygonEnd()),m=y=M=null}};return q}}function ue(n,t,e){if(Math.abs(t)<Go)return 0>=n;var r=n/t;if(t>0){if(r>e[1])return!1;r>e[0]&&(e[0]=r)}else{if(r<e[0])return!1;r<e[1]&&(e[1]=r)}return!0}function ie(n,t){function e(e,r){return e=n(e,r),t(e[0],e[1])}return n.invert&&t.invert&&(e.invert=function(e,r){return e=t.invert(e,r),e&&n.invert(e[0],e[1])}),e}function oe(n){var t=0,e=Bo/3,r=be(n),u=r(t,e);return u.parallels=function(n){return arguments.length?r(t=n[0]*Bo/180,e=n[1]*Bo/180):[180*(t/Bo),180*(e/Bo)]},u}function ae(n,t){function e(n,t){var e=Math.sqrt(i-2*u*Math.sin(t))/u;return[e*Math.sin(n*=u),o-e*Math.cos(n)]}var r=Math.sin(n),u=(r+Math.sin(t))/2,i=1+r*(2*u-r),o=Math.sqrt(i)/u;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/u,O((i-(n*n+e*e)*u*u)/(2*u))]},e}function ce(){function n(n,t){Ka+=u*n-r*t,r=n,u=t}var t,e,r,u;rc.point=function(i,o){rc.point=n,t=r=i,e=u=o},rc.lineEnd=function(){n(t,e)}}function le(n,t){Qa>n&&(Qa=n),n>tc&&(tc=n),nc>t&&(nc=t),t>ec&&(ec=t)}function se(){function n(n,t){o.push("M",n,",",t,i)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function u(){o.push("Z")}var i=fe(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return i=fe(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function fe(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function he(n,t){Oa+=n,Ra+=t,++Ya}function ge(){function n(n,r){var u=n-t,i=r-e,o=Math.sqrt(u*u+i*i);Ia+=o*(t+n)/2,Ua+=o*(e+r)/2,Za+=o,he(t=n,e=r)}var t,e;ic.point=function(r,u){ic.point=n,he(t=r,e=u)}}function pe(){ic.point=he}function de(){function n(n,t){var e=n-r,i=t-u,o=Math.sqrt(e*e+i*i);Ia+=o*(r+n)/2,Ua+=o*(u+t)/2,Za+=o,o=u*n-r*t,Va+=o*(r+n),Xa+=o*(u+t),$a+=3*o,he(r=n,u=t)}var t,e,r,u;ic.point=function(i,o){ic.point=n,he(t=r=i,e=u=o)},ic.lineEnd=function(){n(t,e)}}function ve(n){function t(t,e){n.moveTo(t,e),n.arc(t,e,o,0,Wo)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function u(){a.point=t}function i(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:u,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=u,a.point=t},pointRadius:function(n){return o=n,a},result:c};return a}function me(n){function t(t){function r(e,r){e=n(e,r),t.point(e[0],e[1])}function u(){M=0/0,S.point=o,t.lineStart()}function o(r,u){var o=Ct([r,u]),a=n(r,u);e(M,x,y,b,_,w,M=a[0],x=a[1],y=r,b=o[0],_=o[1],w=o[2],i,t),t.point(M,x)}function a(){S.point=r,t.lineEnd()}function c(){u(),S.point=l,S.lineEnd=s}function l(n,t){o(f=n,h=t),g=M,p=x,d=b,v=_,m=w,S.point=o}function s(){e(M,x,y,b,_,w,g,p,f,d,v,m,i,t),S.lineEnd=a,a()}var f,h,g,p,d,v,m,y,M,x,b,_,w,S={point:r,lineStart:u,lineEnd:a,polygonStart:function(){t.polygonStart(),S.lineStart=c},polygonEnd:function(){t.polygonEnd(),S.lineStart=u}};return S}function e(t,i,o,a,c,l,s,f,h,g,p,d,v,m){var y=s-t,M=f-i,x=y*y+M*M;if(x>4*r&&v--){var b=a+g,_=c+p,w=l+d,S=Math.sqrt(b*b+_*_+w*w),E=Math.asin(w/=S),k=Math.abs(Math.abs(w)-1)<Go?(o+h)/2:Math.atan2(_,b),A=n(k,E),N=A[0],T=A[1],q=N-t,z=T-i,C=M*q-y*z;(C*C/x>r||Math.abs((y*q+M*z)/x-.5)>.3||u>a*g+c*p+l*d)&&(e(t,i,o,a,c,l,N,T,k,b/=S,_/=S,w,v,m),m.point(N,T),e(N,T,k,b,_,w,s,f,h,g,p,d,v,m))}}var r=.5,u=Math.cos(30*Qo),i=16;return t.precision=function(n){return arguments.length?(i=(r=n*n)>0&&16,t):Math.sqrt(r)},t}function ye(n){this.stream=n}function Me(n){var t=me(function(t,e){return n([t*na,e*na])});return function(n){var e=new ye(n=t(n));return e.point=function(t,e){n.point(t*Qo,e*Qo)},e}}function xe(n){return be(function(){return n})()}function be(n){function t(n){return n=a(n[0]*Qo,n[1]*Qo),[n[0]*h+c,l-n[1]*h]}function e(n){return n=a.invert((n[0]-c)/h,(l-n[1])/h),n&&[n[0]*na,n[1]*na]}function r(){a=ie(o=Ee(m,y,M),i);var n=i(d,v);return c=g-n[0]*h,l=p+n[1]*h,u()}function u(){return s&&(s.valid=!1,s=null),t}var i,o,a,c,l,s,f=me(function(n,t){return n=i(n,t),[n[0]*h+c,l-n[1]*h]}),h=150,g=480,p=250,d=0,v=0,m=0,y=0,M=0,x=Wa,b=dt,_=null,w=null;return t.stream=function(n){return s&&(s.valid=!1),s=_e(x(o,f(b(n)))),s.valid=!0,s},t.clipAngle=function(n){return arguments.length?(x=null==n?(_=n,Wa):ee((_=+n)*Qo),u()):_},t.clipExtent=function(n){return arguments.length?(w=n,b=n?re(n[0][0],n[0][1],n[1][0],n[1][1]):dt,u()):w},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(g=+n[0],p=+n[1],r()):[g,p]},t.center=function(n){return arguments.length?(d=n[0]%360*Qo,v=n[1]%360*Qo,r()):[d*na,v*na]},t.rotate=function(n){return arguments.length?(m=n[0]%360*Qo,y=n[1]%360*Qo,M=n.length>2?n[2]%360*Qo:0,r()):[m*na,y*na,M*na]},mo.rebind(t,f,"precision"),function(){return i=n.apply(this,arguments),t.invert=i.invert&&e,r()}}function _e(n){var t=new ye(n);return t.point=function(t,e){n.point(t*Qo,e*Qo)},t}function we(n,t){return[n,t]}function Se(n,t){return[n>Bo?n-Wo:-Bo>n?n+Wo:n,t]}function Ee(n,t,e){return n?t||e?ie(Ae(n),Ne(t,e)):Ae(n):t||e?Ne(t,e):Se}function ke(n){return function(t,e){return t+=n,[t>Bo?t-Wo:-Bo>t?t+Wo:t,e]}}function Ae(n){var t=ke(n);return t.invert=ke(-n),t}function Ne(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,l=Math.sin(t),s=l*r+a*u;return[Math.atan2(c*i-s*o,a*r-l*u),O(s*i+c*o)]}var r=Math.cos(n),u=Math.sin(n),i=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,l=Math.sin(t),s=l*i-c*o;return[Math.atan2(c*i+l*o,a*r+s*u),O(s*r-a*u)]},e}function Te(n,t){var e=Math.cos(n),r=Math.sin(n);return function(u,i,o,a){var c=o*t;null!=u?(u=qe(e,u),i=qe(e,i),(o>0?i>u:u>i)&&(u+=o*Wo)):(u=n+o*Wo,i=n-.5*c);for(var l,s=u;o>0?s>i:i>s;s-=c)a.point((l=Pt([e,-r*Math.cos(s),-r*Math.sin(s)]))[0],l[1])}}function qe(n,t){var e=Ct(t);e[0]-=n,Ft(e);var r=P(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Go)%(2*Math.PI)}function ze(n,t,e){var r=mo.range(n,t-Go,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function Ce(n,t,e){var r=mo.range(n,t-Go,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function De(n){return n.source}function je(n){return n.target}function Le(n,t,e,r){var u=Math.cos(t),i=Math.sin(t),o=Math.cos(r),a=Math.sin(r),c=u*Math.cos(n),l=u*Math.sin(n),s=o*Math.cos(e),f=o*Math.sin(e),h=2*Math.asin(Math.sqrt(U(r-t)+u*o*U(e-n))),g=1/Math.sin(h),p=h?function(n){var t=Math.sin(n*=h)*g,e=Math.sin(h-n)*g,r=e*c+t*s,u=e*l+t*f,o=e*i+t*a;return[Math.atan2(u,r)*na,Math.atan2(o,Math.sqrt(r*r+u*u))*na]}:function(){return[n*na,t*na]};return p.distance=h,p}function He(){function n(n,u){var i=Math.sin(u*=Qo),o=Math.cos(u),a=Math.abs((n*=Qo)-t),c=Math.cos(a);oc+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*i-e*o*c)*a),e*i+r*o*c),t=n,e=i,r=o}var t,e,r;ac.point=function(u,i){t=u*Qo,e=Math.sin(i*=Qo),r=Math.cos(i),ac.point=n},ac.lineEnd=function(){ac.point=ac.lineEnd=c}}function Fe(n,t){function e(t,e){var r=Math.cos(t),u=Math.cos(e),i=n(r*u);return[i*u*Math.sin(t),i*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),u=t(r),i=Math.sin(u),o=Math.cos(u);return[Math.atan2(n*i,r*o),Math.asin(r&&e*i/r)]},e}function Pe(n,t){function e(n,t){var e=Math.abs(Math.abs(t)-Jo)<Go?0:o/Math.pow(u(t),i);return[e*Math.sin(i*n),o-e*Math.cos(i*n)]}var r=Math.cos(n),u=function(n){return Math.tan(Bo/4+n/2)},i=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(u(t)/u(n)),o=r*Math.pow(u(n),i)/i;return i?(e.invert=function(n,t){var e=o-t,r=F(i)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/i,2*Math.atan(Math.pow(o/r,1/i))-Jo]},e):Re}function Oe(n,t){function e(n,t){var e=i-t;return[e*Math.sin(u*n),i-e*Math.cos(u*n)]}var r=Math.cos(n),u=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),i=r/u+n;return Math.abs(u)<Go?we:(e.invert=function(n,t){var e=i-t;return[Math.atan2(n,e)/u,i-F(u)*Math.sqrt(n*n+e*e)]},e)}function Re(n,t){return[n,Math.log(Math.tan(Bo/4+t/2))]}function Ye(n){var t,e=xe(n),r=e.scale,u=e.translate,i=e.clipExtent;return e.scale=function(){var n=r.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.translate=function(){var n=u.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.clipExtent=function(n){var o=i.apply(e,arguments);if(o===e){if(t=null==n){var a=Bo*r(),c=u();i([[c[0]-a,c[1]-a],[c[0]+a,c[1]+a]])}}else t&&(o=null);return o},e.clipExtent(null)}function Ie(n,t){var e=Math.cos(t)*Math.sin(n);return[Math.log((1+e)/(1-e))/2,Math.atan2(Math.tan(t),Math.cos(n))]}function Ue(n){function t(t){function o(){l.push("M",i(n(s),a))}for(var c,l=[],s=[],f=-1,h=t.length,g=pt(e),p=pt(r);++f<h;)u.call(this,c=t[f],f)?s.push([+g.call(this,c,f),+p.call(this,c,f)]):s.length&&(o(),s=[]);return s.length&&o(),l.length?l.join(""):null}var e=Ze,r=Ve,u=Vt,i=Xe,o=i.key,a=.7;return t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.defined=function(n){return arguments.length?(u=n,t):u},t.interpolate=function(n){return arguments.length?(o="function"==typeof n?i=n:(i=gc.get(n)||Xe).key,t):o},t.tension=function(n){return arguments.length?(a=n,t):a},t}function Ze(n){return n[0]}function Ve(n){return n[1]}function Xe(n){return n.join("L")}function $e(n){return Xe(n)+"Z"}function Be(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("H",(r[0]+(r=n[t])[0])/2,"V",r[1]);return e>1&&u.push("H",r[0]),u.join("")}function We(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("V",(r=n[t])[1],"H",r[0]);return u.join("")}function Je(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t<e;)u.push("H",(r=n[t])[0],"V",r[1]);return u.join("")}function Ge(n,t){return n.length<4?Xe(n):n[1]+nr(n.slice(1,n.length-1),tr(n,t))}function Ke(n,t){return n.length<3?Xe(n):n[0]+nr((n.push(n[0]),n),tr([n[n.length-2]].concat(n,[n[1]]),t))}function Qe(n,t){return n.length<3?Xe(n):n[0]+nr(n,tr(n,t))}function nr(n,t){if(t.length<1||n.length!=t.length&&n.length!=t.length+2)return Xe(n);var e=n.length!=t.length,r="",u=n[0],i=n[1],o=t[0],a=o,c=1;if(e&&(r+="Q"+(i[0]-2*o[0]/3)+","+(i[1]-2*o[1]/3)+","+i[0]+","+i[1],u=n[1],c=2),t.length>1){a=t[1],i=n[c],c++,r+="C"+(u[0]+o[0])+","+(u[1]+o[1])+","+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1];for(var l=2;l<t.length;l++,c++)i=n[c],a=t[l],r+="S"+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1]}if(e){var s=n[c];r+="Q"+(i[0]+2*a[0]/3)+","+(i[1]+2*a[1]/3)+","+s[0]+","+s[1]}return r}function tr(n,t){for(var e,r=[],u=(1-t)/2,i=n[0],o=n[1],a=1,c=n.length;++a<c;)e=i,i=o,o=n[a],r.push([u*(o[0]-e[0]),u*(o[1]-e[1])]);return r}function er(n){if(n.length<3)return Xe(n);var t=1,e=n.length,r=n[0],u=r[0],i=r[1],o=[u,u,u,(r=n[1])[0]],a=[i,i,i,r[1]],c=[u,",",i,"L",or(vc,o),",",or(vc,a)];for(n.push(n[e-1]);++t<=e;)r=n[t],o.shift(),o.push(r[0]),a.shift(),a.push(r[1]),ar(c,o,a);return n.pop(),c.push("L",r),c.join("")}function rr(n){if(n.length<4)return Xe(n);for(var t,e=[],r=-1,u=n.length,i=[0],o=[0];++r<3;)t=n[r],i.push(t[0]),o.push(t[1]);for(e.push(or(vc,i)+","+or(vc,o)),--r;++r<u;)t=n[r],i.shift(),i.push(t[0]),o.shift(),o.push(t[1]),ar(e,i,o);return e.join("")}function ur(n){for(var t,e,r=-1,u=n.length,i=u+4,o=[],a=[];++r<4;)e=n[r%u],o.push(e[0]),a.push(e[1]);for(t=[or(vc,o),",",or(vc,a)],--r;++r<i;)e=n[r%u],o.shift(),o.push(e[0]),a.shift(),a.push(e[1]),ar(t,o,a);return t.join("")}function ir(n,t){var e=n.length-1;if(e)for(var r,u,i=n[0][0],o=n[0][1],a=n[e][0]-i,c=n[e][1]-o,l=-1;++l<=e;)r=n[l],u=l/e,r[0]=t*r[0]+(1-t)*(i+u*a),r[1]=t*r[1]+(1-t)*(o+u*c);return er(n)}function or(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]+n[3]*t[3]}function ar(n,t,e){n.push("C",or(pc,t),",",or(pc,e),",",or(dc,t),",",or(dc,e),",",or(vc,t),",",or(vc,e))}function cr(n,t){return(t[1]-n[1])/(t[0]-n[0])}function lr(n){for(var t=0,e=n.length-1,r=[],u=n[0],i=n[1],o=r[0]=cr(u,i);++t<e;)r[t]=(o+(o=cr(u=i,i=n[t+1])))/2;return r[t]=o,r
-}function sr(n){for(var t,e,r,u,i=[],o=lr(n),a=-1,c=n.length-1;++a<c;)t=cr(n[a],n[a+1]),Math.abs(t)<Go?o[a]=o[a+1]=0:(e=o[a]/t,r=o[a+1]/t,u=e*e+r*r,u>9&&(u=3*t/Math.sqrt(u),o[a]=u*e,o[a+1]=u*r));for(a=-1;++a<=c;)u=(n[Math.min(c,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),i.push([u||0,o[a]*u||0]);return i}function fr(n){return n.length<3?Xe(n):n[0]+nr(n,sr(n))}function hr(n,t,e,r){var u,i,o,a,c,l,s;return u=r[n],i=u[0],o=u[1],u=r[t],a=u[0],c=u[1],u=r[e],l=u[0],s=u[1],(s-o)*(a-i)-(c-o)*(l-i)>0}function gr(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function pr(n,t,e,r){var u=n[0],i=e[0],o=t[0]-u,a=r[0]-i,c=n[1],l=e[1],s=t[1]-c,f=r[1]-l,h=(a*(c-l)-f*(u-i))/(f*o-a*s);return[u+h*o,c+h*s]}function dr(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function vr(n,t){var e={list:n.map(function(n,t){return{index:t,x:n[0],y:n[1]}}).sort(function(n,t){return n.y<t.y?-1:n.y>t.y?1:n.x<t.x?-1:n.x>t.x?1:0}),bottomSite:null},r={list:[],leftEnd:null,rightEnd:null,init:function(){r.leftEnd=r.createHalfEdge(null,"l"),r.rightEnd=r.createHalfEdge(null,"l"),r.leftEnd.r=r.rightEnd,r.rightEnd.l=r.leftEnd,r.list.unshift(r.leftEnd,r.rightEnd)},createHalfEdge:function(n,t){return{edge:n,side:t,vertex:null,l:null,r:null}},insert:function(n,t){t.l=n,t.r=n.r,n.r.l=t,n.r=t},leftBound:function(n){var t=r.leftEnd;do t=t.r;while(t!=r.rightEnd&&u.rightOf(t,n));return t=t.l},del:function(n){n.l.r=n.r,n.r.l=n.l,n.edge=null},right:function(n){return n.r},left:function(n){return n.l},leftRegion:function(n){return null==n.edge?e.bottomSite:n.edge.region[n.side]},rightRegion:function(n){return null==n.edge?e.bottomSite:n.edge.region[yc[n.side]]}},u={bisect:function(n,t){var e={region:{l:n,r:t},ep:{l:null,r:null}},r=t.x-n.x,u=t.y-n.y,i=r>0?r:-r,o=u>0?u:-u;return e.c=n.x*r+n.y*u+.5*(r*r+u*u),i>o?(e.a=1,e.b=u/r,e.c/=r):(e.b=1,e.a=r/u,e.c/=u),e},intersect:function(n,t){var e=n.edge,r=t.edge;if(!e||!r||e.region.r==r.region.r)return null;var u=e.a*r.b-e.b*r.a;if(Math.abs(u)<1e-10)return null;var i,o,a=(e.c*r.b-r.c*e.b)/u,c=(r.c*e.a-e.c*r.a)/u,l=e.region.r,s=r.region.r;l.y<s.y||l.y==s.y&&l.x<s.x?(i=n,o=e):(i=t,o=r);var f=a>=o.region.r.x;return f&&"l"===i.side||!f&&"r"===i.side?null:{x:a,y:c}},rightOf:function(n,t){var e=n.edge,r=e.region.r,u=t.x>r.x;if(u&&"l"===n.side)return 1;if(!u&&"r"===n.side)return 0;if(1===e.a){var i=t.y-r.y,o=t.x-r.x,a=0,c=0;if(!u&&e.b<0||u&&e.b>=0?c=a=i>=e.b*o:(c=t.x+t.y*e.b>e.c,e.b<0&&(c=!c),c||(a=1)),!a){var l=r.x-e.region.l.x;c=e.b*(o*o-i*i)<l*i*(1+2*o/l+e.b*e.b),e.b<0&&(c=!c)}}else{var s=e.c-e.a*t.x,f=t.y-s,h=t.x-r.x,g=s-r.y;c=f*f>h*h+g*g}return"l"===n.side?c:!c},endPoint:function(n,e,r){n.ep[e]=r,n.ep[yc[e]]&&t(n)},distance:function(n,t){var e=n.x-t.x,r=n.y-t.y;return Math.sqrt(e*e+r*r)}},i={list:[],insert:function(n,t,e){n.vertex=t,n.ystar=t.y+e;for(var r=0,u=i.list,o=u.length;o>r;r++){var a=u[r];if(!(n.ystar>a.ystar||n.ystar==a.ystar&&t.x>a.vertex.x))break}u.splice(r,0,n)},del:function(n){for(var t=0,e=i.list,r=e.length;r>t&&e[t]!=n;++t);e.splice(t,1)},empty:function(){return 0===i.list.length},nextEvent:function(n){for(var t=0,e=i.list,r=e.length;r>t;++t)if(e[t]==n)return e[t+1];return null},min:function(){var n=i.list[0];return{x:n.vertex.x,y:n.ystar}},extractMin:function(){return i.list.shift()}};r.init(),e.bottomSite=e.list.shift();for(var o,a,c,l,s,f,h,g,p,d,v,m,y,M=e.list.shift();;)if(i.empty()||(o=i.min()),M&&(i.empty()||M.y<o.y||M.y==o.y&&M.x<o.x))a=r.leftBound(M),c=r.right(a),h=r.rightRegion(a),m=u.bisect(h,M),f=r.createHalfEdge(m,"l"),r.insert(a,f),d=u.intersect(a,f),d&&(i.del(a),i.insert(a,d,u.distance(d,M))),a=f,f=r.createHalfEdge(m,"r"),r.insert(a,f),d=u.intersect(f,c),d&&i.insert(f,d,u.distance(d,M)),M=e.list.shift();else{if(i.empty())break;a=i.extractMin(),l=r.left(a),c=r.right(a),s=r.right(c),h=r.leftRegion(a),g=r.rightRegion(c),v=a.vertex,u.endPoint(a.edge,a.side,v),u.endPoint(c.edge,c.side,v),r.del(a),i.del(c),r.del(c),y="l",h.y>g.y&&(p=h,h=g,g=p,y="r"),m=u.bisect(h,g),f=r.createHalfEdge(m,y),r.insert(l,f),u.endPoint(m,yc[y],v),d=u.intersect(l,f),d&&(i.del(l),i.insert(l,d,u.distance(d,h))),d=u.intersect(f,s),d&&i.insert(f,d,u.distance(d,h))}for(a=r.right(r.leftEnd);a!=r.rightEnd;a=r.right(a))t(a.edge)}function mr(n){return n.x}function yr(n){return n.y}function Mr(){return{leaf:!0,nodes:[],point:null,x:null,y:null}}function xr(n,t,e,r,u,i){if(!n(t,e,r,u,i)){var o=.5*(e+u),a=.5*(r+i),c=t.nodes;c[0]&&xr(n,c[0],e,r,o,a),c[1]&&xr(n,c[1],o,r,u,a),c[2]&&xr(n,c[2],e,a,o,i),c[3]&&xr(n,c[3],o,a,u,i)}}function br(n,t){n=mo.rgb(n),t=mo.rgb(t);var e=n.r,r=n.g,u=n.b,i=t.r-e,o=t.g-r,a=t.b-u;return function(n){return"#"+ct(Math.round(e+i*n))+ct(Math.round(r+o*n))+ct(Math.round(u+a*n))}}function _r(n,t){var e,r={},u={};for(e in n)e in t?r[e]=Er(n[e],t[e]):u[e]=n[e];for(e in t)e in n||(u[e]=t[e]);return function(n){for(e in r)u[e]=r[e](n);return u}}function wr(n,t){return t-=n=+n,function(e){return n+t*e}}function Sr(n,t){var e,r,u,i,o,a=0,c=0,l=[],s=[];for(n+="",t+="",Mc.lastIndex=0,r=0;e=Mc.exec(t);++r)e.index&&l.push(t.substring(a,c=e.index)),s.push({i:l.length,x:e[0]}),l.push(null),a=Mc.lastIndex;for(a<t.length&&l.push(t.substring(a)),r=0,i=s.length;(e=Mc.exec(n))&&i>r;++r)if(o=s[r],o.x==e[0]){if(o.i)if(null==l[o.i+1])for(l[o.i-1]+=o.x,l.splice(o.i,1),u=r+1;i>u;++u)s[u].i--;else for(l[o.i-1]+=o.x+l[o.i+1],l.splice(o.i,2),u=r+1;i>u;++u)s[u].i-=2;else if(null==l[o.i+1])l[o.i]=o.x;else for(l[o.i]=o.x+l[o.i+1],l.splice(o.i+1,1),u=r+1;i>u;++u)s[u].i--;s.splice(r,1),i--,r--}else o.x=wr(parseFloat(e[0]),parseFloat(o.x));for(;i>r;)o=s.pop(),null==l[o.i+1]?l[o.i]=o.x:(l[o.i]=o.x+l[o.i+1],l.splice(o.i+1,1)),i--;return 1===l.length?null==l[0]?(o=s[0].x,function(n){return o(n)+""}):function(){return t}:function(n){for(r=0;i>r;++r)l[(o=s[r]).i]=o.x(n);return l.join("")}}function Er(n,t){for(var e,r=mo.interpolators.length;--r>=0&&!(e=mo.interpolators[r](n,t)););return e}function kr(n,t){var e,r=[],u=[],i=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(Er(n[e],t[e]));for(;i>e;++e)u[e]=n[e];for(;o>e;++e)u[e]=t[e];return function(n){for(e=0;a>e;++e)u[e]=r[e](n);return u}}function Ar(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function Nr(n){return function(t){return 1-n(1-t)}}function Tr(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function qr(n){return n*n}function zr(n){return n*n*n}function Cr(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Dr(n){return function(t){return Math.pow(t,n)}}function jr(n){return 1-Math.cos(n*Jo)}function Lr(n){return Math.pow(2,10*(n-1))}function Hr(n){return 1-Math.sqrt(1-n*n)}function Fr(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/Wo*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*Wo/t)}}function Pr(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Or(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Rr(n,t){n=mo.hcl(n),t=mo.hcl(t);var e=n.h,r=n.c,u=n.l,i=t.h-e,o=t.c-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return J(e+i*n,r+o*n,u+a*n)+""}}function Yr(n,t){n=mo.hsl(n),t=mo.hsl(t);var e=n.h,r=n.s,u=n.l,i=t.h-e,o=t.s-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return $(e+i*n,r+o*n,u+a*n)+""}}function Ir(n,t){n=mo.lab(n),t=mo.lab(t);var e=n.l,r=n.a,u=n.b,i=t.l-e,o=t.a-r,a=t.b-u;return function(n){return Q(e+i*n,r+o*n,u+a*n)+""}}function Ur(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function Zr(n){var t=[n.a,n.b],e=[n.c,n.d],r=Xr(t),u=Vr(t,e),i=Xr($r(e,t,-u))||0;t[0]*e[1]<e[0]*t[1]&&(t[0]*=-1,t[1]*=-1,r*=-1,u*=-1),this.rotate=(r?Math.atan2(t[1],t[0]):Math.atan2(-e[0],e[1]))*na,this.translate=[n.e,n.f],this.scale=[r,i],this.skew=i?Math.atan2(u,i)*na:0}function Vr(n,t){return n[0]*t[0]+n[1]*t[1]}function Xr(n){var t=Math.sqrt(Vr(n,n));return t&&(n[0]/=t,n[1]/=t),t}function $r(n,t,e){return n[0]+=e*t[0],n[1]+=e*t[1],n}function Br(n,t){var e,r=[],u=[],i=mo.transform(n),o=mo.transform(t),a=i.translate,c=o.translate,l=i.rotate,s=o.rotate,f=i.skew,h=o.skew,g=i.scale,p=o.scale;return a[0]!=c[0]||a[1]!=c[1]?(r.push("translate(",null,",",null,")"),u.push({i:1,x:wr(a[0],c[0])},{i:3,x:wr(a[1],c[1])})):c[0]||c[1]?r.push("translate("+c+")"):r.push(""),l!=s?(l-s>180?s+=360:s-l>180&&(l+=360),u.push({i:r.push(r.pop()+"rotate(",null,")")-2,x:wr(l,s)})):s&&r.push(r.pop()+"rotate("+s+")"),f!=h?u.push({i:r.push(r.pop()+"skewX(",null,")")-2,x:wr(f,h)}):h&&r.push(r.pop()+"skewX("+h+")"),g[0]!=p[0]||g[1]!=p[1]?(e=r.push(r.pop()+"scale(",null,",",null,")"),u.push({i:e-4,x:wr(g[0],p[0])},{i:e-2,x:wr(g[1],p[1])})):(1!=p[0]||1!=p[1])&&r.push(r.pop()+"scale("+p+")"),e=u.length,function(n){for(var t,i=-1;++i<e;)r[(t=u[i]).i]=t.x(n);return r.join("")}}function Wr(n,t){return t=t-(n=+n)?1/(t-n):0,function(e){return(e-n)*t}}function Jr(n,t){return t=t-(n=+n)?1/(t-n):0,function(e){return Math.max(0,Math.min(1,(e-n)*t))}}function Gr(n){for(var t=n.source,e=n.target,r=Qr(t,e),u=[t];t!==r;)t=t.parent,u.push(t);for(var i=u.length;e!==r;)u.splice(i,0,e),e=e.parent;return u}function Kr(n){for(var t=[],e=n.parent;null!=e;)t.push(n),n=e,e=e.parent;return t.push(n),t}function Qr(n,t){if(n===t)return n;for(var e=Kr(n),r=Kr(t),u=e.pop(),i=r.pop(),o=null;u===i;)o=u,u=e.pop(),i=r.pop();return o}function nu(n){n.fixed|=2}function tu(n){n.fixed&=-7}function eu(n){n.fixed|=4,n.px=n.x,n.py=n.y}function ru(n){n.fixed&=-5}function uu(n,t,e){var r=0,u=0;if(n.charge=0,!n.leaf)for(var i,o=n.nodes,a=o.length,c=-1;++c<a;)i=o[c],null!=i&&(uu(i,t,e),n.charge+=i.charge,r+=i.charge*i.cx,u+=i.charge*i.cy);if(n.point){n.leaf||(n.point.x+=Math.random()-.5,n.point.y+=Math.random()-.5);var l=t*e[n.point.index];n.charge+=n.pointCharge=l,r+=l*n.point.x,u+=l*n.point.y}n.cx=r/n.charge,n.cy=u/n.charge}function iu(n,t){return mo.rebind(n,t,"sort","children","value"),n.nodes=n,n.links=lu,n}function ou(n){return n.children}function au(n){return n.value}function cu(n,t){return t.value-n.value}function lu(n){return mo.merge(n.map(function(n){return(n.children||[]).map(function(t){return{source:n,target:t}})}))}function su(n){return n.x}function fu(n){return n.y}function hu(n,t,e){n.y0=t,n.y=e}function gu(n){return mo.range(n.length)}function pu(n){for(var t=-1,e=n[0].length,r=[];++t<e;)r[t]=0;return r}function du(n){for(var t,e=1,r=0,u=n[0][1],i=n.length;i>e;++e)(t=n[e][1])>u&&(r=e,u=t);return r}function vu(n){return n.reduce(mu,0)}function mu(n,t){return n+t[1]}function yu(n,t){return Mu(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function Mu(n,t){for(var e=-1,r=+n[0],u=(n[1]-r)/t,i=[];++e<=t;)i[e]=u*e+r;return i}function xu(n){return[mo.min(n),mo.max(n)]}function bu(n,t){return n.parent==t.parent?1:2}function _u(n){var t=n.children;return t&&t.length?t[0]:n._tree.thread}function wu(n){var t,e=n.children;return e&&(t=e.length)?e[t-1]:n._tree.thread}function Su(n,t){var e=n.children;if(e&&(u=e.length))for(var r,u,i=-1;++i<u;)t(r=Su(e[i],t),n)>0&&(n=r);return n}function Eu(n,t){return n.x-t.x}function ku(n,t){return t.x-n.x}function Au(n,t){return n.depth-t.depth}function Nu(n,t){function e(n,r){var u=n.children;if(u&&(o=u.length))for(var i,o,a=null,c=-1;++c<o;)i=u[c],e(i,a),a=i;t(n,r)}e(n,null)}function Tu(n){for(var t,e=0,r=0,u=n.children,i=u.length;--i>=0;)t=u[i]._tree,t.prelim+=e,t.mod+=e,e+=t.shift+(r+=t.change)}function qu(n,t,e){n=n._tree,t=t._tree;var r=e/(t.number-n.number);n.change+=r,t.change-=r,t.shift+=e,t.prelim+=e,t.mod+=e}function zu(n,t,e){return n._tree.ancestor.parent==t.parent?n._tree.ancestor:e}function Cu(n,t){return n.value-t.value}function Du(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function ju(n,t){n._pack_next=t,t._pack_prev=n}function Lu(n,t){var e=t.x-n.x,r=t.y-n.y,u=n.r+t.r;return.999*u*u>e*e+r*r}function Hu(n){function t(n){s=Math.min(n.x-n.r,s),f=Math.max(n.x+n.r,f),h=Math.min(n.y-n.r,h),g=Math.max(n.y+n.r,g)}if((e=n.children)&&(l=e.length)){var e,r,u,i,o,a,c,l,s=1/0,f=-1/0,h=1/0,g=-1/0;if(e.forEach(Fu),r=e[0],r.x=-r.r,r.y=0,t(r),l>1&&(u=e[1],u.x=u.r,u.y=0,t(u),l>2))for(i=e[2],Ru(r,u,i),t(i),Du(r,i),r._pack_prev=i,Du(i,u),u=r._pack_next,o=3;l>o;o++){Ru(r,u,i=e[o]);var p=0,d=1,v=1;for(a=u._pack_next;a!==u;a=a._pack_next,d++)if(Lu(a,i)){p=1;break}if(1==p)for(c=r._pack_prev;c!==a._pack_prev&&!Lu(c,i);c=c._pack_prev,v++);p?(v>d||d==v&&u.r<r.r?ju(r,u=a):ju(r=c,u),o--):(Du(r,i),u=i,t(i))}var m=(s+f)/2,y=(h+g)/2,M=0;for(o=0;l>o;o++)i=e[o],i.x-=m,i.y-=y,M=Math.max(M,i.r+Math.sqrt(i.x*i.x+i.y*i.y));n.r=M,e.forEach(Pu)}}function Fu(n){n._pack_next=n._pack_prev=n}function Pu(n){delete n._pack_next,delete n._pack_prev}function Ou(n,t,e,r){var u=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,u)for(var i=-1,o=u.length;++i<o;)Ou(u[i],t,e,r)}function Ru(n,t,e){var r=n.r+e.r,u=t.x-n.x,i=t.y-n.y;if(r&&(u||i)){var o=t.r+e.r,a=u*u+i*i;o*=o,r*=r;var c=.5+(r-o)/(2*a),l=Math.sqrt(Math.max(0,2*o*(r+a)-(r-=a)*r-o*o))/(2*a);e.x=n.x+c*u+l*i,e.y=n.y+c*i-l*u}else e.x=n.x+r,e.y=n.y}function Yu(n){return 1+mo.max(n,function(n){return n.y})}function Iu(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Uu(n){var t=n.children;return t&&t.length?Uu(t[0]):n}function Zu(n){var t,e=n.children;return e&&(t=e.length)?Zu(e[t-1]):n}function Vu(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Xu(n,t){var e=n.x+t[3],r=n.y+t[0],u=n.dx-t[1]-t[3],i=n.dy-t[0]-t[2];return 0>u&&(e+=u/2,u=0),0>i&&(r+=i/2,i=0),{x:e,y:r,dx:u,dy:i}}function $u(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Bu(n){return n.rangeExtent?n.rangeExtent():$u(n.range())}function Wu(n,t,e,r){var u=e(n[0],n[1]),i=r(t[0],t[1]);return function(n){return i(u(n))}}function Ju(n,t){var e,r=0,u=n.length-1,i=n[r],o=n[u];return i>o&&(e=r,r=u,u=e,e=i,i=o,o=e),n[r]=t.floor(i),n[u]=t.ceil(o),n}function Gu(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:Tc}function Ku(n,t,e,r){var u=[],i=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]<n[0]&&(n=n.slice().reverse(),t=t.slice().reverse());++o<=a;)u.push(e(n[o-1],n[o])),i.push(r(t[o-1],t[o]));return function(t){var e=mo.bisect(n,t,1,a)-1;return i[e](u[e](t))}}function Qu(n,t,e,r){function u(){var u=Math.min(n.length,t.length)>2?Ku:Wu,c=r?Jr:Wr;return o=u(n,t,c,e),a=u(t,n,c,Er),i}function i(n){return o(n)}var o,a;return i.invert=function(n){return a(n)},i.domain=function(t){return arguments.length?(n=t.map(Number),u()):n},i.range=function(n){return arguments.length?(t=n,u()):t},i.rangeRound=function(n){return i.range(n).interpolate(Ur)},i.clamp=function(n){return arguments.length?(r=n,u()):r},i.interpolate=function(n){return arguments.length?(e=n,u()):e},i.ticks=function(t){return ri(n,t)},i.tickFormat=function(t,e){return ui(n,t,e)},i.nice=function(t){return ti(n,t),u()},i.copy=function(){return Qu(n,t,e,r)},u()}function ni(n,t){return mo.rebind(n,t,"range","rangeRound","interpolate","clamp")}function ti(n,t){return Ju(n,Gu(ei(n,t)[2]))}function ei(n,t){null==t&&(t=10);var e=$u(n),r=e[1]-e[0],u=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),i=t/r*u;return.15>=i?u*=10:.35>=i?u*=5:.75>=i&&(u*=2),e[0]=Math.ceil(e[0]/u)*u,e[1]=Math.floor(e[1]/u)*u+.5*u,e[2]=u,e}function ri(n,t){return mo.range.apply(mo,ei(n,t))}function ui(n,t,e){var r=-Math.floor(Math.log(ei(n,t)[2])/Math.LN10+.01);return mo.format(e?e.replace(Aa,function(n,t,e,u,i,o,a,c,l,s){return[t,e,u,i,o,a,c,l||"."+(r-2*("%"===s)),s].join("")}):",."+r+"f")}function ii(n,t,e,r){function u(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function i(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(u(t))}return o.invert=function(t){return i(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(u)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(u)),o):t},o.nice=function(){var t=Ju(r.map(u),e?Math:zc);return n.domain(t),r=t.map(i),o},o.ticks=function(){var n=$u(r),o=[],a=n[0],c=n[1],l=Math.floor(u(a)),s=Math.ceil(u(c)),f=t%1?2:t;if(isFinite(s-l)){if(e){for(;s>l;l++)for(var h=1;f>h;h++)o.push(i(l)*h);o.push(i(l))}else for(o.push(i(l));l++<s;)for(var h=f-1;h>0;h--)o.push(i(l)*h);for(l=0;o[l]<a;l++);for(s=o.length;o[s-1]>c;s--);o=o.slice(l,s)}return o},o.tickFormat=function(n,t){if(!arguments.length)return qc;arguments.length<2?t=qc:"function"!=typeof t&&(t=mo.format(t));var r,a=Math.max(.1,n/o.ticks().length),c=e?(r=1e-12,Math.ceil):(r=-1e-12,Math.floor);return function(n){return n/i(c(u(n)+r))<=a?t(n):""}},o.copy=function(){return ii(n.copy(),t,e,r)},ni(o,n)}function oi(n,t,e){function r(t){return n(u(t))}var u=ai(t),i=ai(1/t);return r.invert=function(t){return i(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(u)),r):e},r.ticks=function(n){return ri(e,n)},r.tickFormat=function(n,t){return ui(e,n,t)},r.nice=function(n){return r.domain(ti(e,n))},r.exponent=function(o){return arguments.length?(u=ai(t=o),i=ai(1/t),n.domain(e.map(u)),r):t},r.copy=function(){return oi(n.copy(),t,e)},ni(r,n)}function ai(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function ci(n,t){function e(e){return o[((i.get(e)||"range"===t.t&&i.set(e,n.push(e)))-1)%o.length]}function r(t,e){return mo.range(n.length).map(function(n){return t+e*n})}var i,o,a;return e.domain=function(r){if(!arguments.length)return n;n=[],i=new u;for(var o,a=-1,c=r.length;++a<c;)i.has(o=r[a])||i.set(o,n.push(o));return e[t.t].apply(e,t.a)},e.range=function(n){return arguments.length?(o=n,a=0,t={t:"range",a:arguments},e):o},e.rangePoints=function(u,i){arguments.length<2&&(i=0);var c=u[0],l=u[1],s=(l-c)/(Math.max(1,n.length-1)+i);return o=r(n.length<2?(c+l)/2:c+s*i/2,s),a=0,t={t:"rangePoints",a:arguments},e},e.rangeBands=function(u,i,c){arguments.length<2&&(i=0),arguments.length<3&&(c=i);var l=u[1]<u[0],s=u[l-0],f=u[1-l],h=(f-s)/(n.length-i+2*c);return o=r(s+h*c,h),l&&o.reverse(),a=h*(1-i),t={t:"rangeBands",a:arguments},e},e.rangeRoundBands=function(u,i,c){arguments.length<2&&(i=0),arguments.length<3&&(c=i);var l=u[1]<u[0],s=u[l-0],f=u[1-l],h=Math.floor((f-s)/(n.length-i+2*c)),g=f-s-(n.length-i)*h;return o=r(s+Math.round(g/2),h),l&&o.reverse(),a=Math.round(h*(1-i)),t={t:"rangeRoundBands",a:arguments},e},e.rangeBand=function(){return a},e.rangeExtent=function(){return $u(t.a[0])},e.copy=function(){return ci(n,t)},e.domain(n)}function li(n,t){function e(){var e=0,i=t.length;for(u=[];++e<i;)u[e-1]=mo.quantile(n,e/i);return r}function r(n){return isNaN(n=+n)?void 0:t[mo.bisect(u,n)]}var u;return r.domain=function(t){return arguments.length?(n=t.filter(function(n){return!isNaN(n)}).sort(mo.ascending),e()):n},r.range=function(n){return arguments.length?(t=n,e()):t},r.quantiles=function(){return u},r.invertExtent=function(e){return e=t.indexOf(e),0>e?[0/0,0/0]:[e>0?u[e-1]:n[0],e<u.length?u[e]:n[n.length-1]]},r.copy=function(){return li(n,t)},e()}function si(n,t,e){function r(t){return e[Math.max(0,Math.min(o,Math.floor(i*(t-n))))]}function u(){return i=e.length/(t-n),o=e.length-1,r}var i,o;return r.domain=function(e){return arguments.length?(n=+e[0],t=+e[e.length-1],u()):[n,t]},r.range=function(n){return arguments.length?(e=n,u()):e},r.invertExtent=function(t){return t=e.indexOf(t),t=0>t?0/0:t/i+n,[t,t+1/i]},r.copy=function(){return si(n,t,e)},u()}function fi(n,t){function e(e){return e>=e?t[mo.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return fi(n,t)},e}function hi(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return ri(n,t)},t.tickFormat=function(t,e){return ui(n,t,e)},t.copy=function(){return hi(n)},t}function gi(n){return n.innerRadius}function pi(n){return n.outerRadius}function di(n){return n.startAngle}function vi(n){return n.endAngle}function mi(n){for(var t,e,r,u=-1,i=n.length;++u<i;)t=n[u],e=t[0],r=t[1]+Hc,t[0]=e*Math.cos(r),t[1]=e*Math.sin(r);return n}function yi(n){function t(t){function c(){d.push("M",a(n(m),f),s,l(n(v.reverse()),f),"Z")}for(var h,g,p,d=[],v=[],m=[],y=-1,M=t.length,x=pt(e),b=pt(u),_=e===r?function(){return g}:pt(r),w=u===i?function(){return p}:pt(i);++y<M;)o.call(this,h=t[y],y)?(v.push([g=+x.call(this,h,y),p=+b.call(this,h,y)]),m.push([+_.call(this,h,y),+w.call(this,h,y)])):v.length&&(c(),v=[],m=[]);return v.length&&c(),d.length?d.join(""):null}var e=Ze,r=Ze,u=0,i=Ve,o=Vt,a=Xe,c=a.key,l=a,s="L",f=.7;return t.x=function(n){return arguments.length?(e=r=n,t):r},t.x0=function(n){return arguments.length?(e=n,t):e},t.x1=function(n){return arguments.length?(r=n,t):r},t.y=function(n){return arguments.length?(u=i=n,t):i},t.y0=function(n){return arguments.length?(u=n,t):u},t.y1=function(n){return arguments.length?(i=n,t):i},t.defined=function(n){return arguments.length?(o=n,t):o},t.interpolate=function(n){return arguments.length?(c="function"==typeof n?a=n:(a=gc.get(n)||Xe).key,l=a.reverse||a,s=a.closed?"M":"L",t):c},t.tension=function(n){return arguments.length?(f=n,t):f},t}function Mi(n){return n.radius}function xi(n){return[n.x,n.y]}function bi(n){return function(){var t=n.apply(this,arguments),e=t[0],r=t[1]+Hc;return[e*Math.cos(r),e*Math.sin(r)]}}function _i(){return 64}function wi(){return"circle"}function Si(n){var t=Math.sqrt(n/Bo);return"M0,"+t+"A"+t+","+t+" 0 1,1 0,"+-t+"A"+t+","+t+" 0 1,1 0,"+t+"Z"}function Ei(n,t){return Lo(n,Uc),n.id=t,n}function ki(n,t,e,r){var u=n.id;return N(n,"function"==typeof e?function(n,i,o){n.__transition__[u].tween.set(t,r(e.call(n,n.__data__,i,o)))}:(e=r(e),function(n){n.__transition__[u].tween.set(t,e)}))}function Ai(n){return null==n&&(n=""),function(){this.textContent=n}}function Ni(n,t,e,r){var i=n.__transition__||(n.__transition__={active:0,count:0}),o=i[e];if(!o){var a=r.time;o=i[e]={tween:new u,time:a,ease:r.ease,delay:r.delay,duration:r.duration},++i.count,mo.timer(function(r){function u(r){return i.active>e?l():(i.active=e,o.event&&o.event.start.call(n,s,t),o.tween.forEach(function(e,r){(r=r.call(n,s,t))&&p.push(r)}),c(r||1)?1:(xt(c,h,a),void 0))}function c(r){if(i.active!==e)return l();for(var u=r/g,a=f(u),c=p.length;c>0;)p[--c].call(n,a);return u>=1?(o.event&&o.event.end.call(n,s,t),l()):void 0}function l(){return--i.count?delete i[e]:delete n.__transition__,1}var s=n.__data__,f=o.ease,h=o.delay,g=o.duration,p=[];return r>=h?u(r-h):(xt(u,h,a),void 0)},0,a)}}function Ti(n,t){n.attr("transform",function(n){return"translate("+t(n)+",0)"})}function qi(n,t){n.attr("transform",function(n){return"translate(0,"+t(n)+")"})}function zi(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function Ci(n,t,e){function r(t){var e=n(t),r=i(e,1);return r-t>t-e?e:r}function u(e){return t(e=n(new Jc(e-1)),1),e}function i(n,e){return t(n=new Jc(+n),e),n}function o(n,r,i){var o=u(n),a=[];if(i>1)for(;r>o;)e(o)%i||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{Jc=zi;var r=new zi;return r._=n,o(r,t,e)}finally{Jc=Date}}n.floor=n,n.round=r,n.ceil=u,n.offset=i,n.range=o;var c=n.utc=Di(n);return c.floor=c,c.round=Di(r),c.ceil=Di(u),c.offset=Di(i),c.range=a,n}function Di(n){return function(t,e){try{Jc=zi;var r=new zi;return r._=t,n(r,e)._}finally{Jc=Date}}}function ji(n){function t(t){for(var r,u,i,o=[],a=-1,c=0;++a<e;)37===n.charCodeAt(a)&&(o.push(n.substring(c,a)),null!=(u=dl[r=n.charAt(++a)])&&(r=n.charAt(++a)),(i=vl[r])&&(r=i(t,null==u?"e"===r?" ":"0":u)),o.push(r),c=a+1);return o.push(n.substring(c,a)),o.join("")}var e=n.length;return t.parse=function(t){var e={y:1900,m:0,d:1,H:0,M:0,S:0,L:0,Z:null},r=Li(e,n,t,0);if(r!=t.length)return null;"p"in e&&(e.H=e.H%12+12*e.p);var u=null!=e.Z&&Jc!==zi,i=new(u?zi:Jc);return"j"in e?i.setFullYear(e.y,0,e.j):"w"in e&&("W"in e||"U"in e)?(i.setFullYear(e.y,0,1),i.setFullYear(e.y,0,"W"in e?(e.w+6)%7+7*e.W-(i.getDay()+5)%7:e.w+7*e.U-(i.getDay()+6)%7)):i.setFullYear(e.y,e.m,e.d),i.setHours(e.H+Math.floor(e.Z/100),e.M+e.Z%100,e.S,e.L),u?i._:i},t.toString=function(){return n},t}function Li(n,t,e,r){for(var u,i,o,a=0,c=t.length,l=e.length;c>a;){if(r>=l)return-1;if(u=t.charCodeAt(a++),37===u){if(o=t.charAt(a++),i=ml[o in dl?t.charAt(a++):o],!i||(r=i(n,e,r))<0)return-1}else if(u!=e.charCodeAt(r++))return-1}return r}function Hi(n){return new RegExp("^(?:"+n.map(mo.requote).join("|")+")","i")}function Fi(n){for(var t=new u,e=-1,r=n.length;++e<r;)t.set(n[e].toLowerCase(),e);return t}function Pi(n,t,e){var r=0>n?"-":"",u=(r?-n:n)+"",i=u.length;return r+(e>i?new Array(e-i+1).join(t)+u:u)}function Oi(n,t,e){cl.lastIndex=0;var r=cl.exec(t.substring(e));return r?(n.w=ll.get(r[0].toLowerCase()),e+r[0].length):-1}function Ri(n,t,e){ol.lastIndex=0;var r=ol.exec(t.substring(e));return r?(n.w=al.get(r[0].toLowerCase()),e+r[0].length):-1}function Yi(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+1));return r?(n.w=+r[0],e+r[0].length):-1}function Ii(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e));return r?(n.U=+r[0],e+r[0].length):-1}function Ui(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e));return r?(n.W=+r[0],e+r[0].length):-1}function Zi(n,t,e){hl.lastIndex=0;var r=hl.exec(t.substring(e));return r?(n.m=gl.get(r[0].toLowerCase()),e+r[0].length):-1}function Vi(n,t,e){sl.lastIndex=0;var r=sl.exec(t.substring(e));return r?(n.m=fl.get(r[0].toLowerCase()),e+r[0].length):-1}function Xi(n,t,e){return Li(n,vl.c.toString(),t,e)}function $i(n,t,e){return Li(n,vl.x.toString(),t,e)}function Bi(n,t,e){return Li(n,vl.X.toString(),t,e)}function Wi(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+4));return r?(n.y=+r[0],e+r[0].length):-1}function Ji(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+2));return r?(n.y=Ki(+r[0]),e+r[0].length):-1}function Gi(n,t,e){return/^[+-]\d{4}$/.test(t=t.substring(e,e+5))?(n.Z=+t,e+5):-1}function Ki(n){return n+(n>68?1900:2e3)}function Qi(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function no(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function to(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function eo(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+2));return r?(n.H=+r[0],e+r[0].length):-1}function ro(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function uo(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function io(n,t,e){yl.lastIndex=0;var r=yl.exec(t.substring(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function oo(n,t,e){var r=Ml.get(t.substring(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}function ao(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=~~(Math.abs(t)/60),u=Math.abs(t)%60;return e+Pi(r,"0",2)+Pi(u,"0",2)}function co(n,t,e){pl.lastIndex=0;var r=pl.exec(t.substring(e,e+1));return r?e+r[0].length:-1}function lo(n){function t(n){try{Jc=zi;var t=new Jc;return t._=n,e(t)}finally{Jc=Date}}var e=ji(n);return t.parse=function(n){try{Jc=zi;var t=e.parse(n);return t&&t._}finally{Jc=Date}},t.toString=e.toString,t}function so(n){return n.toISOString()}function fo(n,t,e){function r(t){return n(t)}function u(n,e){var r=n[1]-n[0],u=r/e,i=mo.bisect(bl,u);return i==bl.length?[t.year,ei(n.map(function(n){return n/31536e6}),e)[2]]:i?t[u/bl[i-1]<bl[i]/u?i-1:i]:[El,ei(n,e)[2]]}return r.invert=function(t){return ho(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain(t),r):n.domain().map(ho)},r.nice=function(n,t){function e(e){return!isNaN(e)&&!n.range(e,ho(+e+1),t).length}var i=r.domain(),o=$u(i),a=null==n?u(o,10):"number"==typeof n&&u(o,n);return a&&(n=a[0],t=a[1]),r.domain(Ju(i,t>1?{floor:function(t){for(;e(t=n.floor(t));)t=ho(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=ho(+t+1);return t}}:n))},r.ticks=function(n,t){var e=$u(r.domain()),i=null==n?u(e,10):"number"==typeof n?u(e,n):!n.range&&[{range:n},t];return i&&(n=i[0],t=i[1]),n.range(e[0],ho(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return fo(n.copy(),t,e)},ni(r,n)}function ho(n){return new Date(n)}function go(n){return function(t){for(var e=n.length-1,r=n[e];!r[1](t);)r=n[--e];return r[0](t)}}function po(n){return JSON.parse(n.responseText)}function vo(n){var t=xo.createRange();return t.selectNode(xo.body),t.createContextualFragment(n.responseText)}var mo={version:"3.3.6"};Date.now||(Date.now=function(){return+new Date});var yo=[].slice,Mo=function(n){return yo.call(n)},xo=document,bo=xo.documentElement,_o=window;try{Mo(bo.childNodes)[0].nodeType}catch(wo){Mo=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}try{xo.createElement("div").style.setProperty("opacity",0,"")}catch(So){var Eo=_o.Element.prototype,ko=Eo.setAttribute,Ao=Eo.setAttributeNS,No=_o.CSSStyleDeclaration.prototype,To=No.setProperty;Eo.setAttribute=function(n,t){ko.call(this,n,t+"")},Eo.setAttributeNS=function(n,t,e){Ao.call(this,n,t,e+"")},No.setProperty=function(n,t,e){To.call(this,n,t+"",e)}}mo.ascending=function(n,t){return t>n?-1:n>t?1:n>=t?0:0/0},mo.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:0/0},mo.min=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u<i&&!(null!=(e=n[u])&&e>=e);)e=void 0;for(;++u<i;)null!=(r=n[u])&&e>r&&(e=r)}else{for(;++u<i&&!(null!=(e=t.call(n,n[u],u))&&e>=e);)e=void 0;for(;++u<i;)null!=(r=t.call(n,n[u],u))&&e>r&&(e=r)}return e},mo.max=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u<i&&!(null!=(e=n[u])&&e>=e);)e=void 0;for(;++u<i;)null!=(r=n[u])&&r>e&&(e=r)}else{for(;++u<i&&!(null!=(e=t.call(n,n[u],u))&&e>=e);)e=void 0;for(;++u<i;)null!=(r=t.call(n,n[u],u))&&r>e&&(e=r)}return e},mo.extent=function(n,t){var e,r,u,i=-1,o=n.length;if(1===arguments.length){for(;++i<o&&!(null!=(e=u=n[i])&&e>=e);)e=u=void 0;for(;++i<o;)null!=(r=n[i])&&(e>r&&(e=r),r>u&&(u=r))}else{for(;++i<o&&!(null!=(e=u=t.call(n,n[i],i))&&e>=e);)e=void 0;for(;++i<o;)null!=(r=t.call(n,n[i],i))&&(e>r&&(e=r),r>u&&(u=r))}return[e,u]},mo.sum=function(n,t){var e,r=0,u=n.length,i=-1;if(1===arguments.length)for(;++i<u;)isNaN(e=+n[i])||(r+=e);else for(;++i<u;)isNaN(e=+t.call(n,n[i],i))||(r+=e);return r},mo.mean=function(t,e){var r,u=t.length,i=0,o=-1,a=0;if(1===arguments.length)for(;++o<u;)n(r=t[o])&&(i+=(r-i)/++a);else for(;++o<u;)n(r=e.call(t,t[o],o))&&(i+=(r-i)/++a);return a?i:void 0},mo.quantile=function(n,t){var e=(n.length-1)*t+1,r=Math.floor(e),u=+n[r-1],i=e-r;return i?u+i*(n[r]-u):u},mo.median=function(t,e){return arguments.length>1&&(t=t.map(e)),t=t.filter(n),t.length?mo.quantile(t.sort(mo.ascending),.5):void 0},mo.bisector=function(n){return{left:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;n.call(t,t[i],i)<e?r=i+1:u=i}return r},right:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;e<n.call(t,t[i],i)?u=i:r=i+1}return r}}};var qo=mo.bisector(function(n){return n});mo.bisectLeft=qo.left,mo.bisect=mo.bisectRight=qo.right,mo.shuffle=function(n){for(var t,e,r=n.length;r;)e=0|Math.random()*r--,t=n[r],n[r]=n[e],n[e]=t;return n},mo.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},mo.pairs=function(n){for(var t,e=0,r=n.length-1,u=n[0],i=new Array(0>r?0:r);r>e;)i[e]=[t=u,u=n[++e]];return i},mo.zip=function(){if(!(u=arguments.length))return[];for(var n=-1,e=mo.min(arguments,t),r=new Array(e);++n<e;)for(var u,i=-1,o=r[n]=new Array(u);++i<u;)o[i]=arguments[i][n];return r},mo.transpose=function(n){return mo.zip.apply(mo,n)},mo.keys=function(n){var t=[];for(var e in n)t.push(e);return t},mo.values=function(n){var t=[];for(var e in n)t.push(n[e]);return t},mo.entries=function(n){var t=[];
-for(var e in n)t.push({key:e,value:n[e]});return t},mo.merge=function(n){return Array.prototype.concat.apply([],n)},mo.range=function(n,t,r){if(arguments.length<3&&(r=1,arguments.length<2&&(t=n,n=0)),1/0===(t-n)/r)throw new Error("infinite range");var u,i=[],o=e(Math.abs(r)),a=-1;if(n*=o,t*=o,r*=o,0>r)for(;(u=n+r*++a)>t;)i.push(u/o);else for(;(u=n+r*++a)<t;)i.push(u/o);return i},mo.map=function(n){var t=new u;if(n instanceof u)n.forEach(function(n,e){t.set(n,e)});else for(var e in n)t.set(e,n[e]);return t},r(u,{has:function(n){return zo+n in this},get:function(n){return this[zo+n]},set:function(n,t){return this[zo+n]=t},remove:function(n){return n=zo+n,n in this&&delete this[n]},keys:function(){var n=[];return this.forEach(function(t){n.push(t)}),n},values:function(){var n=[];return this.forEach(function(t,e){n.push(e)}),n},entries:function(){var n=[];return this.forEach(function(t,e){n.push({key:t,value:e})}),n},forEach:function(n){for(var t in this)t.charCodeAt(0)===Co&&n.call(this,t.substring(1),this[t])}});var zo="\x00",Co=zo.charCodeAt(0);mo.nest=function(){function n(t,a,c){if(c>=o.length)return r?r.call(i,a):e?a.sort(e):a;for(var l,s,f,h,g=-1,p=a.length,d=o[c++],v=new u;++g<p;)(h=v.get(l=d(s=a[g])))?h.push(s):v.set(l,[s]);return t?(s=t(),f=function(e,r){s.set(e,n(t,r,c))}):(s={},f=function(e,r){s[e]=n(t,r,c)}),v.forEach(f),s}function t(n,e){if(e>=o.length)return n;var r=[],u=a[e++];return n.forEach(function(n,u){r.push({key:n,values:t(u,e)})}),u?r.sort(function(n,t){return u(n.key,t.key)}):r}var e,r,i={},o=[],a=[];return i.map=function(t,e){return n(e,t,0)},i.entries=function(e){return t(n(mo.map,e,0),0)},i.key=function(n){return o.push(n),i},i.sortKeys=function(n){return a[o.length-1]=n,i},i.sortValues=function(n){return e=n,i},i.rollup=function(n){return r=n,i},i},mo.set=function(n){var t=new i;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},r(i,{has:function(n){return zo+n in this},add:function(n){return this[zo+n]=!0,n},remove:function(n){return n=zo+n,n in this&&delete this[n]},values:function(){var n=[];return this.forEach(function(t){n.push(t)}),n},forEach:function(n){for(var t in this)t.charCodeAt(0)===Co&&n.call(this,t.substring(1))}}),mo.behavior={},mo.rebind=function(n,t){for(var e,r=1,u=arguments.length;++r<u;)n[e=arguments[r]]=o(n,t,t[e]);return n};var Do=["webkit","ms","moz","Moz","o","O"];mo.dispatch=function(){for(var n=new l,t=-1,e=arguments.length;++t<e;)n[arguments[t]]=s(n);return n},l.prototype.on=function(n,t){var e=n.indexOf("."),r="";if(e>=0&&(r=n.substring(e+1),n=n.substring(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},mo.event=null,mo.requote=function(n){return n.replace(jo,"\\$&")};var jo=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,Lo={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},Ho=function(n,t){return t.querySelector(n)},Fo=function(n,t){return t.querySelectorAll(n)},Po=bo[a(bo,"matchesSelector")],Oo=function(n,t){return Po.call(n,t)};"function"==typeof Sizzle&&(Ho=function(n,t){return Sizzle(n,t)[0]||null},Fo=function(n,t){return Sizzle.uniqueSort(Sizzle(n,t))},Oo=Sizzle.matchesSelector),mo.selection=function(){return Uo};var Ro=mo.selection.prototype=[];Ro.select=function(n){var t,e,r,u,i=[];n=d(n);for(var o=-1,a=this.length;++o<a;){i.push(t=[]),t.parentNode=(r=this[o]).parentNode;for(var c=-1,l=r.length;++c<l;)(u=r[c])?(t.push(e=n.call(u,u.__data__,c,o)),e&&"__data__"in u&&(e.__data__=u.__data__)):t.push(null)}return p(i)},Ro.selectAll=function(n){var t,e,r=[];n=v(n);for(var u=-1,i=this.length;++u<i;)for(var o=this[u],a=-1,c=o.length;++a<c;)(e=o[a])&&(r.push(t=Mo(n.call(e,e.__data__,a,u))),t.parentNode=e);return p(r)};var Yo={svg:"http://www.w3.org/2000/svg",xhtml:"http://www.w3.org/1999/xhtml",xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};mo.ns={prefix:Yo,qualify:function(n){var t=n.indexOf(":"),e=n;return t>=0&&(e=n.substring(0,t),n=n.substring(t+1)),Yo.hasOwnProperty(e)?{space:Yo[e],local:n}:n}},Ro.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=mo.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(m(t,n[t]));return this}return this.each(m(n,t))},Ro.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=n.trim().split(/^|\s+/g)).length,u=-1;if(t=e.classList){for(;++u<r;)if(!t.contains(n[u]))return!1}else for(t=e.getAttribute("class");++u<r;)if(!M(n[u]).test(t))return!1;return!0}for(t in n)this.each(x(t,n[t]));return this}return this.each(x(n,t))},Ro.style=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t="");for(e in n)this.each(_(e,n[e],t));return this}if(2>r)return _o.getComputedStyle(this.node(),null).getPropertyValue(n);e=""}return this.each(_(n,t,e))},Ro.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(w(t,n[t]));return this}return this.each(w(n,t))},Ro.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},Ro.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},Ro.append=function(n){return n=S(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},Ro.insert=function(n,t){return n=S(n),t=d(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments))})},Ro.remove=function(){return this.each(function(){var n=this.parentNode;n&&n.removeChild(this)})},Ro.data=function(n,t){function e(n,e){var r,i,o,a=n.length,f=e.length,h=Math.min(a,f),g=new Array(f),p=new Array(f),d=new Array(a);if(t){var v,m=new u,y=new u,M=[];for(r=-1;++r<a;)v=t.call(i=n[r],i.__data__,r),m.has(v)?d[r]=i:m.set(v,i),M.push(v);for(r=-1;++r<f;)v=t.call(e,o=e[r],r),(i=m.get(v))?(g[r]=i,i.__data__=o):y.has(v)||(p[r]=E(o)),y.set(v,o),m.remove(v);for(r=-1;++r<a;)m.has(M[r])&&(d[r]=n[r])}else{for(r=-1;++r<h;)i=n[r],o=e[r],i?(i.__data__=o,g[r]=i):p[r]=E(o);for(;f>r;++r)p[r]=E(e[r]);for(;a>r;++r)d[r]=n[r]}p.update=g,p.parentNode=g.parentNode=d.parentNode=n.parentNode,c.push(p),l.push(g),s.push(d)}var r,i,o=-1,a=this.length;if(!arguments.length){for(n=new Array(a=(r=this[0]).length);++o<a;)(i=r[o])&&(n[o]=i.__data__);return n}var c=T([]),l=p([]),s=p([]);if("function"==typeof n)for(;++o<a;)e(r=this[o],n.call(r,r.parentNode.__data__,o));else for(;++o<a;)e(r=this[o],n);return l.enter=function(){return c},l.exit=function(){return s},l},Ro.datum=function(n){return arguments.length?this.property("__data__",n):this.property("__data__")},Ro.filter=function(n){var t,e,r,u=[];"function"!=typeof n&&(n=k(n));for(var i=0,o=this.length;o>i;i++){u.push(t=[]),t.parentNode=(e=this[i]).parentNode;for(var a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a)&&t.push(r)}return p(u)},Ro.order=function(){for(var n=-1,t=this.length;++n<t;)for(var e,r=this[n],u=r.length-1,i=r[u];--u>=0;)(e=r[u])&&(i&&i!==e.nextSibling&&i.parentNode.insertBefore(e,i),i=e);return this},Ro.sort=function(n){n=A.apply(this,arguments);for(var t=-1,e=this.length;++t<e;)this[t].sort(n);return this.order()},Ro.each=function(n){return N(this,function(t,e,r){n.call(t,t.__data__,e,r)})},Ro.call=function(n){var t=Mo(arguments);return n.apply(t[0]=this,t),this},Ro.empty=function(){return!this.node()},Ro.node=function(){for(var n=0,t=this.length;t>n;n++)for(var e=this[n],r=0,u=e.length;u>r;r++){var i=e[r];if(i)return i}return null},Ro.size=function(){var n=0;return this.each(function(){++n}),n};var Io=[];mo.selection.enter=T,mo.selection.enter.prototype=Io,Io.append=Ro.append,Io.empty=Ro.empty,Io.node=Ro.node,Io.call=Ro.call,Io.size=Ro.size,Io.select=function(n){for(var t,e,r,u,i,o=[],a=-1,c=this.length;++a<c;){r=(u=this[a]).update,o.push(t=[]),t.parentNode=u.parentNode;for(var l=-1,s=u.length;++l<s;)(i=u[l])?(t.push(r[l]=e=n.call(u.parentNode,i.__data__,l,a)),e.__data__=i.__data__):t.push(null)}return p(o)},Io.insert=function(n,t){return arguments.length<2&&(t=q(this)),Ro.insert.call(this,n,t)},Ro.transition=function(){for(var n,t,e=Oc||++Zc,r=[],u=Rc||{time:Date.now(),ease:Cr,delay:0,duration:250},i=-1,o=this.length;++i<o;){r.push(n=[]);for(var a=this[i],c=-1,l=a.length;++c<l;)(t=a[c])&&Ni(t,c,e,u),n.push(t)}return Ei(r,e)},Ro.interrupt=function(){return this.each(z)},mo.select=function(n){var t=["string"==typeof n?Ho(n,xo):n];return t.parentNode=bo,p([t])},mo.selectAll=function(n){var t=Mo("string"==typeof n?Fo(n,xo):n);return t.parentNode=bo,p([t])};var Uo=mo.select(bo);Ro.on=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(C(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(C(n,t,e))};var Zo=mo.map({mouseenter:"mouseover",mouseleave:"mouseout"});Zo.forEach(function(n){"on"+n in xo&&Zo.remove(n)});var Vo=a(bo.style,"userSelect"),Xo=0;mo.mouse=function(n){return H(n,h())};var $o=/WebKit/.test(_o.navigator.userAgent)?-1:0;mo.touches=function(n,t){return arguments.length<2&&(t=h().touches),t?Mo(t).map(function(t){var e=H(n,t);return e.identifier=t.identifier,e}):[]},mo.behavior.drag=function(){function n(){this.on("mousedown.drag",o).on("touchstart.drag",a)}function t(){return mo.event.changedTouches[0].identifier}function e(n,t){return mo.touches(n).filter(function(n){return n.identifier===t})[0]}function r(n,t,e,r){return function(){function o(){var n=t(s,g),e=n[0]-d[0],r=n[1]-d[1];v|=e|r,d=n,f({type:"drag",x:n[0]+c[0],y:n[1]+c[1],dx:e,dy:r})}function a(){m.on(e+"."+p,null).on(r+"."+p,null),y(v&&mo.event.target===h),f({type:"dragend"})}var c,l=this,s=l.parentNode,f=u.of(l,arguments),h=mo.event.target,g=n(),p=null==g?"drag":"drag-"+g,d=t(s,g),v=0,m=mo.select(_o).on(e+"."+p,o).on(r+"."+p,a),y=L();i?(c=i.apply(l,arguments),c=[c.x-d[0],c.y-d[1]]):c=[0,0],f({type:"dragstart"})}}var u=g(n,"drag","dragstart","dragend"),i=null,o=r(c,mo.mouse,"mousemove","mouseup"),a=r(t,e,"touchmove","touchend");return n.origin=function(t){return arguments.length?(i=t,n):i},mo.rebind(n,u,"on")};var Bo=Math.PI,Wo=2*Bo,Jo=Bo/2,Go=1e-6,Ko=Go*Go,Qo=Bo/180,na=180/Bo,ta=Math.SQRT2,ea=2,ra=4;mo.interpolateZoom=function(n,t){function e(n){var t=n*y;if(m){var e=Y(d),o=i/(ea*h)*(e*I(ta*t+d)-R(d));return[r+o*l,u+o*s,i*e/Y(ta*t+d)]}return[r+n*l,u+n*s,i*Math.exp(ta*t)]}var r=n[0],u=n[1],i=n[2],o=t[0],a=t[1],c=t[2],l=o-r,s=a-u,f=l*l+s*s,h=Math.sqrt(f),g=(c*c-i*i+ra*f)/(2*i*ea*h),p=(c*c-i*i-ra*f)/(2*c*ea*h),d=Math.log(Math.sqrt(g*g+1)-g),v=Math.log(Math.sqrt(p*p+1)-p),m=v-d,y=(m||Math.log(c/i))/ta;return e.duration=1e3*y,e},mo.behavior.zoom=function(){function n(n){n.on(A,l).on(oa+".zoom",h).on(N,p).on("dblclick.zoom",d).on(q,s)}function t(n){return[(n[0]-S.x)/S.k,(n[1]-S.y)/S.k]}function e(n){return[n[0]*S.k+S.x,n[1]*S.k+S.y]}function r(n){S.k=Math.max(k[0],Math.min(k[1],n))}function u(n,t){t=e(t),S.x+=n[0]-t[0],S.y+=n[1]-t[1]}function i(){b&&b.domain(x.range().map(function(n){return(n-S.x)/S.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-S.y)/S.k}).map(_.invert))}function o(n){n({type:"zoomstart"})}function a(n){i(),n({type:"zoom",scale:S.k,translate:[S.x,S.y]})}function c(n){n({type:"zoomend"})}function l(){function n(){s=1,u(mo.mouse(r),h),a(i)}function e(){f.on(N,_o===r?p:null).on(T,null),g(s&&mo.event.target===l),c(i)}var r=this,i=C.of(r,arguments),l=mo.event.target,s=0,f=mo.select(_o).on(N,n).on(T,e),h=t(mo.mouse(r)),g=L();z.call(r),o(i)}function s(){function n(){var n=mo.touches(p);return g=S.k,n.forEach(function(n){n.identifier in v&&(v[n.identifier]=t(n))}),n}function e(){for(var t=mo.event.changedTouches,e=0,i=t.length;i>e;++e)v[t[e].identifier]=null;var o=n(),c=Date.now();if(1===o.length){if(500>c-M){var l=o[0],s=v[l.identifier];r(2*S.k),u(l,s),f(),a(d)}M=c}else if(o.length>1){var l=o[0],h=o[1],g=l[0]-h[0],p=l[1]-h[1];m=g*g+p*p}}function i(){for(var n,t,e,i,o=mo.touches(p),c=0,l=o.length;l>c;++c,i=null)if(e=o[c],i=v[e.identifier]){if(t)break;n=e,t=i}if(i){var s=(s=e[0]-n[0])*s+(s=e[1]-n[1])*s,f=m&&Math.sqrt(s/m);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+i[0])/2,(t[1]+i[1])/2],r(f*g)}M=null,u(n,t),a(d)}function h(){if(mo.event.touches.length){for(var t=mo.event.changedTouches,e=0,r=t.length;r>e;++e)delete v[t[e].identifier];for(var u in v)return void n()}_.on(x,null).on(b,null),w.on(A,l).on(q,s),E(),c(d)}var g,p=this,d=C.of(p,arguments),v={},m=0,y=mo.event.changedTouches[0].identifier,x="touchmove.zoom-"+y,b="touchend.zoom-"+y,_=mo.select(_o).on(x,i).on(b,h),w=mo.select(p).on(A,null).on(q,e),E=L();z.call(p),e(),o(d)}function h(){var n=C.of(this,arguments);y?clearTimeout(y):(z.call(this),o(n)),y=setTimeout(function(){y=null,c(n)},50),f();var e=m||mo.mouse(this);v||(v=t(e)),r(Math.pow(2,.002*ua())*S.k),u(e,v),a(n)}function p(){v=null}function d(){var n=C.of(this,arguments),e=mo.mouse(this),i=t(e),l=Math.log(S.k)/Math.LN2;o(n),r(Math.pow(2,mo.event.shiftKey?Math.ceil(l)-1:Math.floor(l)+1)),u(e,i),a(n),c(n)}var v,m,y,M,x,b,_,w,S={x:0,y:0,k:1},E=[960,500],k=ia,A="mousedown.zoom",N="mousemove.zoom",T="mouseup.zoom",q="touchstart.zoom",C=g(n,"zoomstart","zoom","zoomend");return n.event=function(n){n.each(function(){var n=C.of(this,arguments),t=S;Oc?mo.select(this).transition().each("start.zoom",function(){S=this.__chart__||{x:0,y:0,k:1},o(n)}).tween("zoom:zoom",function(){var e=E[0],r=E[1],u=e/2,i=r/2,o=mo.interpolateZoom([(u-S.x)/S.k,(i-S.y)/S.k,e/S.k],[(u-t.x)/t.k,(i-t.y)/t.k,e/t.k]);return function(t){var r=o(t),c=e/r[2];this.__chart__=S={x:u-r[0]*c,y:i-r[1]*c,k:c},a(n)}}).each("end.zoom",function(){c(n)}):(this.__chart__=S,o(n),a(n),c(n))})},n.translate=function(t){return arguments.length?(S={x:+t[0],y:+t[1],k:S.k},i(),n):[S.x,S.y]},n.scale=function(t){return arguments.length?(S={x:S.x,y:S.y,k:+t},i(),n):S.k},n.scaleExtent=function(t){return arguments.length?(k=null==t?ia:[+t[0],+t[1]],n):k},n.center=function(t){return arguments.length?(m=t&&[+t[0],+t[1]],n):m},n.size=function(t){return arguments.length?(E=t&&[+t[0],+t[1]],n):E},n.x=function(t){return arguments.length?(b=t,x=t.copy(),S={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),S={x:0,y:0,k:1},n):w},mo.rebind(n,C,"on")};var ua,ia=[0,1/0],oa="onwheel"in xo?(ua=function(){return-mo.event.deltaY*(mo.event.deltaMode?120:1)},"wheel"):"onmousewheel"in xo?(ua=function(){return mo.event.wheelDelta},"mousewheel"):(ua=function(){return-mo.event.detail},"MozMousePixelScroll");Z.prototype.toString=function(){return this.rgb()+""},mo.hsl=function(n,t,e){return 1===arguments.length?n instanceof X?V(n.h,n.s,n.l):lt(""+n,st,V):V(+n,+t,+e)};var aa=X.prototype=new Z;aa.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),V(this.h,this.s,this.l/n)},aa.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),V(this.h,this.s,n*this.l)},aa.rgb=function(){return $(this.h,this.s,this.l)},mo.hcl=function(n,t,e){return 1===arguments.length?n instanceof W?B(n.h,n.c,n.l):n instanceof K?nt(n.l,n.a,n.b):nt((n=ft((n=mo.rgb(n)).r,n.g,n.b)).l,n.a,n.b):B(+n,+t,+e)};var ca=W.prototype=new Z;ca.brighter=function(n){return B(this.h,this.c,Math.min(100,this.l+la*(arguments.length?n:1)))},ca.darker=function(n){return B(this.h,this.c,Math.max(0,this.l-la*(arguments.length?n:1)))},ca.rgb=function(){return J(this.h,this.c,this.l).rgb()},mo.lab=function(n,t,e){return 1===arguments.length?n instanceof K?G(n.l,n.a,n.b):n instanceof W?J(n.l,n.c,n.h):ft((n=mo.rgb(n)).r,n.g,n.b):G(+n,+t,+e)};var la=18,sa=.95047,fa=1,ha=1.08883,ga=K.prototype=new Z;ga.brighter=function(n){return G(Math.min(100,this.l+la*(arguments.length?n:1)),this.a,this.b)},ga.darker=function(n){return G(Math.max(0,this.l-la*(arguments.length?n:1)),this.a,this.b)},ga.rgb=function(){return Q(this.l,this.a,this.b)},mo.rgb=function(n,t,e){return 1===arguments.length?n instanceof at?ot(n.r,n.g,n.b):lt(""+n,ot,$):ot(~~n,~~t,~~e)};var pa=at.prototype=new Z;pa.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,u=30;return t||e||r?(t&&u>t&&(t=u),e&&u>e&&(e=u),r&&u>r&&(r=u),ot(Math.min(255,~~(t/n)),Math.min(255,~~(e/n)),Math.min(255,~~(r/n)))):ot(u,u,u)},pa.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),ot(~~(n*this.r),~~(n*this.g),~~(n*this.b))},pa.hsl=function(){return st(this.r,this.g,this.b)},pa.toString=function(){return"#"+ct(this.r)+ct(this.g)+ct(this.b)};var da=mo.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});da.forEach(function(n,t){da.set(n,ut(t))}),mo.functor=pt,mo.xhr=vt(dt),mo.dsv=function(n,t){function e(n,e,i){arguments.length<3&&(i=e,e=null);var o=mo.xhr(n,t,i);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:u(n)):e},o.row(e)}function r(n){return e.parse(n.responseText)}function u(n){return function(t){return e.parse(t.responseText,n)}}function o(t){return t.map(a).join(n)}function a(n){return c.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var c=new RegExp('["'+n+"\n]"),l=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var u=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(u(n),e)}:u})},e.parseRows=function(n,t){function e(){if(s>=c)return o;if(u)return u=!1,i;var t=s;if(34===n.charCodeAt(t)){for(var e=t;e++<c;)if(34===n.charCodeAt(e)){if(34!==n.charCodeAt(e+1))break;++e}s=e+2;var r=n.charCodeAt(e+1);return 13===r?(u=!0,10===n.charCodeAt(e+2)&&++s):10===r&&(u=!0),n.substring(t+1,e).replace(/""/g,'"')}for(;c>s;){var r=n.charCodeAt(s++),a=1;if(10===r)u=!0;else if(13===r)u=!0,10===n.charCodeAt(s)&&(++s,++a);else if(r!==l)continue;return n.substring(t,s-a)}return n.substring(t)}for(var r,u,i={},o={},a=[],c=n.length,s=0,f=0;(r=e())!==o;){for(var h=[];r!==i&&r!==o;)h.push(r),r=e();(!t||(h=t(h,f++)))&&a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new i,u=[];return t.forEach(function(n){for(var t in n)r.has(t)||u.push(r.add(t))}),[u.map(a).join(n)].concat(t.map(function(t){return u.map(function(n){return a(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(o).join("\n")},e},mo.csv=mo.dsv(",","text/csv"),mo.tsv=mo.dsv("	","text/tab-separated-values");var va,ma,ya,Ma,xa,ba=_o[a(_o,"requestAnimationFrame")]||function(n){setTimeout(n,17)};mo.timer=function(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var u=e+t,i={callback:n,time:u,next:null};ma?ma.next=i:va=i,ma=i,ya||(Ma=clearTimeout(Ma),ya=1,ba(Mt))},mo.timer.flush=function(){bt(),_t()};var _a=".",wa=",",Sa=[3,3],Ea="$",ka=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(wt);mo.formatPrefix=function(n,t){var e=0;return n&&(0>n&&(n*=-1),t&&(n=mo.round(n,St(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((0>=e?e+1:e-1)/3)))),ka[8+e/3]},mo.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)},mo.format=function(n){var t=Aa.exec(n),e=t[1]||" ",r=t[2]||">",u=t[3]||"",i=t[4]||"",o=t[5],a=+t[6],c=t[7],l=t[8],s=t[9],f=1,h="",g=!1;switch(l&&(l=+l.substring(1)),(o||"0"===e&&"="===r)&&(o=e="0",r="=",c&&(a-=Math.floor((a-1)/4))),s){case"n":c=!0,s="g";break;case"%":f=100,h="%",s="f";break;case"p":f=100,h="%",s="r";break;case"b":case"o":case"x":case"X":"#"===i&&(i="0"+s.toLowerCase());case"c":case"d":g=!0,l=0;break;case"s":f=-1,s="r"}"#"===i?i="":"$"===i&&(i=Ea),"r"!=s||l||(s="g"),null!=l&&("g"==s?l=Math.max(1,Math.min(21,l)):("e"==s||"f"==s)&&(l=Math.max(0,Math.min(20,l)))),s=Na.get(s)||Et;var p=o&&c;return function(n){if(g&&n%1)return"";var t=0>n||0===n&&0>1/n?(n=-n,"-"):u;if(0>f){var d=mo.formatPrefix(n,l);n=d.scale(n),h=d.symbol}else n*=f;n=s(n,l);var v=n.lastIndexOf("."),m=0>v?n:n.substring(0,v),y=0>v?"":_a+n.substring(v+1);!o&&c&&(m=Ta(m));var M=i.length+m.length+y.length+(p?0:t.length),x=a>M?new Array(M=a-M+1).join(e):"";return p&&(m=Ta(x+m)),t+=i,n=m+y,("<"===r?t+n+x:">"===r?x+t+n:"^"===r?x.substring(0,M>>=1)+t+n+x.substring(M):t+(p?n:x+n))+h}};var Aa=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,Na=mo.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=mo.round(n,St(n,t))).toFixed(Math.max(0,Math.min(20,St(n*(1+1e-15),t))))}}),Ta=dt;if(Sa){var qa=Sa.length;Ta=function(n){for(var t=n.length,e=[],r=0,u=Sa[0];t>0&&u>0;)e.push(n.substring(t-=u,t+u)),u=Sa[r=(r+1)%qa];return e.reverse().join(wa)}}mo.geo={},kt.prototype={s:0,t:0,add:function(n){At(n,this.t,za),At(za.s,this.s,this),this.s?this.t+=za.t:this.s=za.t},reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var za=new kt;mo.geo.stream=function(n,t){n&&Ca.hasOwnProperty(n.type)?Ca[n.type](n,t):Nt(n,t)};var Ca={Feature:function(n,t){Nt(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,u=e.length;++r<u;)Nt(e[r].geometry,t)}},Da={Sphere:function(n,t){t.sphere()},Point:function(n,t){n=n.coordinates,t.point(n[0],n[1],n[2])},MultiPoint:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)n=e[r],t.point(n[0],n[1],n[2])},LineString:function(n,t){Tt(n.coordinates,t,0)},MultiLineString:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)Tt(e[r],t,0)},Polygon:function(n,t){qt(n.coordinates,t)},MultiPolygon:function(n,t){for(var e=n.coordinates,r=-1,u=e.length;++r<u;)qt(e[r],t)},GeometryCollection:function(n,t){for(var e=n.geometries,r=-1,u=e.length;++r<u;)Nt(e[r],t)}};mo.geo.area=function(n){return ja=0,mo.geo.stream(n,Ha),ja};var ja,La=new kt,Ha={sphere:function(){ja+=4*Bo},point:c,lineStart:c,lineEnd:c,polygonStart:function(){La.reset(),Ha.lineStart=zt},polygonEnd:function(){var n=2*La;ja+=0>n?4*Bo+n:n,Ha.lineStart=Ha.lineEnd=Ha.point=c}};mo.geo.bounds=function(){function n(n,t){M.push(x=[s=n,h=n]),f>t&&(f=t),t>g&&(g=t)}function t(t,e){var r=Ct([t*Qo,e*Qo]);if(m){var u=jt(m,r),i=[u[1],-u[0],0],o=jt(i,u);Ft(o),o=Pt(o);var c=t-p,l=c>0?1:-1,d=o[0]*na*l,v=Math.abs(c)>180;if(v^(d>l*p&&l*t>d)){var y=o[1]*na;y>g&&(g=y)}else if(d=(d+360)%360-180,v^(d>l*p&&l*t>d)){var y=-o[1]*na;f>y&&(f=y)}else f>e&&(f=e),e>g&&(g=e);v?p>t?a(s,t)>a(s,h)&&(h=t):a(t,h)>a(s,h)&&(s=t):h>=s?(s>t&&(s=t),t>h&&(h=t)):t>p?a(s,t)>a(s,h)&&(h=t):a(t,h)>a(s,h)&&(s=t)}else n(t,e);m=r,p=t}function e(){b.point=t}function r(){x[0]=s,x[1]=h,b.point=n,m=null}function u(n,e){if(m){var r=n-p;y+=Math.abs(r)>180?r+(r>0?360:-360):r}else d=n,v=e;Ha.point(n,e),t(n,e)}function i(){Ha.lineStart()}function o(){u(d,v),Ha.lineEnd(),Math.abs(y)>Go&&(s=-(h=180)),x[0]=s,x[1]=h,m=null}function a(n,t){return(t-=n)<0?t+360:t}function c(n,t){return n[0]-t[0]}function l(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:n<t[0]||t[1]<n}var s,f,h,g,p,d,v,m,y,M,x,b={point:n,lineStart:e,lineEnd:r,polygonStart:function(){b.point=u,b.lineStart=i,b.lineEnd=o,y=0,Ha.polygonStart()},polygonEnd:function(){Ha.polygonEnd(),b.point=n,b.lineStart=e,b.lineEnd=r,0>La?(s=-(h=180),f=-(g=90)):y>Go?g=90:-Go>y&&(f=-90),x[0]=s,x[1]=h}};return function(n){g=h=-(s=f=1/0),M=[],mo.geo.stream(n,b);var t=M.length;if(t){M.sort(c);for(var e,r=1,u=M[0],i=[u];t>r;++r)e=M[r],l(e[0],u)||l(e[1],u)?(a(u[0],e[1])>a(u[0],u[1])&&(u[1]=e[1]),a(e[0],u[1])>a(u[0],u[1])&&(u[0]=e[0])):i.push(u=e);for(var o,e,p=-1/0,t=i.length-1,r=0,u=i[t];t>=r;u=e,++r)e=i[r],(o=a(u[1],e[0]))>p&&(p=o,s=e[0],h=u[1])}return M=x=null,1/0===s||1/0===f?[[0/0,0/0],[0/0,0/0]]:[[s,f],[h,g]]}}(),mo.geo.centroid=function(n){Fa=Pa=Oa=Ra=Ya=Ia=Ua=Za=Va=Xa=$a=0,mo.geo.stream(n,Ba);var t=Va,e=Xa,r=$a,u=t*t+e*e+r*r;return Ko>u&&(t=Ia,e=Ua,r=Za,Go>Pa&&(t=Oa,e=Ra,r=Ya),u=t*t+e*e+r*r,Ko>u)?[0/0,0/0]:[Math.atan2(e,t)*na,O(r/Math.sqrt(u))*na]};var Fa,Pa,Oa,Ra,Ya,Ia,Ua,Za,Va,Xa,$a,Ba={sphere:c,point:Rt,lineStart:It,lineEnd:Ut,polygonStart:function(){Ba.lineStart=Zt},polygonEnd:function(){Ba.lineStart=It}},Wa=Bt(Vt,Qt,te,[-Bo,-Bo/2]),Ja=1e9;mo.geo.clipExtent=function(){var n,t,e,r,u,i,o={stream:function(n){return u&&(u.valid=!1),u=i(n),u.valid=!0,u},extent:function(a){return arguments.length?(i=re(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),u&&(u.valid=!1,u=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},(mo.geo.conicEqualArea=function(){return oe(ae)}).raw=ae,mo.geo.albers=function(){return mo.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},mo.geo.albersUsa=function(){function n(n){var i=n[0],o=n[1];return t=null,e(i,o),t||(r(i,o),t)||u(i,o),t}var t,e,r,u,i=mo.geo.albers(),o=mo.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=mo.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),c={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=i.scale(),e=i.translate(),r=(n[0]-e[0])/t,u=(n[1]-e[1])/t;return(u>=.12&&.234>u&&r>=-.425&&-.214>r?o:u>=.166&&.234>u&&r>=-.214&&-.115>r?a:i).invert(n)},n.stream=function(n){var t=i.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,u){t.point(n,u),e.point(n,u),r.point(n,u)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(i.precision(t),o.precision(t),a.precision(t),n):i.precision()},n.scale=function(t){return arguments.length?(i.scale(t),o.scale(.35*t),a.scale(t),n.translate(i.translate())):i.scale()},n.translate=function(t){if(!arguments.length)return i.translate();var l=i.scale(),s=+t[0],f=+t[1];return e=i.translate(t).clipExtent([[s-.455*l,f-.238*l],[s+.455*l,f+.238*l]]).stream(c).point,r=o.translate([s-.307*l,f+.201*l]).clipExtent([[s-.425*l+Go,f+.12*l+Go],[s-.214*l-Go,f+.234*l-Go]]).stream(c).point,u=a.translate([s-.205*l,f+.212*l]).clipExtent([[s-.214*l+Go,f+.166*l+Go],[s-.115*l-Go,f+.234*l-Go]]).stream(c).point,n},n.scale(1070)};var Ga,Ka,Qa,nc,tc,ec,rc={point:c,lineStart:c,lineEnd:c,polygonStart:function(){Ka=0,rc.lineStart=ce},polygonEnd:function(){rc.lineStart=rc.lineEnd=rc.point=c,Ga+=Math.abs(Ka/2)}},uc={point:le,lineStart:c,lineEnd:c,polygonStart:c,polygonEnd:c},ic={point:he,lineStart:ge,lineEnd:pe,polygonStart:function(){ic.lineStart=de},polygonEnd:function(){ic.point=he,ic.lineStart=ge,ic.lineEnd=pe}};mo.geo.transform=function(n){return{stream:function(t){var e=new ye(t);for(var r in n)e[r]=n[r];return e}}},ye.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},mo.geo.path=function(){function n(n){return n&&("function"==typeof a&&i.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=u(i)),mo.geo.stream(n,o)),i.result()}function t(){return o=null,n}var e,r,u,i,o,a=4.5;return n.area=function(n){return Ga=0,mo.geo.stream(n,u(rc)),Ga},n.centroid=function(n){return Oa=Ra=Ya=Ia=Ua=Za=Va=Xa=$a=0,mo.geo.stream(n,u(ic)),$a?[Va/$a,Xa/$a]:Za?[Ia/Za,Ua/Za]:Ya?[Oa/Ya,Ra/Ya]:[0/0,0/0]},n.bounds=function(n){return tc=ec=-(Qa=nc=1/0),mo.geo.stream(n,u(uc)),[[Qa,nc],[tc,ec]]},n.projection=function(n){return arguments.length?(u=(e=n)?n.stream||Me(n):dt,t()):e},n.context=function(n){return arguments.length?(i=null==(r=n)?new se:new ve(n),"function"!=typeof a&&i.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(i.pointRadius(+t),+t),n):a},n.projection(mo.geo.albersUsa()).context(null)},mo.geo.projection=xe,mo.geo.projectionMutator=be,(mo.geo.equirectangular=function(){return xe(we)}).raw=we.invert=we,mo.geo.rotation=function(n){function t(t){return t=n(t[0]*Qo,t[1]*Qo),t[0]*=na,t[1]*=na,t}return n=Ee(n[0]%360*Qo,n[1]*Qo,n.length>2?n[2]*Qo:0),t.invert=function(t){return t=n.invert(t[0]*Qo,t[1]*Qo),t[0]*=na,t[1]*=na,t},t},Se.invert=we,mo.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=Ee(-n[0]*Qo,-n[1]*Qo,0).invert,u=[];return e(null,null,1,{point:function(n,e){u.push(n=t(n,e)),n[0]*=na,n[1]*=na}}),{type:"Polygon",coordinates:[u]}}var t,e,r=[0,0],u=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=Te((t=+r)*Qo,u*Qo),n):t},n.precision=function(r){return arguments.length?(e=Te(t*Qo,(u=+r)*Qo),n):u},n.angle(90)},mo.geo.distance=function(n,t){var e,r=(t[0]-n[0])*Qo,u=n[1]*Qo,i=t[1]*Qo,o=Math.sin(r),a=Math.cos(r),c=Math.sin(u),l=Math.cos(u),s=Math.sin(i),f=Math.cos(i);return Math.atan2(Math.sqrt((e=f*o)*e+(e=l*s-c*f*a)*e),c*s+l*f*a)},mo.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return mo.range(Math.ceil(i/v)*v,u,v).map(h).concat(mo.range(Math.ceil(l/m)*m,c,m).map(g)).concat(mo.range(Math.ceil(r/p)*p,e,p).filter(function(n){return Math.abs(n%v)>Go
-}).map(s)).concat(mo.range(Math.ceil(a/d)*d,o,d).filter(function(n){return Math.abs(n%m)>Go}).map(f))}var e,r,u,i,o,a,c,l,s,f,h,g,p=10,d=p,v=90,m=360,y=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(i).concat(g(c).slice(1),h(u).reverse().slice(1),g(l).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(i=+t[0][0],u=+t[1][0],l=+t[0][1],c=+t[1][1],i>u&&(t=i,i=u,u=t),l>c&&(t=l,l=c,c=t),n.precision(y)):[[i,l],[u,c]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(y)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(v=+t[0],m=+t[1],n):[v,m]},n.minorStep=function(t){return arguments.length?(p=+t[0],d=+t[1],n):[p,d]},n.precision=function(t){return arguments.length?(y=+t,s=ze(a,o,90),f=Ce(r,e,y),h=ze(l,c,90),g=Ce(i,u,y),n):y},n.majorExtent([[-180,-90+Go],[180,90-Go]]).minorExtent([[-180,-80-Go],[180,80+Go]])},mo.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||u.apply(this,arguments)]}}var t,e,r=De,u=je;return n.distance=function(){return mo.geo.distance(t||r.apply(this,arguments),e||u.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(u=t,e="function"==typeof t?null:t,n):u},n.precision=function(){return arguments.length?n:0},n},mo.geo.interpolate=function(n,t){return Le(n[0]*Qo,n[1]*Qo,t[0]*Qo,t[1]*Qo)},mo.geo.length=function(n){return oc=0,mo.geo.stream(n,ac),oc};var oc,ac={sphere:c,point:c,lineStart:He,lineEnd:c,polygonStart:c,polygonEnd:c},cc=Fe(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});(mo.geo.azimuthalEqualArea=function(){return xe(cc)}).raw=cc;var lc=Fe(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},dt);(mo.geo.azimuthalEquidistant=function(){return xe(lc)}).raw=lc,(mo.geo.conicConformal=function(){return oe(Pe)}).raw=Pe,(mo.geo.conicEquidistant=function(){return oe(Oe)}).raw=Oe;var sc=Fe(function(n){return 1/n},Math.atan);(mo.geo.gnomonic=function(){return xe(sc)}).raw=sc,Re.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Jo]},(mo.geo.mercator=function(){return Ye(Re)}).raw=Re;var fc=Fe(function(){return 1},Math.asin);(mo.geo.orthographic=function(){return xe(fc)}).raw=fc;var hc=Fe(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});(mo.geo.stereographic=function(){return xe(hc)}).raw=hc,Ie.invert=function(n,t){return[Math.atan2(R(n),Math.cos(t)),O(Math.sin(t)/Y(n))]},(mo.geo.transverseMercator=function(){return Ye(Ie)}).raw=Ie,mo.geom={},mo.svg={},mo.svg.line=function(){return Ue(dt)};var gc=mo.map({linear:Xe,"linear-closed":$e,step:Be,"step-before":We,"step-after":Je,basis:er,"basis-open":rr,"basis-closed":ur,bundle:ir,cardinal:Qe,"cardinal-open":Ge,"cardinal-closed":Ke,monotone:fr});gc.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var pc=[0,2/3,1/3,0],dc=[0,1/3,2/3,0],vc=[0,1/6,2/3,1/6];mo.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,u,i,o,a,c,l,s,f,h,g,p,d=pt(e),v=pt(r),m=n.length,y=m-1,M=[],x=[],b=0;if(d===Ze&&r===Ve)t=n;else for(i=0,t=[];m>i;++i)t.push([+d.call(this,u=n[i],i),+v.call(this,u,i)]);for(i=1;m>i;++i)(t[i][1]<t[b][1]||t[i][1]==t[b][1]&&t[i][0]<t[b][0])&&(b=i);for(i=0;m>i;++i)i!==b&&(c=t[i][1]-t[b][1],a=t[i][0]-t[b][0],M.push({angle:Math.atan2(c,a),index:i}));for(M.sort(function(n,t){return n.angle-t.angle}),g=M[0].angle,h=M[0].index,f=0,i=1;y>i;++i){if(o=M[i].index,g==M[i].angle){if(a=t[h][0]-t[b][0],c=t[h][1]-t[b][1],l=t[o][0]-t[b][0],s=t[o][1]-t[b][1],a*a+c*c>=l*l+s*s){M[i].index=-1;continue}M[f].index=-1}g=M[i].angle,f=i,h=o}for(x.push(b),i=0,o=0;2>i;++o)M[o].index>-1&&(x.push(M[o].index),i++);for(p=x.length;y>o;++o)if(!(M[o].index<0)){for(;!hr(x[p-2],x[p-1],M[o].index,t);)--p;x[p++]=M[o].index}var _=[];for(i=p-1;i>=0;--i)_.push(n[x[i]]);return _}var e=Ze,r=Ve;return arguments.length?t(n):(t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t)},mo.geom.polygon=function(n){return Lo(n,mc),n};var mc=mo.geom.polygon.prototype=[];mc.area=function(){for(var n,t=-1,e=this.length,r=this[e-1],u=0;++t<e;)n=r,r=this[t],u+=n[1]*r[0]-n[0]*r[1];return.5*u},mc.centroid=function(n){var t,e,r=-1,u=this.length,i=0,o=0,a=this[u-1];for(arguments.length||(n=-1/(6*this.area()));++r<u;)t=a,a=this[r],e=t[0]*a[1]-a[0]*t[1],i+=(t[0]+a[0])*e,o+=(t[1]+a[1])*e;return[i*n,o*n]},mc.clip=function(n){for(var t,e,r,u,i,o,a=dr(n),c=-1,l=this.length-dr(this),s=this[l-1];++c<l;){for(t=n.slice(),n.length=0,u=this[c],i=t[(r=t.length-a)-1],e=-1;++e<r;)o=t[e],gr(o,s,u)?(gr(i,s,u)||n.push(pr(i,o,s,u)),n.push(o)):gr(i,s,u)&&n.push(pr(i,o,s,u)),i=o;a&&n.push(n[0]),s=u}return n},mo.geom.delaunay=function(n){var t=n.map(function(){return[]}),e=[];return vr(n,function(e){t[e.region.l.index].push(n[e.region.r.index])}),t.forEach(function(t,r){var u=n[r],i=u[0],o=u[1];t.forEach(function(n){n.angle=Math.atan2(n[0]-i,n[1]-o)}),t.sort(function(n,t){return n.angle-t.angle});for(var a=0,c=t.length-1;c>a;a++)e.push([u,t[a],t[a+1]])}),e},mo.geom.voronoi=function(n){function t(n){var t,i,o,a=n.map(function(){return[]}),c=pt(e),l=pt(r),s=n.length,f=1e6;if(c===Ze&&l===Ve)t=n;else for(t=new Array(s),o=0;s>o;++o)t[o]=[+c.call(this,i=n[o],o),+l.call(this,i,o)];if(vr(t,function(n){var t,e,r,u,i,o;1===n.a&&n.b>=0?(t=n.ep.r,e=n.ep.l):(t=n.ep.l,e=n.ep.r),1===n.a?(i=t?t.y:-f,r=n.c-n.b*i,o=e?e.y:f,u=n.c-n.b*o):(r=t?t.x:-f,i=n.c-n.a*r,u=e?e.x:f,o=n.c-n.a*u);var c=[r,i],l=[u,o];a[n.region.l.index].push(c,l),a[n.region.r.index].push(c,l)}),a=a.map(function(n,e){var r=t[e][0],u=t[e][1],i=n.map(function(n){return Math.atan2(n[0]-r,n[1]-u)}),o=mo.range(n.length).sort(function(n,t){return i[n]-i[t]});return o.filter(function(n,t){return!t||i[n]-i[o[t-1]]>Go}).map(function(t){return n[t]})}),a.forEach(function(n,e){var r=n.length;if(!r)return n.push([-f,-f],[-f,f],[f,f],[f,-f]);if(!(r>2)){var u=t[e],i=n[0],o=n[1],a=u[0],c=u[1],l=i[0],s=i[1],h=o[0],g=o[1],p=Math.abs(h-l),d=g-s;if(Math.abs(d)<Go){var v=s>c?-f:f;n.push([-f,v],[f,v])}else if(Go>p){var m=l>a?-f:f;n.push([m,-f],[m,f])}else{var v=(l-a)*(g-s)>(h-l)*(s-c)?f:-f,y=Math.abs(d)-p;Math.abs(y)<Go?n.push([0>d?v:-v,v]):(y>0&&(v*=-1),n.push([-f,v],[f,v]))}}}),u)for(o=0;s>o;++o)u.clip(a[o]);for(o=0;s>o;++o)a[o].point=n[o];return a}var e=Ze,r=Ve,u=null;return arguments.length?t(n):(t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.clipExtent=function(n){if(!arguments.length)return u&&[u[0],u[2]];if(null==n)u=null;else{var e=+n[0][0],r=+n[0][1],i=+n[1][0],o=+n[1][1];u=mo.geom.polygon([[e,r],[e,o],[i,o],[i,r]])}return t},t.size=function(n){return arguments.length?t.clipExtent(n&&[[0,0],n]):u&&u[2]},t.links=function(n){var t,u,i,o=n.map(function(){return[]}),a=[],c=pt(e),l=pt(r),s=n.length;if(c===Ze&&l===Ve)t=n;else for(t=new Array(s),i=0;s>i;++i)t[i]=[+c.call(this,u=n[i],i),+l.call(this,u,i)];return vr(t,function(t){var e=t.region.l.index,r=t.region.r.index;o[e][r]||(o[e][r]=o[r][e]=!0,a.push({source:n[e],target:n[r]}))}),a},t.triangles=function(n){if(e===Ze&&r===Ve)return mo.geom.delaunay(n);for(var t,u=new Array(c),i=pt(e),o=pt(r),a=-1,c=n.length;++a<c;)(u[a]=[+i.call(this,t=n[a],a),+o.call(this,t,a)]).data=t;return mo.geom.delaunay(u).map(function(n){return n.map(function(n){return n.data})})},t)};var yc={l:"r",r:"l"};mo.geom.quadtree=function(n,t,e,r,u){function i(n){function i(n,t,e,r,u,i,o,a){if(!isNaN(e)&&!isNaN(r))if(n.leaf){var c=n.x,s=n.y;if(null!=c)if(Math.abs(c-e)+Math.abs(s-r)<.01)l(n,t,e,r,u,i,o,a);else{var f=n.point;n.x=n.y=n.point=null,l(n,f,c,s,u,i,o,a),l(n,t,e,r,u,i,o,a)}else n.x=e,n.y=r,n.point=t}else l(n,t,e,r,u,i,o,a)}function l(n,t,e,r,u,o,a,c){var l=.5*(u+a),s=.5*(o+c),f=e>=l,h=r>=s,g=(h<<1)+f;n.leaf=!1,n=n.nodes[g]||(n.nodes[g]=Mr()),f?u=l:a=l,h?o=s:c=s,i(n,t,e,r,u,o,a,c)}var s,f,h,g,p,d,v,m,y,M=pt(a),x=pt(c);if(null!=t)d=t,v=e,m=r,y=u;else if(m=y=-(d=v=1/0),f=[],h=[],p=n.length,o)for(g=0;p>g;++g)s=n[g],s.x<d&&(d=s.x),s.y<v&&(v=s.y),s.x>m&&(m=s.x),s.y>y&&(y=s.y),f.push(s.x),h.push(s.y);else for(g=0;p>g;++g){var b=+M(s=n[g],g),_=+x(s,g);d>b&&(d=b),v>_&&(v=_),b>m&&(m=b),_>y&&(y=_),f.push(b),h.push(_)}var w=m-d,S=y-v;w>S?y=v+w:m=d+S;var E=Mr();if(E.add=function(n){i(E,n,+M(n,++g),+x(n,g),d,v,m,y)},E.visit=function(n){xr(n,E,d,v,m,y)},g=-1,null==t){for(;++g<p;)i(E,n[g],f[g],h[g],d,v,m,y);--g}else n.forEach(E.add);return f=h=n=s=null,E}var o,a=Ze,c=Ve;return(o=arguments.length)?(a=mr,c=yr,3===o&&(u=e,r=t,e=t=0),i(n)):(i.x=function(n){return arguments.length?(a=n,i):a},i.y=function(n){return arguments.length?(c=n,i):c},i.extent=function(n){return arguments.length?(null==n?t=e=r=u=null:(t=+n[0][0],e=+n[0][1],r=+n[1][0],u=+n[1][1]),i):null==t?null:[[t,e],[r,u]]},i.size=function(n){return arguments.length?(null==n?t=e=r=u=null:(t=e=0,r=+n[0],u=+n[1]),i):null==t?null:[r-t,u-e]},i)},mo.interpolateRgb=br,mo.interpolateObject=_r,mo.interpolateNumber=wr,mo.interpolateString=Sr;var Mc=/[-+]?(?:\d+\.?\d*|\.?\d+)(?:[eE][-+]?\d+)?/g;mo.interpolate=Er,mo.interpolators=[function(n,t){var e=typeof t;return("string"===e?da.has(t)||/^(#|rgb\(|hsl\()/.test(t)?br:Sr:t instanceof Z?br:"object"===e?Array.isArray(t)?kr:_r:wr)(n,t)}],mo.interpolateArray=kr;var xc=function(){return dt},bc=mo.map({linear:xc,poly:Dr,quad:function(){return qr},cubic:function(){return zr},sin:function(){return jr},exp:function(){return Lr},circle:function(){return Hr},elastic:Fr,back:Pr,bounce:function(){return Or}}),_c=mo.map({"in":dt,out:Nr,"in-out":Tr,"out-in":function(n){return Tr(Nr(n))}});mo.ease=function(n){var t=n.indexOf("-"),e=t>=0?n.substring(0,t):n,r=t>=0?n.substring(t+1):"in";return e=bc.get(e)||xc,r=_c.get(r)||dt,Ar(r(e.apply(null,Array.prototype.slice.call(arguments,1))))},mo.interpolateHcl=Rr,mo.interpolateHsl=Yr,mo.interpolateLab=Ir,mo.interpolateRound=Ur,mo.transform=function(n){var t=xo.createElementNS(mo.ns.prefix.svg,"g");return(mo.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new Zr(e?e.matrix:wc)})(n)},Zr.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var wc={a:1,b:0,c:0,d:1,e:0,f:0};mo.interpolateTransform=Br,mo.layout={},mo.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++e<r;)t.push(Gr(n[e]));return t}},mo.layout.chord=function(){function n(){var n,l,f,h,g,p={},d=[],v=mo.range(i),m=[];for(e=[],r=[],n=0,h=-1;++h<i;){for(l=0,g=-1;++g<i;)l+=u[h][g];d.push(l),m.push(mo.range(i)),n+=l}for(o&&v.sort(function(n,t){return o(d[n],d[t])}),a&&m.forEach(function(n,t){n.sort(function(n,e){return a(u[t][n],u[t][e])})}),n=(Wo-s*i)/n,l=0,h=-1;++h<i;){for(f=l,g=-1;++g<i;){var y=v[h],M=m[y][g],x=u[y][M],b=l,_=l+=x*n;p[y+"-"+M]={index:y,subindex:M,startAngle:b,endAngle:_,value:x}}r[y]={index:y,startAngle:f,endAngle:l,value:(l-f)/n},l+=s}for(h=-1;++h<i;)for(g=h-1;++g<i;){var w=p[h+"-"+g],S=p[g+"-"+h];(w.value||S.value)&&e.push(w.value<S.value?{source:S,target:w}:{source:w,target:S})}c&&t()}function t(){e.sort(function(n,t){return c((n.source.value+n.target.value)/2,(t.source.value+t.target.value)/2)})}var e,r,u,i,o,a,c,l={},s=0;return l.matrix=function(n){return arguments.length?(i=(u=n)&&u.length,e=r=null,l):u},l.padding=function(n){return arguments.length?(s=n,e=r=null,l):s},l.sortGroups=function(n){return arguments.length?(o=n,e=r=null,l):o},l.sortSubgroups=function(n){return arguments.length?(a=n,e=null,l):a},l.sortChords=function(n){return arguments.length?(c=n,e&&t(),l):c},l.chords=function(){return e||n(),e},l.groups=function(){return r||n(),r},l},mo.layout.force=function(){function n(n){return function(t,e,r,u){if(t.point!==n){var i=t.cx-n.x,o=t.cy-n.y,a=1/Math.sqrt(i*i+o*o);if(d>(u-e)*a){var c=t.charge*a*a;return n.px-=i*c,n.py-=o*c,!0}if(t.point&&isFinite(a)){var c=t.pointCharge*a*a;n.px-=i*c,n.py-=o*c}}return!t.charge}}function t(n){n.px=mo.event.x,n.py=mo.event.y,a.resume()}var e,r,u,i,o,a={},c=mo.dispatch("start","tick","end"),l=[1,1],s=.9,f=Sc,h=Ec,g=-30,p=.1,d=.8,v=[],m=[];return a.tick=function(){if((r*=.99)<.005)return c.end({type:"end",alpha:r=0}),!0;var t,e,a,f,h,d,y,M,x,b=v.length,_=m.length;for(e=0;_>e;++e)a=m[e],f=a.source,h=a.target,M=h.x-f.x,x=h.y-f.y,(d=M*M+x*x)&&(d=r*i[e]*((d=Math.sqrt(d))-u[e])/d,M*=d,x*=d,h.x-=M*(y=f.weight/(h.weight+f.weight)),h.y-=x*y,f.x+=M*(y=1-y),f.y+=x*y);if((y=r*p)&&(M=l[0]/2,x=l[1]/2,e=-1,y))for(;++e<b;)a=v[e],a.x+=(M-a.x)*y,a.y+=(x-a.y)*y;if(g)for(uu(t=mo.geom.quadtree(v),r,o),e=-1;++e<b;)(a=v[e]).fixed||t.visit(n(a));for(e=-1;++e<b;)a=v[e],a.fixed?(a.x=a.px,a.y=a.py):(a.x-=(a.px-(a.px=a.x))*s,a.y-=(a.py-(a.py=a.y))*s);c.tick({type:"tick",alpha:r})},a.nodes=function(n){return arguments.length?(v=n,a):v},a.links=function(n){return arguments.length?(m=n,a):m},a.size=function(n){return arguments.length?(l=n,a):l},a.linkDistance=function(n){return arguments.length?(f="function"==typeof n?n:+n,a):f},a.distance=a.linkDistance,a.linkStrength=function(n){return arguments.length?(h="function"==typeof n?n:+n,a):h},a.friction=function(n){return arguments.length?(s=+n,a):s},a.charge=function(n){return arguments.length?(g="function"==typeof n?n:+n,a):g},a.gravity=function(n){return arguments.length?(p=+n,a):p},a.theta=function(n){return arguments.length?(d=+n,a):d},a.alpha=function(n){return arguments.length?(n=+n,r?r=n>0?n:0:n>0&&(c.start({type:"start",alpha:r=n}),mo.timer(a.tick)),a):r},a.start=function(){function n(n,r){for(var u,i=t(e),o=-1,a=i.length;++o<a;)if(!isNaN(u=i[o][n]))return u;return Math.random()*r}function t(){if(!c){for(c=[],r=0;p>r;++r)c[r]=[];for(r=0;d>r;++r){var n=m[r];c[n.source.index].push(n.target),c[n.target.index].push(n.source)}}return c[e]}var e,r,c,s,p=v.length,d=m.length,y=l[0],M=l[1];for(e=0;p>e;++e)(s=v[e]).index=e,s.weight=0;for(e=0;d>e;++e)s=m[e],"number"==typeof s.source&&(s.source=v[s.source]),"number"==typeof s.target&&(s.target=v[s.target]),++s.source.weight,++s.target.weight;for(e=0;p>e;++e)s=v[e],isNaN(s.x)&&(s.x=n("x",y)),isNaN(s.y)&&(s.y=n("y",M)),isNaN(s.px)&&(s.px=s.x),isNaN(s.py)&&(s.py=s.y);if(u=[],"function"==typeof f)for(e=0;d>e;++e)u[e]=+f.call(this,m[e],e);else for(e=0;d>e;++e)u[e]=f;if(i=[],"function"==typeof h)for(e=0;d>e;++e)i[e]=+h.call(this,m[e],e);else for(e=0;d>e;++e)i[e]=h;if(o=[],"function"==typeof g)for(e=0;p>e;++e)o[e]=+g.call(this,v[e],e);else for(e=0;p>e;++e)o[e]=g;return a.resume()},a.resume=function(){return a.alpha(.1)},a.stop=function(){return a.alpha(0)},a.drag=function(){return e||(e=mo.behavior.drag().origin(dt).on("dragstart.force",nu).on("drag.force",t).on("dragend.force",tu)),arguments.length?(this.on("mouseover.force",eu).on("mouseout.force",ru).call(e),void 0):e},mo.rebind(a,c,"on")};var Sc=20,Ec=1;mo.layout.hierarchy=function(){function n(t,o,a){var c=u.call(e,t,o);if(t.depth=o,a.push(t),c&&(l=c.length)){for(var l,s,f=-1,h=t.children=[],g=0,p=o+1;++f<l;)s=n(c[f],p,a),s.parent=t,h.push(s),g+=s.value;r&&h.sort(r),i&&(t.value=g)}else i&&(t.value=+i.call(e,t,o)||0);return t}function t(n,r){var u=n.children,o=0;if(u&&(a=u.length))for(var a,c=-1,l=r+1;++c<a;)o+=t(u[c],l);else i&&(o=+i.call(e,n,r)||0);return i&&(n.value=o),o}function e(t){var e=[];return n(t,0,e),e}var r=cu,u=ou,i=au;return e.sort=function(n){return arguments.length?(r=n,e):r},e.children=function(n){return arguments.length?(u=n,e):u},e.value=function(n){return arguments.length?(i=n,e):i},e.revalue=function(n){return t(n,0),n},e},mo.layout.partition=function(){function n(t,e,r,u){var i=t.children;if(t.x=e,t.y=t.depth*u,t.dx=r,t.dy=u,i&&(o=i.length)){var o,a,c,l=-1;for(r=t.value?r/t.value:0;++l<o;)n(a=i[l],e,c=a.value*r,u),e+=c}}function t(n){var e=n.children,r=0;if(e&&(u=e.length))for(var u,i=-1;++i<u;)r=Math.max(r,t(e[i]));return 1+r}function e(e,i){var o=r.call(this,e,i);return n(o[0],0,u[0],u[1]/t(o[0])),o}var r=mo.layout.hierarchy(),u=[1,1];return e.size=function(n){return arguments.length?(u=n,e):u},iu(e,r)},mo.layout.pie=function(){function n(i){var o=i.map(function(e,r){return+t.call(n,e,r)}),a=+("function"==typeof r?r.apply(this,arguments):r),c=(("function"==typeof u?u.apply(this,arguments):u)-a)/mo.sum(o),l=mo.range(i.length);null!=e&&l.sort(e===kc?function(n,t){return o[t]-o[n]}:function(n,t){return e(i[n],i[t])});var s=[];return l.forEach(function(n){var t;s[n]={data:i[n],value:t=o[n],startAngle:a,endAngle:a+=t*c}}),s}var t=Number,e=kc,r=0,u=Wo;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(u=t,n):u},n};var kc={};mo.layout.stack=function(){function n(a,c){var l=a.map(function(e,r){return t.call(n,e,r)}),s=l.map(function(t){return t.map(function(t,e){return[i.call(n,t,e),o.call(n,t,e)]})}),f=e.call(n,s,c);l=mo.permute(l,f),s=mo.permute(s,f);var h,g,p,d=r.call(n,s,c),v=l.length,m=l[0].length;for(g=0;m>g;++g)for(u.call(n,l[0][g],p=d[g],s[0][g][1]),h=1;v>h;++h)u.call(n,l[h][g],p+=s[h-1][g][1],s[h][g][1]);return a}var t=dt,e=gu,r=pu,u=hu,i=su,o=fu;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:Ac.get(t)||gu,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:Nc.get(t)||pu,n):r},n.x=function(t){return arguments.length?(i=t,n):i},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(u=t,n):u},n};var Ac=mo.map({"inside-out":function(n){var t,e,r=n.length,u=n.map(du),i=n.map(vu),o=mo.range(r).sort(function(n,t){return u[n]-u[t]}),a=0,c=0,l=[],s=[];for(t=0;r>t;++t)e=o[t],c>a?(a+=i[e],l.push(e)):(c+=i[e],s.push(e));return s.reverse().concat(l)},reverse:function(n){return mo.range(n.length).reverse()},"default":gu}),Nc=mo.map({silhouette:function(n){var t,e,r,u=n.length,i=n[0].length,o=[],a=0,c=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;i>e;++e)c[e]=(a-o[e])/2;return c},wiggle:function(n){var t,e,r,u,i,o,a,c,l,s=n.length,f=n[0],h=f.length,g=[];for(g[0]=c=l=0,e=1;h>e;++e){for(t=0,u=0;s>t;++t)u+=n[t][e][1];for(t=0,i=0,a=f[e][0]-f[e-1][0];s>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;i+=o*n[t][e][1]}g[e]=c-=u?i/u*a:0,l>c&&(l=c)}for(e=0;h>e;++e)g[e]-=l;return g},expand:function(n){var t,e,r,u=n.length,i=n[0].length,o=1/u,a=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];if(r)for(t=0;u>t;t++)n[t][e][1]/=r;else for(t=0;u>t;t++)n[t][e][1]=o}for(e=0;i>e;++e)a[e]=0;return a},zero:pu});mo.layout.histogram=function(){function n(n,i){for(var o,a,c=[],l=n.map(e,this),s=r.call(this,l,i),f=u.call(this,s,l,i),i=-1,h=l.length,g=f.length-1,p=t?1:1/h;++i<g;)o=c[i]=[],o.dx=f[i+1]-(o.x=f[i]),o.y=0;if(g>0)for(i=-1;++i<h;)a=l[i],a>=s[0]&&a<=s[1]&&(o=c[mo.bisect(f,a,1,g)-1],o.y+=p,o.push(n[i]));return c}var t=!0,e=Number,r=xu,u=yu;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=pt(t),n):r},n.bins=function(t){return arguments.length?(u="number"==typeof t?function(n){return Mu(n,t)}:pt(t),n):u},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},mo.layout.tree=function(){function n(n,i){function o(n,t){var r=n.children,u=n._tree;if(r&&(i=r.length)){for(var i,a,l,s=r[0],f=s,h=-1;++h<i;)l=r[h],o(l,a),f=c(l,a,f),a=l;Tu(n);var g=.5*(s._tree.prelim+l._tree.prelim);t?(u.prelim=t._tree.prelim+e(n,t),u.mod=u.prelim-g):u.prelim=g}else t&&(u.prelim=t._tree.prelim+e(n,t))}function a(n,t){n.x=n._tree.prelim+t;var e=n.children;if(e&&(r=e.length)){var r,u=-1;for(t+=n._tree.mod;++u<r;)a(e[u],t)}}function c(n,t,r){if(t){for(var u,i=n,o=n,a=t,c=n.parent.children[0],l=i._tree.mod,s=o._tree.mod,f=a._tree.mod,h=c._tree.mod;a=wu(a),i=_u(i),a&&i;)c=_u(c),o=wu(o),o._tree.ancestor=n,u=a._tree.prelim+f-i._tree.prelim-l+e(a,i),u>0&&(qu(zu(a,n,r),n,u),l+=u,s+=u),f+=a._tree.mod,l+=i._tree.mod,h+=c._tree.mod,s+=o._tree.mod;a&&!wu(o)&&(o._tree.thread=a,o._tree.mod+=f-s),i&&!_u(c)&&(c._tree.thread=i,c._tree.mod+=l-h,r=n)}return r}var l=t.call(this,n,i),s=l[0];Nu(s,function(n,t){n._tree={ancestor:n,prelim:0,mod:0,change:0,shift:0,number:t?t._tree.number+1:0}}),o(s),a(s,-s._tree.prelim);var f=Su(s,ku),h=Su(s,Eu),g=Su(s,Au),p=f.x-e(f,h)/2,d=h.x+e(h,f)/2,v=g.depth||1;return Nu(s,u?function(n){n.x*=r[0],n.y=n.depth*r[1],delete n._tree}:function(n){n.x=(n.x-p)/(d-p)*r[0],n.y=n.depth/v*r[1],delete n._tree}),l}var t=mo.layout.hierarchy().sort(null).value(null),e=bu,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},iu(n,t)},mo.layout.pack=function(){function n(n,i){var o=e.call(this,n,i),a=o[0],c=u[0],l=u[1],s=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,Nu(a,function(n){n.r=+s(n.value)}),Nu(a,Hu),r){var f=r*(t?1:Math.max(2*a.r/c,2*a.r/l))/2;Nu(a,function(n){n.r+=f}),Nu(a,Hu),Nu(a,function(n){n.r-=f})}return Ou(a,c/2,l/2,t?1:1/Math.max(2*a.r/c,2*a.r/l)),o}var t,e=mo.layout.hierarchy().sort(Cu),r=0,u=[1,1];return n.size=function(t){return arguments.length?(u=t,n):u},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},iu(n,e)},mo.layout.cluster=function(){function n(n,i){var o,a=t.call(this,n,i),c=a[0],l=0;Nu(c,function(n){var t=n.children;t&&t.length?(n.x=Iu(t),n.y=Yu(t)):(n.x=o?l+=e(n,o):0,n.y=0,o=n)});var s=Uu(c),f=Zu(c),h=s.x-e(s,f)/2,g=f.x+e(f,s)/2;return Nu(c,u?function(n){n.x=(n.x-c.x)*r[0],n.y=(c.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(g-h)*r[0],n.y=(1-(c.y?n.y/c.y:1))*r[1]}),a}var t=mo.layout.hierarchy().sort(null).value(null),e=bu,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},iu(n,t)},mo.layout.treemap=function(){function n(n,t){for(var e,r,u=-1,i=n.length;++u<i;)r=(e=n[u]).value*(0>t?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var i=e.children;if(i&&i.length){var o,a,c,l=f(e),s=[],h=i.slice(),p=1/0,d="slice"===g?l.dx:"dice"===g?l.dy:"slice-dice"===g?1&e.depth?l.dy:l.dx:Math.min(l.dx,l.dy);for(n(h,l.dx*l.dy/e.value),s.area=0;(c=h.length)>0;)s.push(o=h[c-1]),s.area+=o.area,"squarify"!==g||(a=r(s,d))<=p?(h.pop(),p=a):(s.area-=s.pop().area,u(s,d,l,!1),d=Math.min(l.dx,l.dy),s.length=s.area=0,p=1/0);s.length&&(u(s,d,l,!0),s.length=s.area=0),i.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var i,o=f(t),a=r.slice(),c=[];for(n(a,o.dx*o.dy/t.value),c.area=0;i=a.pop();)c.push(i),c.area+=i.area,null!=i.z&&(u(c,i.z?o.dx:o.dy,o,!a.length),c.length=c.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,u=0,i=1/0,o=-1,a=n.length;++o<a;)(e=n[o].area)&&(i>e&&(i=e),e>u&&(u=e));return r*=r,t*=t,r?Math.max(t*u*p/r,r/(t*i*p)):1/0}function u(n,t,e,r){var u,i=-1,o=n.length,a=e.x,l=e.y,s=t?c(n.area/t):0;if(t==e.dx){for((r||s>e.dy)&&(s=e.dy);++i<o;)u=n[i],u.x=a,u.y=l,u.dy=s,a+=u.dx=Math.min(e.x+e.dx-a,s?c(u.area/s):0);u.z=!0,u.dx+=e.x+e.dx-a,e.y+=s,e.dy-=s}else{for((r||s>e.dx)&&(s=e.dx);++i<o;)u=n[i],u.x=a,u.y=l,u.dx=s,l+=u.dy=Math.min(e.y+e.dy-l,s?c(u.area/s):0);u.z=!1,u.dy+=e.y+e.dy-l,e.x+=s,e.dx-=s}}function i(r){var u=o||a(r),i=u[0];return i.x=0,i.y=0,i.dx=l[0],i.dy=l[1],o&&a.revalue(i),n([i],i.dx*i.dy/i.value),(o?e:t)(i),h&&(o=u),u}var o,a=mo.layout.hierarchy(),c=Math.round,l=[1,1],s=null,f=Vu,h=!1,g="squarify",p=.5*(1+Math.sqrt(5));return i.size=function(n){return arguments.length?(l=n,i):l},i.padding=function(n){function t(t){var e=n.call(i,t,t.depth);return null==e?Vu(t):Xu(t,"number"==typeof e?[e,e,e,e]:e)}function e(t){return Xu(t,n)}if(!arguments.length)return s;var r;return f=null==(s=n)?Vu:"function"==(r=typeof n)?t:"number"===r?(n=[n,n,n,n],e):e,i},i.round=function(n){return arguments.length?(c=n?Math.round:Number,i):c!=Number},i.sticky=function(n){return arguments.length?(h=n,o=null,i):h},i.ratio=function(n){return arguments.length?(p=n,i):p},i.mode=function(n){return arguments.length?(g=n+"",i):g},iu(i,a)},mo.random={normal:function(n,t){var e=arguments.length;return 2>e&&(t=1),1>e&&(n=0),function(){var e,r,u;do e=2*Math.random()-1,r=2*Math.random()-1,u=e*e+r*r;while(!u||u>1);return n+t*e*Math.sqrt(-2*Math.log(u)/u)}},logNormal:function(){var n=mo.random.normal.apply(mo,arguments);return function(){return Math.exp(n())}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t/n}}},mo.scale={};var Tc={floor:dt,ceil:dt};mo.scale.linear=function(){return Qu([0,1],[0,1],Er,!1)},mo.scale.log=function(){return ii(mo.scale.linear().domain([0,1]),10,!0,[1,10])};var qc=mo.format(".0e"),zc={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};mo.scale.pow=function(){return oi(mo.scale.linear(),1,[0,1])},mo.scale.sqrt=function(){return mo.scale.pow().exponent(.5)},mo.scale.ordinal=function(){return ci([],{t:"range",a:[[]]})},mo.scale.category10=function(){return mo.scale.ordinal().range(Cc)},mo.scale.category20=function(){return mo.scale.ordinal().range(Dc)},mo.scale.category20b=function(){return mo.scale.ordinal().range(jc)},mo.scale.category20c=function(){return mo.scale.ordinal().range(Lc)};var Cc=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(it),Dc=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(it),jc=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(it),Lc=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(it);mo.scale.quantile=function(){return li([],[])},mo.scale.quantize=function(){return si(0,1,[0,1])},mo.scale.threshold=function(){return fi([.5],[0,1])},mo.scale.identity=function(){return hi([0,1])},mo.svg.arc=function(){function n(){var n=t.apply(this,arguments),i=e.apply(this,arguments),o=r.apply(this,arguments)+Hc,a=u.apply(this,arguments)+Hc,c=(o>a&&(c=o,o=a,a=c),a-o),l=Bo>c?"0":"1",s=Math.cos(o),f=Math.sin(o),h=Math.cos(a),g=Math.sin(a);return c>=Fc?n?"M0,"+i+"A"+i+","+i+" 0 1,1 0,"+-i+"A"+i+","+i+" 0 1,1 0,"+i+"M0,"+n+"A"+n+","+n+" 0 1,0 0,"+-n+"A"+n+","+n+" 0 1,0 0,"+n+"Z":"M0,"+i+"A"+i+","+i+" 0 1,1 0,"+-i+"A"+i+","+i+" 0 1,1 0,"+i+"Z":n?"M"+i*s+","+i*f+"A"+i+","+i+" 0 "+l+",1 "+i*h+","+i*g+"L"+n*h+","+n*g+"A"+n+","+n+" 0 "+l+",0 "+n*s+","+n*f+"Z":"M"+i*s+","+i*f+"A"+i+","+i+" 0 "+l+",1 "+i*h+","+i*g+"L0,0"+"Z"}var t=gi,e=pi,r=di,u=vi;return n.innerRadius=function(e){return arguments.length?(t=pt(e),n):t},n.outerRadius=function(t){return arguments.length?(e=pt(t),n):e},n.startAngle=function(t){return arguments.length?(r=pt(t),n):r},n.endAngle=function(t){return arguments.length?(u=pt(t),n):u},n.centroid=function(){var n=(t.apply(this,arguments)+e.apply(this,arguments))/2,i=(r.apply(this,arguments)+u.apply(this,arguments))/2+Hc;return[Math.cos(i)*n,Math.sin(i)*n]},n};var Hc=-Jo,Fc=Wo-Go;mo.svg.line.radial=function(){var n=Ue(mi);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},We.reverse=Je,Je.reverse=We,mo.svg.area=function(){return yi(dt)},mo.svg.area.radial=function(){var n=yi(mi);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},mo.svg.chord=function(){function n(n,a){var c=t(this,i,n,a),l=t(this,o,n,a);return"M"+c.p0+r(c.r,c.p1,c.a1-c.a0)+(e(c,l)?u(c.r,c.p1,c.r,c.p0):u(c.r,c.p1,l.r,l.p0)+r(l.r,l.p1,l.a1-l.a0)+u(l.r,l.p1,c.r,c.p0))+"Z"}function t(n,t,e,r){var u=t.call(n,e,r),i=a.call(n,u,r),o=c.call(n,u,r)+Hc,s=l.call(n,u,r)+Hc;return{r:i,a0:o,a1:s,p0:[i*Math.cos(o),i*Math.sin(o)],p1:[i*Math.cos(s),i*Math.sin(s)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>Bo)+",1 "+t}function u(n,t,e,r){return"Q 0,0 "+r}var i=De,o=je,a=Mi,c=di,l=vi;return n.radius=function(t){return arguments.length?(a=pt(t),n):a},n.source=function(t){return arguments.length?(i=pt(t),n):i},n.target=function(t){return arguments.length?(o=pt(t),n):o},n.startAngle=function(t){return arguments.length?(c=pt(t),n):c},n.endAngle=function(t){return arguments.length?(l=pt(t),n):l},n},mo.svg.diagonal=function(){function n(n,u){var i=t.call(this,n,u),o=e.call(this,n,u),a=(i.y+o.y)/2,c=[i,{x:i.x,y:a},{x:o.x,y:a},o];return c=c.map(r),"M"+c[0]+"C"+c[1]+" "+c[2]+" "+c[3]}var t=De,e=je,r=xi;return n.source=function(e){return arguments.length?(t=pt(e),n):t},n.target=function(t){return arguments.length?(e=pt(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},mo.svg.diagonal.radial=function(){var n=mo.svg.diagonal(),t=xi,e=n.projection;return n.projection=function(n){return arguments.length?e(bi(t=n)):t},n},mo.svg.symbol=function(){function n(n,r){return(Pc.get(t.call(this,n,r))||Si)(e.call(this,n,r))}var t=wi,e=_i;return n.type=function(e){return arguments.length?(t=pt(e),n):t},n.size=function(t){return arguments.length?(e=pt(t),n):e},n};var Pc=mo.map({circle:Si,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*Ic)),e=t*Ic;return"M0,"+-t+"L"+e+",0"+" 0,"+t+" "+-e+",0"+"Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/Yc),e=t*Yc/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/Yc),e=t*Yc/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});mo.svg.symbolTypes=Pc.keys();var Oc,Rc,Yc=Math.sqrt(3),Ic=Math.tan(30*Qo),Uc=[],Zc=0;Uc.call=Ro.call,Uc.empty=Ro.empty,Uc.node=Ro.node,Uc.size=Ro.size,mo.transition=function(n){return arguments.length?Oc?n.transition():n:Uo.transition()},mo.transition.prototype=Uc,Uc.select=function(n){var t,e,r,u=this.id,i=[];n=d(n);for(var o=-1,a=this.length;++o<a;){i.push(t=[]);for(var c=this[o],l=-1,s=c.length;++l<s;)(r=c[l])&&(e=n.call(r,r.__data__,l,o))?("__data__"in r&&(e.__data__=r.__data__),Ni(e,l,u,r.__transition__[u]),t.push(e)):t.push(null)}return Ei(i,u)},Uc.selectAll=function(n){var t,e,r,u,i,o=this.id,a=[];n=v(n);for(var c=-1,l=this.length;++c<l;)for(var s=this[c],f=-1,h=s.length;++f<h;)if(r=s[f]){i=r.__transition__[o],e=n.call(r,r.__data__,f,c),a.push(t=[]);for(var g=-1,p=e.length;++g<p;)(u=e[g])&&Ni(u,g,o,i),t.push(u)}return Ei(a,o)},Uc.filter=function(n){var t,e,r,u=[];"function"!=typeof n&&(n=k(n));for(var i=0,o=this.length;o>i;i++){u.push(t=[]);for(var e=this[i],a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a)&&t.push(r)}return Ei(u,this.id)},Uc.tween=function(n,t){var e=this.id;return arguments.length<2?this.node().__transition__[e].tween.get(n):N(this,null==t?function(t){t.__transition__[e].tween.remove(n)}:function(r){r.__transition__[e].tween.set(n,t)})},Uc.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function u(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function i(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))
-})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?Br:Er,a=mo.ns.qualify(n);return ki(this,"attr."+n,t,a.local?i:u)},Uc.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(u));return r&&function(n){this.setAttribute(u,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(u.space,u.local));return r&&function(n){this.setAttributeNS(u.space,u.local,r(n))}}var u=mo.ns.qualify(n);return this.tween("attr."+n,u.local?r:e)},Uc.style=function(n,t,e){function r(){this.style.removeProperty(n)}function u(t){return null==t?r:(t+="",function(){var r,u=_o.getComputedStyle(this,null).getPropertyValue(n);return u!==t&&(r=Er(u,t),function(t){this.style.setProperty(n,r(t),e)})})}var i=arguments.length;if(3>i){if("string"!=typeof n){2>i&&(t="");for(e in n)this.style(e,n[e],t);return this}e=""}return ki(this,"style."+n,t,u)},Uc.styleTween=function(n,t,e){function r(r,u){var i=t.call(this,r,u,_o.getComputedStyle(this,null).getPropertyValue(n));return i&&function(t){this.style.setProperty(n,i(t),e)}}return arguments.length<3&&(e=""),this.tween("style."+n,r)},Uc.text=function(n){return ki(this,"text",n,Ai)},Uc.remove=function(){return this.each("end.transition",function(){var n;this.__transition__.count<2&&(n=this.parentNode)&&n.removeChild(this)})},Uc.ease=function(n){var t=this.id;return arguments.length<1?this.node().__transition__[t].ease:("function"!=typeof n&&(n=mo.ease.apply(mo,arguments)),N(this,function(e){e.__transition__[t].ease=n}))},Uc.delay=function(n){var t=this.id;return N(this,"function"==typeof n?function(e,r,u){e.__transition__[t].delay=+n.call(e,e.__data__,r,u)}:(n=+n,function(e){e.__transition__[t].delay=n}))},Uc.duration=function(n){var t=this.id;return N(this,"function"==typeof n?function(e,r,u){e.__transition__[t].duration=Math.max(1,n.call(e,e.__data__,r,u))}:(n=Math.max(1,n),function(e){e.__transition__[t].duration=n}))},Uc.each=function(n,t){var e=this.id;if(arguments.length<2){var r=Rc,u=Oc;Oc=e,N(this,function(t,r,u){Rc=t.__transition__[e],n.call(t,t.__data__,r,u)}),Rc=r,Oc=u}else N(this,function(r){var u=r.__transition__[e];(u.event||(u.event=mo.dispatch("start","end"))).on(n,t)});return this},Uc.transition=function(){for(var n,t,e,r,u=this.id,i=++Zc,o=[],a=0,c=this.length;c>a;a++){o.push(n=[]);for(var t=this[a],l=0,s=t.length;s>l;l++)(e=t[l])&&(r=Object.create(e.__transition__[u]),r.delay+=r.duration,Ni(e,l,i,r)),n.push(e)}return Ei(o,i)},mo.svg.axis=function(){function n(n){n.each(function(){var n,l=mo.select(this),s=this.__chart__||e,f=this.__chart__=e.copy(),h=null==c?f.ticks?f.ticks.apply(f,a):f.domain():c,g=null==t?f.tickFormat?f.tickFormat.apply(f,a):dt:t,p=l.selectAll(".tick").data(h,f),d=p.enter().insert("g",".domain").attr("class","tick").style("opacity",Go),v=mo.transition(p.exit()).style("opacity",Go).remove(),m=mo.transition(p).style("opacity",1),y=Bu(f),M=l.selectAll(".domain").data([0]),x=(M.enter().append("path").attr("class","domain"),mo.transition(M));d.append("line"),d.append("text");var b=d.select("line"),_=m.select("line"),w=p.select("text").text(g),S=d.select("text"),E=m.select("text");switch(r){case"bottom":n=Ti,b.attr("y2",u),S.attr("y",Math.max(u,0)+o),_.attr("x2",0).attr("y2",u),E.attr("x",0).attr("y",Math.max(u,0)+o),w.attr("dy",".71em").style("text-anchor","middle"),x.attr("d","M"+y[0]+","+i+"V0H"+y[1]+"V"+i);break;case"top":n=Ti,b.attr("y2",-u),S.attr("y",-(Math.max(u,0)+o)),_.attr("x2",0).attr("y2",-u),E.attr("x",0).attr("y",-(Math.max(u,0)+o)),w.attr("dy","0em").style("text-anchor","middle"),x.attr("d","M"+y[0]+","+-i+"V0H"+y[1]+"V"+-i);break;case"left":n=qi,b.attr("x2",-u),S.attr("x",-(Math.max(u,0)+o)),_.attr("x2",-u).attr("y2",0),E.attr("x",-(Math.max(u,0)+o)).attr("y",0),w.attr("dy",".32em").style("text-anchor","end"),x.attr("d","M"+-i+","+y[0]+"H0V"+y[1]+"H"+-i);break;case"right":n=qi,b.attr("x2",u),S.attr("x",Math.max(u,0)+o),_.attr("x2",u).attr("y2",0),E.attr("x",Math.max(u,0)+o).attr("y",0),w.attr("dy",".32em").style("text-anchor","start"),x.attr("d","M"+i+","+y[0]+"H0V"+y[1]+"H"+i)}if(f.rangeBand){var k=f.rangeBand()/2,A=function(n){return f(n)+k};d.call(n,A),m.call(n,A)}else d.call(n,s),m.call(n,f),v.call(n,f)})}var t,e=mo.scale.linear(),r=Vc,u=6,i=6,o=3,a=[10],c=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Xc?t+"":Vc,n):r},n.ticks=function(){return arguments.length?(a=arguments,n):a},n.tickValues=function(t){return arguments.length?(c=t,n):c},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(u=+t,i=+arguments[e-1],n):u},n.innerTickSize=function(t){return arguments.length?(u=+t,n):u},n.outerTickSize=function(t){return arguments.length?(i=+t,n):i},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var Vc="bottom",Xc={top:1,right:1,bottom:1,left:1};mo.svg.brush=function(){function n(i){i.each(function(){var i=mo.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",u).on("touchstart.brush",u),o=i.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),i.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=i.selectAll(".resize").data(v,dt);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return $c[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var s,f=mo.transition(i),h=mo.transition(o);c&&(s=Bu(c),h.attr("x",s[0]).attr("width",s[1]-s[0]),e(f)),l&&(s=Bu(l),h.attr("y",s[0]).attr("height",s[1]-s[0]),r(f)),t(f)})}function t(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+s[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function e(n){n.select(".extent").attr("x",s[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",s[1]-s[0])}function r(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function u(){function u(){32==mo.event.keyCode&&(N||(M=null,q[0]-=s[1],q[1]-=h[1],N=2),f())}function g(){32==mo.event.keyCode&&2==N&&(q[0]+=s[1],q[1]+=h[1],N=0,f())}function v(){var n=mo.mouse(b),u=!1;x&&(n[0]+=x[0],n[1]+=x[1]),N||(mo.event.altKey?(M||(M=[(s[0]+s[1])/2,(h[0]+h[1])/2]),q[0]=s[+(n[0]<M[0])],q[1]=h[+(n[1]<M[1])]):M=null),k&&m(n,c,0)&&(e(S),u=!0),A&&m(n,l,1)&&(r(S),u=!0),u&&(t(S),w({type:"brush",mode:N?"move":"resize"}))}function m(n,t,e){var r,u,a=Bu(t),c=a[0],l=a[1],f=q[e],g=e?h:s,v=g[1]-g[0];return N&&(c-=f,l-=v+f),r=(e?d:p)?Math.max(c,Math.min(l,n[e])):n[e],N?u=(r+=f)+v:(M&&(f=Math.max(c,Math.min(l,2*M[e]-r))),r>f?(u=r,r=f):u=f),g[0]!=r||g[1]!=u?(e?o=null:i=null,g[0]=r,g[1]=u,!0):void 0}function y(){v(),S.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),mo.select("body").style("cursor",null),z.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),T(),w({type:"brushend"})}var M,x,b=this,_=mo.select(mo.event.target),w=a.of(b,arguments),S=mo.select(b),E=_.datum(),k=!/^(n|s)$/.test(E)&&c,A=!/^(e|w)$/.test(E)&&l,N=_.classed("extent"),T=L(),q=mo.mouse(b),z=mo.select(_o).on("keydown.brush",u).on("keyup.brush",g);if(mo.event.changedTouches?z.on("touchmove.brush",v).on("touchend.brush",y):z.on("mousemove.brush",v).on("mouseup.brush",y),S.interrupt().selectAll("*").interrupt(),N)q[0]=s[0]-q[0],q[1]=h[0]-q[1];else if(E){var C=+/w$/.test(E),D=+/^n/.test(E);x=[s[1-C]-q[0],h[1-D]-q[1]],q[0]=s[C],q[1]=h[D]}else mo.event.altKey&&(M=q.slice());S.style("pointer-events","none").selectAll(".resize").style("display",null),mo.select("body").style("cursor",_.style("cursor")),w({type:"brushstart"}),v()}var i,o,a=g(n,"brushstart","brush","brushend"),c=null,l=null,s=[0,0],h=[0,0],p=!0,d=!0,v=Bc[0];return n.event=function(n){n.each(function(){var n=a.of(this,arguments),t={x:s,y:h,i:i,j:o},e=this.__chart__||t;this.__chart__=t,Oc?mo.select(this).transition().each("start.brush",function(){i=e.i,o=e.j,s=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=kr(s,t.x),r=kr(h,t.y);return i=o=null,function(u){s=t.x=e(u),h=t.y=r(u),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){i=t.i,o=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(c=t,v=Bc[!c<<1|!l],n):c},n.y=function(t){return arguments.length?(l=t,v=Bc[!c<<1|!l],n):l},n.clamp=function(t){return arguments.length?(c&&l?(p=!!t[0],d=!!t[1]):c?p=!!t:l&&(d=!!t),n):c&&l?[p,d]:c?p:l?d:null},n.extent=function(t){var e,r,u,a,f;return arguments.length?(c&&(e=t[0],r=t[1],l&&(e=e[0],r=r[0]),i=[e,r],c.invert&&(e=c(e),r=c(r)),e>r&&(f=e,e=r,r=f),(e!=s[0]||r!=s[1])&&(s=[e,r])),l&&(u=t[0],a=t[1],c&&(u=u[1],a=a[1]),o=[u,a],l.invert&&(u=l(u),a=l(a)),u>a&&(f=u,u=a,a=f),(u!=h[0]||a!=h[1])&&(h=[u,a])),n):(c&&(i?(e=i[0],r=i[1]):(e=s[0],r=s[1],c.invert&&(e=c.invert(e),r=c.invert(r)),e>r&&(f=e,e=r,r=f))),l&&(o?(u=o[0],a=o[1]):(u=h[0],a=h[1],l.invert&&(u=l.invert(u),a=l.invert(a)),u>a&&(f=u,u=a,a=f))),c&&l?[[e,u],[r,a]]:c?[e,r]:l&&[u,a])},n.clear=function(){return n.empty()||(s=[0,0],h=[0,0],i=o=null),n},n.empty=function(){return!!c&&s[0]==s[1]||!!l&&h[0]==h[1]},mo.rebind(n,a,"on")};var $c={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},Bc=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Wc=mo.time={},Jc=Date,Gc=["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"];zi.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){Kc.setUTCDate.apply(this._,arguments)},setDay:function(){Kc.setUTCDay.apply(this._,arguments)},setFullYear:function(){Kc.setUTCFullYear.apply(this._,arguments)},setHours:function(){Kc.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){Kc.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){Kc.setUTCMinutes.apply(this._,arguments)},setMonth:function(){Kc.setUTCMonth.apply(this._,arguments)},setSeconds:function(){Kc.setUTCSeconds.apply(this._,arguments)},setTime:function(){Kc.setTime.apply(this._,arguments)}};var Kc=Date.prototype,Qc="%a %b %e %X %Y",nl="%m/%d/%Y",tl="%H:%M:%S",el=["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],rl=["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],ul=["January","February","March","April","May","June","July","August","September","October","November","December"],il=["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"];Wc.year=Ci(function(n){return n=Wc.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),Wc.years=Wc.year.range,Wc.years.utc=Wc.year.utc.range,Wc.day=Ci(function(n){var t=new Jc(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),Wc.days=Wc.day.range,Wc.days.utc=Wc.day.utc.range,Wc.dayOfYear=function(n){var t=Wc.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},Gc.forEach(function(n,t){n=n.toLowerCase(),t=7-t;var e=Wc[n]=Ci(function(n){return(n=Wc.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=Wc.year(n).getDay();return Math.floor((Wc.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});Wc[n+"s"]=e.range,Wc[n+"s"].utc=e.utc.range,Wc[n+"OfYear"]=function(n){var e=Wc.year(n).getDay();return Math.floor((Wc.dayOfYear(n)+(e+t)%7)/7)}}),Wc.week=Wc.sunday,Wc.weeks=Wc.sunday.range,Wc.weeks.utc=Wc.sunday.utc.range,Wc.weekOfYear=Wc.sundayOfYear,Wc.format=ji;var ol=Hi(el),al=Fi(el),cl=Hi(rl),ll=Fi(rl),sl=Hi(ul),fl=Fi(ul),hl=Hi(il),gl=Fi(il),pl=/^%/,dl={"-":"",_:" ",0:"0"},vl={a:function(n){return rl[n.getDay()]},A:function(n){return el[n.getDay()]},b:function(n){return il[n.getMonth()]},B:function(n){return ul[n.getMonth()]},c:ji(Qc),d:function(n,t){return Pi(n.getDate(),t,2)},e:function(n,t){return Pi(n.getDate(),t,2)},H:function(n,t){return Pi(n.getHours(),t,2)},I:function(n,t){return Pi(n.getHours()%12||12,t,2)},j:function(n,t){return Pi(1+Wc.dayOfYear(n),t,3)},L:function(n,t){return Pi(n.getMilliseconds(),t,3)},m:function(n,t){return Pi(n.getMonth()+1,t,2)},M:function(n,t){return Pi(n.getMinutes(),t,2)},p:function(n){return n.getHours()>=12?"PM":"AM"},S:function(n,t){return Pi(n.getSeconds(),t,2)},U:function(n,t){return Pi(Wc.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return Pi(Wc.mondayOfYear(n),t,2)},x:ji(nl),X:ji(tl),y:function(n,t){return Pi(n.getFullYear()%100,t,2)},Y:function(n,t){return Pi(n.getFullYear()%1e4,t,4)},Z:ao,"%":function(){return"%"}},ml={a:Oi,A:Ri,b:Zi,B:Vi,c:Xi,d:no,e:no,H:eo,I:eo,j:to,L:io,m:Qi,M:ro,p:oo,S:uo,U:Ii,w:Yi,W:Ui,x:$i,X:Bi,y:Ji,Y:Wi,Z:Gi,"%":co},yl=/^\s*\d+/,Ml=mo.map({am:0,pm:1});ji.utc=lo;var xl=lo("%Y-%m-%dT%H:%M:%S.%LZ");ji.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?so:xl,so.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},so.toString=xl.toString,Wc.second=Ci(function(n){return new Jc(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),Wc.seconds=Wc.second.range,Wc.seconds.utc=Wc.second.utc.range,Wc.minute=Ci(function(n){return new Jc(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),Wc.minutes=Wc.minute.range,Wc.minutes.utc=Wc.minute.utc.range,Wc.hour=Ci(function(n){var t=n.getTimezoneOffset()/60;return new Jc(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),Wc.hours=Wc.hour.range,Wc.hours.utc=Wc.hour.utc.range,Wc.month=Ci(function(n){return n=Wc.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),Wc.months=Wc.month.range,Wc.months.utc=Wc.month.utc.range;var bl=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],_l=[[Wc.second,1],[Wc.second,5],[Wc.second,15],[Wc.second,30],[Wc.minute,1],[Wc.minute,5],[Wc.minute,15],[Wc.minute,30],[Wc.hour,1],[Wc.hour,3],[Wc.hour,6],[Wc.hour,12],[Wc.day,1],[Wc.day,2],[Wc.week,1],[Wc.month,1],[Wc.month,3],[Wc.year,1]],wl=[[ji("%Y"),Vt],[ji("%B"),function(n){return n.getMonth()}],[ji("%b %d"),function(n){return 1!=n.getDate()}],[ji("%a %d"),function(n){return n.getDay()&&1!=n.getDate()}],[ji("%I %p"),function(n){return n.getHours()}],[ji("%I:%M"),function(n){return n.getMinutes()}],[ji(":%S"),function(n){return n.getSeconds()}],[ji(".%L"),function(n){return n.getMilliseconds()}]],Sl=go(wl);_l.year=Wc.year,Wc.scale=function(){return fo(mo.scale.linear(),_l,Sl)};var El={range:function(n,t,e){return mo.range(+n,+t,e).map(ho)}},kl=_l.map(function(n){return[n[0].utc,n[1]]}),Al=[[lo("%Y"),Vt],[lo("%B"),function(n){return n.getUTCMonth()}],[lo("%b %d"),function(n){return 1!=n.getUTCDate()}],[lo("%a %d"),function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],[lo("%I %p"),function(n){return n.getUTCHours()}],[lo("%I:%M"),function(n){return n.getUTCMinutes()}],[lo(":%S"),function(n){return n.getUTCSeconds()}],[lo(".%L"),function(n){return n.getUTCMilliseconds()}]],Nl=go(Al);return kl.year=Wc.year.utc,Wc.scale.utc=function(){return fo(mo.scale.linear(),kl,Nl)},mo.text=vt(function(n){return n.responseText}),mo.json=function(n,t){return mt(n,"application/json",po,t)},mo.html=function(n,t){return mt(n,"text/html",vo,t)},mo.xml=vt(function(n){return n.responseXML}),mo}();
\ No newline at end of file
diff --git a/airflow/www_rbac/static/dagre-d3.js b/airflow/www_rbac/static/dagre-d3.js
deleted file mode 100644
index 2da7cdda08275..0000000000000
--- a/airflow/www_rbac/static/dagre-d3.js
+++ /dev/null
@@ -1,5007 +0,0 @@
-;(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
-var global=self;
-/**
- * @license
- * Copyright (c) 2012-2013 Chris Pettitt
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-global.dagreD3 = require('./index');
-
-},{"./index":2}],2:[function(require,module,exports){
-/**
- * @license
- * Copyright (c) 2012-2013 Chris Pettitt
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-module.exports =  {
-  Digraph: require('graphlib').Digraph,
-  Graph: require('graphlib').Graph,
-  Renderer: require('./lib/Renderer'),
-  json: require('graphlib').converter.json,
-  layout: require('dagre').layout,
-  version: require('./lib/version'),
-  debug: require('dagre').debug
-};
-
-},{"./lib/Renderer":3,"./lib/version":4,"dagre":11,"graphlib":29}],3:[function(require,module,exports){
-var layout = require('dagre').layout;
-
-var d3;
-try { d3 = require('d3'); } catch (_) { d3 = window.d3; }
-
-module.exports = Renderer;
-
-function Renderer() {
-  // Set up defaults...
-  this._layout = layout();
-
-  this.drawNodes(defaultDrawNodes);
-  this.drawEdgeLabels(defaultDrawEdgeLabels);
-  this.drawEdgePaths(defaultDrawEdgePaths);
-  this.positionNodes(defaultPositionNodes);
-  this.positionEdgeLabels(defaultPositionEdgeLabels);
-  this.positionEdgePaths(defaultPositionEdgePaths);
-  this.zoomSetup(defaultZoomSetup);
-  this.zoom(defaultZoom);
-  this.transition(defaultTransition);
-  this.postLayout(defaultPostLayout);
-  this.postRender(defaultPostRender);
-
-  this.edgeInterpolate('bundle');
-  this.edgeTension(0.95);
-  this.zoom_obj = null;
-}
-
-Renderer.prototype.layout = function(layout) {
-  if (!arguments.length) { return this._layout; }
-  this._layout = layout;
-  return this;
-};
-
-Renderer.prototype.drawNodes = function(drawNodes) {
-  if (!arguments.length) { return this._drawNodes; }
-  this._drawNodes = bind(drawNodes, this);
-  return this;
-};
-
-Renderer.prototype.drawEdgeLabels = function(drawEdgeLabels) {
-  if (!arguments.length) { return this._drawEdgeLabels; }
-  this._drawEdgeLabels = bind(drawEdgeLabels, this);
-  return this;
-};
-
-Renderer.prototype.drawEdgePaths = function(drawEdgePaths) {
-  if (!arguments.length) { return this._drawEdgePaths; }
-  this._drawEdgePaths = bind(drawEdgePaths, this);
-  return this;
-};
-
-Renderer.prototype.positionNodes = function(positionNodes) {
-  if (!arguments.length) { return this._positionNodes; }
-  this._positionNodes = bind(positionNodes, this);
-  return this;
-};
-
-Renderer.prototype.positionEdgeLabels = function(positionEdgeLabels) {
-  if (!arguments.length) { return this._positionEdgeLabels; }
-  this._positionEdgeLabels = bind(positionEdgeLabels, this);
-  return this;
-};
-
-Renderer.prototype.positionEdgePaths = function(positionEdgePaths) {
-  if (!arguments.length) { return this._positionEdgePaths; }
-  this._positionEdgePaths = bind(positionEdgePaths, this);
-  return this;
-};
-
-Renderer.prototype.transition = function(transition) {
-  if (!arguments.length) { return this._transition; }
-  this._transition = bind(transition, this);
-  return this;
-};
-
-Renderer.prototype.zoomSetup = function(zoomSetup) {
-  if (!arguments.length) { return this._zoomSetup; }
-  this._zoomSetup = bind(zoomSetup, this);
-  return this;
-};
-
-Renderer.prototype.zoom = function(zoom) {
-  if (!arguments.length) { return this._zoom; }
-  if (zoom) {
-    this._zoom = bind(zoom, this);
-  } else {
-    delete this._zoom;
-  }
-  return this;
-};
-
-Renderer.prototype.postLayout = function(postLayout) {
-  if (!arguments.length) { return this._postLayout; }
-  this._postLayout = bind(postLayout, this);
-  return this;
-};
-
-Renderer.prototype.postRender = function(postRender) {
-  if (!arguments.length) { return this._postRender; }
-  this._postRender = bind(postRender, this);
-  return this;
-};
-
-Renderer.prototype.edgeInterpolate = function(edgeInterpolate) {
-  if (!arguments.length) { return this._edgeInterpolate; }
-  this._edgeInterpolate = edgeInterpolate;
-  return this;
-};
-
-Renderer.prototype.edgeTension = function(edgeTension) {
-  if (!arguments.length) { return this._edgeTension; }
-  this._edgeTension = edgeTension;
-  return this;
-};
-
-Renderer.prototype.run = function(graph, orgSvg) {
-  // First copy the input graph so that it is not changed by the rendering
-  // process.
-  graph = copyAndInitGraph(graph);
-
-  // Create zoom elements
-  var svg = this._zoomSetup(graph, orgSvg);
-
-  // Create layers
-  svg
-    .selectAll('g.edgePaths, g.edgeLabels, g.nodes')
-    .data(['edgePaths', 'edgeLabels', 'nodes'])
-    .enter()
-      .append('g')
-      .attr('class', function(d) { return d; });
-
-  // Create node and edge roots, attach labels, and capture dimension
-  // information for use with layout.
-  var svgNodes = this._drawNodes(graph, svg.select('g.nodes'));
-  var svgEdgeLabels = this._drawEdgeLabels(graph, svg.select('g.edgeLabels'));
-
-  svgNodes.each(function(u) { calculateDimensions(this, graph.node(u)); });
-  svgEdgeLabels.each(function(e) { calculateDimensions(this, graph.edge(e)); });
-
-  // Now apply the layout function
-  var result = runLayout(graph, this._layout);
-
-  // Copy useDef attribute from input graph to output graph
-  graph.eachNode(function(u, a) {
-    if (a.useDef) {
-      result.node(u).useDef = a.useDef;
-    }
-  });
-
-  // Run any user-specified post layout processing
-  this._postLayout(result, svg);
-
-  var svgEdgePaths = this._drawEdgePaths(graph, svg.select('g.edgePaths'));
-
-  // Apply the layout information to the graph
-  this._positionNodes(result, svgNodes);
-  this._positionEdgeLabels(result, svgEdgeLabels);
-  this._positionEdgePaths(result, svgEdgePaths, orgSvg);
-
-  this._postRender(result, svg);
-
-  return result;
-};
-
-function copyAndInitGraph(graph) {
-  var copy = graph.copy();
-
-  if (copy.graph() === undefined) {
-    copy.graph({});
-  }
-
-  if (!('arrowheadFix' in copy.graph())) {
-    copy.graph().arrowheadFix = true;
-  }
-
-  // Init labels if they were not present in the source graph
-  copy.nodes().forEach(function(u) {
-    var value = copyObject(copy.node(u));
-    copy.node(u, value);
-    if (!('label' in value)) { value.label = ''; }
-  });
-
-  copy.edges().forEach(function(e) {
-    var value = copyObject(copy.edge(e));
-    copy.edge(e, value);
-    if (!('label' in value)) { value.label = ''; }
-  });
-
-  return copy;
-}
-
-function copyObject(obj) {
-  var copy = {};
-  for (var k in obj) {
-    copy[k] = obj[k];
-  }
-  return copy;
-}
-
-function calculateDimensions(group, value) {
-  var bbox = group.getBBox();
-  value.width = bbox.width;
-  value.height = bbox.height;
-}
-
-function runLayout(graph, layout) {
-  var result = layout.run(graph);
-
-  // Copy labels to the result graph
-  graph.eachNode(function(u, value) { result.node(u).label = value.label; });
-  graph.eachEdge(function(e, u, v, value) { result.edge(e).label = value.label; });
-
-  return result;
-}
-
-function defaultDrawNodes(g, root) {
-  var nodes = g.nodes().filter(function(u) { return !isComposite(g, u); });
-
-  var svgNodes = root
-    .selectAll('g.node')
-    .classed('enter', false)
-    .data(nodes, function(u) { return u; });
-
-  svgNodes.selectAll('*').remove();
-
-  svgNodes
-    .enter()
-      .append('g')
-        .style('opacity', 0)
-        .attr('class', 'node enter');
-
-  svgNodes.each(function(u) {
-    var attrs = g.node(u),
-        domNode = d3.select(this);
-    addLabel(attrs, domNode, true, 10, 10);
-  });
-
-  this._transition(svgNodes.exit())
-      .style('opacity', 0)
-      .remove();
-
-  return svgNodes;
-}
-
-function defaultDrawEdgeLabels(g, root) {
-  var svgEdgeLabels = root
-    .selectAll('g.edgeLabel')
-    .classed('enter', false)
-    .data(g.edges(), function (e) { return e; });
-
-  svgEdgeLabels.selectAll('*').remove();
-
-  svgEdgeLabels
-    .enter()
-      .append('g')
-        .style('opacity', 0)
-        .attr('class', 'edgeLabel enter');
-
-  svgEdgeLabels.each(function(e) { addLabel(g.edge(e), d3.select(this), false, 0, 0); });
-
-  this._transition(svgEdgeLabels.exit())
-      .style('opacity', 0)
-      .remove();
-
-  return svgEdgeLabels;
-}
-
-var defaultDrawEdgePaths = function(g, root) {
-  var svgEdgePaths = root
-    .selectAll('g.edgePath')
-    .classed('enter', false)
-    .data(g.edges(), function(e) { return e; });
-
-  var DEFAULT_ARROWHEAD = 'url(#arrowhead)',
-      createArrowhead = DEFAULT_ARROWHEAD;
-  if (!g.isDirected()) {
-    createArrowhead = null;
-  } else if (g.graph().arrowheadFix !== 'false' && g.graph().arrowheadFix !== false) {
-    createArrowhead = function() {
-      var strokeColor = d3.select(this).style('stroke');
-      if (strokeColor) {
-        var id = 'arrowhead-' + strokeColor.replace(/[^a-zA-Z0-9]/g, '_');
-        getOrMakeArrowhead(root, id).style('fill', strokeColor);
-        return 'url(#' + id + ')';
-      }
-      return DEFAULT_ARROWHEAD;
-    };
-  }
-
-  svgEdgePaths
-    .enter()
-      .append('g')
-        .attr('class', 'edgePath enter')
-        .append('path')
-          .style('opacity', 0);
-
-  svgEdgePaths
-    .selectAll('path')
-    .each(function(e) { applyStyle(g.edge(e).style, d3.select(this)); })
-    .attr('marker-end', createArrowhead);
-
-  this._transition(svgEdgePaths.exit())
-      .style('opacity', 0)
-      .remove();
-
-  return svgEdgePaths;
-};
-
-function defaultPositionNodes(g, svgNodes) {
-  function transform(u) {
-    var value = g.node(u);
-    return 'translate(' + value.x + ',' + value.y + ')';
-  }
-
-  // For entering nodes, position immediately without transition
-  svgNodes.filter('.enter').attr('transform', transform);
-
-  this._transition(svgNodes)
-      .style('opacity', 1)
-      .attr('transform', transform);
-}
-
-function defaultPositionEdgeLabels(g, svgEdgeLabels) {
-  function transform(e) {
-    var value = g.edge(e);
-    var point = findMidPoint(value.points);
-    return 'translate(' + point.x + ',' + point.y + ')';
-  }
-
-  // For entering edge labels, position immediately without transition
-  svgEdgeLabels.filter('.enter').attr('transform', transform);
-
-  this._transition(svgEdgeLabels)
-    .style('opacity', 1)
-    .attr('transform', transform);
-}
-
-function isEllipse(obj) {
-  return Object.prototype.toString.call(obj) === '[object SVGEllipseElement]';
-}
-
-function isCircle(obj) {
-  return Object.prototype.toString.call(obj) === '[object SVGCircleElement]';
-}
-
-function isPolygon(obj) {
-  return Object.prototype.toString.call(obj) === '[object SVGPolygonElement]';
-}
-
-function intersectNode(nd, p1, root) {
-  if (nd.useDef) {
-    var definedFig = root.select('defs #' + nd.useDef).node();
-    if (definedFig) {
-      var outerFig = definedFig.childNodes[0];
-      if (isCircle(outerFig) || isEllipse(outerFig)) {
-        return intersectEllipse(nd, outerFig, p1);
-      } else if (isPolygon(outerFig)) {
-        return intersectPolygon(nd, outerFig, p1);
-      }
-    }
-  }
-  // TODO: use bpodgursky's shortening algorithm here
-  return intersectRect(nd, p1);
-}
-
-function defaultPositionEdgePaths(g, svgEdgePaths, root) {
-  var interpolate = this._edgeInterpolate,
-      tension = this._edgeTension;
-
-  function calcPoints(e) {
-    var value = g.edge(e);
-    var source = g.node(g.incidentNodes(e)[0]);
-    var target = g.node(g.incidentNodes(e)[1]);
-    var points = value.points.slice();
-
-    var p0 = points.length === 0 ? target : points[0];
-    var p1 = points.length === 0 ? source : points[points.length - 1];
-
-    points.unshift(intersectNode(source, p0, root));
-    points.push(intersectNode(target, p1, root));
-
-    return d3.svg.line()
-      .x(function(d) { return d.x; })
-      .y(function(d) { return d.y; })
-      .interpolate(interpolate)
-      .tension(tension)
-      (points);
-  }
-
-  svgEdgePaths.filter('.enter').selectAll('path')
-      .attr('d', calcPoints);
-
-  this._transition(svgEdgePaths.selectAll('path'))
-      .attr('d', calcPoints)
-      .style('opacity', 1);
-}
-
-// By default we do not use transitions
-function defaultTransition(selection) {
-  return selection;
-}
-
-// Setup dom for zooming
-function defaultZoomSetup(graph, svg) {
-  var root = svg.property('ownerSVGElement');
-  // If the svg node is the root, we get null, so set to svg.
-  if (!root) {
-    root = svg;
-  } else {
-    root = d3.select(root);
-  }
-
-  if (root.select('rect.overlay').empty()) {
-    // Create an overlay for capturing mouse events that don't touch foreground
-    root.insert('rect', ':first-child')
-      .attr('class', 'overlay')
-      .attr('width', '100%')
-      .attr('height', '100%')
-      .style('fill', 'none')
-      .style('pointer-events', 'all');
-
-    // Capture the zoom behaviour from the svg
-    svg = svg.append('g')
-      .attr('class', 'zoom');
-
-    if (this._zoom) {
-      root.call(this._zoom(graph, svg));
-    }
-  }
-
-  return svg;
-}
-
-// By default allow pan and zoom
-function defaultZoom(graph, svg) {
-
-  this.zoom_obj = d3.behavior.zoom().on('zoom', function() {
-    svg.attr('transform', 'translate(' + d3.event.translate + ')scale(' + d3.event.scale + ')');
-  });
-  return this.zoom_obj;
-}
-
-function defaultPostLayout() {
-  // Do nothing
-}
-
-function defaultPostRender(graph, root) {
-  if (graph.isDirected()) {
-    // Fill = #333 is for backwards compatibility
-    getOrMakeArrowhead(root, 'arrowhead')
-      .attr('fill', '#333');
-  }
-}
-
-function getOrMakeArrowhead(root, id) {
-  var search = root.select('#' + id);
-  if (!search.empty()) { return search; }
-
-  var defs = root.select('defs');
-  if (defs.empty()) {
-    defs = root.append('svg:defs');
-  }
-
-  var marker =
-    defs
-      .append('svg:marker')
-        .attr('id', id)
-        .attr('viewBox', '0 0 10 10')
-        .attr('refX', 8)
-        .attr('refY', 5)
-        .attr('markerUnits', 'strokeWidth')
-        .attr('markerWidth', 8)
-        .attr('markerHeight', 5)
-        .attr('orient', 'auto');
-
-  marker
-    .append('svg:path')
-      .attr('d', 'M 0 0 L 10 5 L 0 10 z');
-
-  return marker;
-}
-
-function addLabel(node, root, addingNode, marginX, marginY) {
-  // If the node has 'useDef' meta data, we rely on that
-  if (node.useDef) {
-    root.append('use').attr('xlink:href', '#' + node.useDef);
-    return;
-  }
-  // Add the rect first so that it appears behind the label
-  var label = node.label;
-  var rect = root.append('rect');
-  if (node.width) {
-    rect.attr('width', node.width);
-  }
-  if (node.height) {
-    rect.attr('height', node.height);
-  }
-
-  var labelSvg = root.append('g'),
-      innerLabelSvg;
-
-  if (label[0] === '<') {
-    addForeignObjectLabel(label, labelSvg);
-    // No margin for HTML elements
-    marginX = marginY = 0;
-  } else {
-    innerLabelSvg = addTextLabel(label,
-                                 labelSvg,
-                                 Math.floor(node.labelCols),
-                                 node.labelCut);
-    applyStyle(node.labelStyle, innerLabelSvg);
-  }
-
-  var labelBBox = labelSvg.node().getBBox();
-  labelSvg.attr('transform',
-                'translate(' + (-labelBBox.width / 2) + ',' + (-labelBBox.height / 2) + ')');
-
-  var bbox = root.node().getBBox();
-
-  rect
-    .attr('rx', node.rx ? node.rx : 5)
-    .attr('ry', node.ry ? node.ry : 5)
-    .attr('x', -(bbox.width / 2 + marginX))
-    .attr('y', -(bbox.height / 2 + marginY))
-    .attr('width', bbox.width + 2 * marginX)
-    .attr('height', bbox.height + 2 * marginY)
-    .attr('fill', '#fff');
-
-  if (addingNode) {
-    applyStyle(node.style, rect);
-
-    if (node.fill) {
-      rect.style('fill', node.fill);
-    }
-
-    if (node.stroke) {
-      rect.style('stroke', node.stroke);
-    }
-
-    if (node['stroke-width']) {
-      rect.style('stroke-width', node['stroke-width'] + 'px');
-    }
-
-    if (node['stroke-dasharray']) {
-      rect.style('stroke-dasharray', node['stroke-dasharray']);
-    }
-
-    if (node.href) {
-      root
-        .attr('class', root.attr('class') + ' clickable')
-        .on('click', function() {
-          window.open(node.href);
-        });
-    }
-  }
-}
-
-function addForeignObjectLabel(label, root) {
-  var fo = root
-    .append('foreignObject')
-      .attr('width', '100000');
-
-  var w, h;
-  fo
-    .append('xhtml:div')
-      .style('float', 'left')
-      // TODO find a better way to get dimensions for foreignObjects...
-      .html(function() { return label; })
-      .each(function() {
-        w = this.clientWidth;
-        h = this.clientHeight;
-      });
-
-  fo
-    .attr('width', w)
-    .attr('height', h);
-}
-
-function addTextLabel(label, root, labelCols, labelCut) {
-  if (labelCut === undefined) { labelCut = 'false'; }
-  labelCut = (labelCut.toString().toLowerCase() === 'true');
-
-  var node = root
-    .append('text')
-    .attr('text-anchor', 'left');
-
-  label = label.replace(/\\n/g, '\n');
-
-  var arr = labelCols ? wordwrap(label, labelCols, labelCut) : label;
-  arr = arr.split('\n');
-  for (var i = 0; i < arr.length; i++) {
-    node
-      .append('tspan')
-        .attr('dy', '1em')
-        .attr('x', '1')
-        .text(arr[i]);
-  }
-
-  return node;
-}
-
-// Thanks to
-// http://james.padolsey.com/javascript/wordwrap-for-javascript/
-function wordwrap (str, width, cut, brk) {
-  brk = brk || '\n';
-  width = width || 75;
-  cut = cut || false;
-
-  if (!str) { return str; }
-
-  var regex = '.{1,' + width + '}(\\s|$)' + (cut ? '|.{' + width + '}|.+$' : '|\\S+?(\\s|$)');
-
-  return str.match(new RegExp(regex, 'g')).join(brk);
-}
-
-function findMidPoint(points) {
-  var midIdx = points.length / 2;
-  if (points.length % 2) {
-    return points[Math.floor(midIdx)];
-  } else {
-    var p0 = points[midIdx - 1];
-    var p1 = points[midIdx];
-    return {x: (p0.x + p1.x) / 2, y: (p0.y + p1.y) / 2};
-  }
-}
-
-function intersectRect(rect, point) {
-  var x = rect.x;
-  var y = rect.y;
-
-  // Rectangle intersection algorithm from:
-  // http://math.stackexchange.com/questions/108113/find-edge-between-two-boxes
-  var dx = point.x - x;
-  var dy = point.y - y;
-  var w = rect.width / 2;
-  var h = rect.height / 2;
-
-  var sx, sy;
-  if (Math.abs(dy) * w > Math.abs(dx) * h) {
-    // Intersection is top or bottom of rect.
-    if (dy < 0) {
-      h = -h;
-    }
-    sx = dy === 0 ? 0 : h * dx / dy;
-    sy = h;
-  } else {
-    // Intersection is left or right of rect.
-    if (dx < 0) {
-      w = -w;
-    }
-    sx = w;
-    sy = dx === 0 ? 0 : w * dy / dx;
-  }
-
-  return {x: x + sx, y: y + sy};
-}
-
-function intersectEllipse(node, ellipseOrCircle, point) {
-  // Formulae from: http://mathworld.wolfram.com/Ellipse-LineIntersection.html
-
-  var cx = node.x;
-  var cy = node.y;
-  var rx, ry;
-
-  if (isCircle(ellipseOrCircle)) {
-    rx = ry = ellipseOrCircle.r.baseVal.value;
-  } else {
-    rx = ellipseOrCircle.rx.baseVal.value;
-    ry = ellipseOrCircle.ry.baseVal.value;
-  }
-
-  var px = cx - point.x;
-  var py = cy - point.y;
-
-  var det = Math.sqrt(rx * rx * py * py + ry * ry * px * px);
-
-  var dx = Math.abs(rx * ry * px / det);
-  if (point.x < cx) {
-    dx = -dx;
-  }
-  var dy = Math.abs(rx * ry * py / det);
-  if (point.y < cy) {
-    dy = -dy;
-  }
-
-  return {x: cx + dx, y: cy + dy};
-}
-
-function sameSign(r1, r2) {
-  return r1 * r2 > 0;
-}
-
-// Add point to the found intersections, but check first that it is unique.
-function addPoint(x, y, intersections) {
-  if (!intersections.some(function (elm) { return elm[0] === x && elm[1] === y; })) {
-    intersections.push([x, y]);
-  }
-}
-
-function intersectLine(x1, y1, x2, y2, x3, y3, x4, y4, intersections) {
-  // Algorithm from J. Avro, (ed.) Graphics Gems, No 2, Morgan Kaufmann, 1994, p7 and p473.
-
-  var a1, a2, b1, b2, c1, c2;
-  var r1, r2 , r3, r4;
-  var denom, offset, num;
-  var x, y;
-
-  // Compute a1, b1, c1, where line joining points 1 and 2 is F(x,y) = a1 x + b1 y + c1 = 0.
-  a1 = y2 - y1;
-  b1 = x1 - x2;
-  c1 = (x2 * y1) - (x1 * y2);
-
-  // Compute r3 and r4.
-  r3 = ((a1 * x3) + (b1 * y3) + c1);
-  r4 = ((a1 * x4) + (b1 * y4) + c1);
-
-  // Check signs of r3 and r4. If both point 3 and point 4 lie on
-  // same side of line 1, the line segments do not intersect.
-  if ((r3 !== 0) && (r4 !== 0) && sameSign(r3, r4)) {
-    return /*DONT_INTERSECT*/;
-  }
-
-  // Compute a2, b2, c2 where line joining points 3 and 4 is G(x,y) = a2 x + b2 y + c2 = 0
-  a2 = y4 - y3;
-  b2 = x3 - x4;
-  c2 = (x4 * y3) - (x3 * y4);
-
-  // Compute r1 and r2
-  r1 = (a2 * x1) + (b2 * y1) + c2;
-  r2 = (a2 * x2) + (b2 * y2) + c2;
-
-  // Check signs of r1 and r2. If both point 1 and point 2 lie
-  // on same side of second line segment, the line segments do
-  // not intersect.
-  if ((r1 !== 0) && (r2 !== 0) && (sameSign(r1, r2))) {
-    return /*DONT_INTERSECT*/;
-  }
-
-  // Line segments intersect: compute intersection point.
-  denom = (a1 * b2) - (a2 * b1);
-  if (denom === 0) {
-    return /*COLLINEAR*/;
-  }
-
-  offset = Math.abs(denom / 2);
-
-  // The denom/2 is to get rounding instead of truncating. It
-  // is added or subtracted to the numerator, depending upon the
-  // sign of the numerator.
-  num = (b1 * c2) - (b2 * c1);
-  x = (num < 0) ? ((num - offset) / denom) : ((num + offset) / denom);
-
-  num = (a2 * c1) - (a1 * c2);
-  y = (num < 0) ? ((num - offset) / denom) : ((num + offset) / denom);
-
-  // lines_intersect
-  addPoint(x, y, intersections);
-}
-
-function intersectPolygon(node, polygon, point) {
-  var x1 = node.x;
-  var y1 = node.y;
-  var x2 = point.x;
-  var y2 = point.y;
-
-  var intersections = [];
-  var points = polygon.points;
-
-  var minx = 100000, miny = 100000;
-  for (var j = 0; j < points.numberOfItems; j++) {
-    var p = points.getItem(j);
-    minx = Math.min(minx, p.x);
-    miny = Math.min(miny, p.y);
-  }
-
-  var left = x1 - node.width / 2 - minx;
-  var top =  y1 - node.height / 2 - miny;
-
-  for (var i = 0; i < points.numberOfItems; i++) {
-    var p1 = points.getItem(i);
-    var p2 = points.getItem(i < points.numberOfItems - 1 ? i + 1 : 0);
-    intersectLine(x1, y1, x2, y2, left + p1.x, top + p1.y, left + p2.x, top + p2.y, intersections);
-  }
-
-  if (intersections.length === 1) {
-    return {x: intersections[0][0], y: intersections[0][1]};
-  }
-
-  if (intersections.length > 1) {
-    // More intersections, find the one nearest to edge end point
-    intersections.sort(function(p, q) {
-      var pdx = p[0] - point.x,
-         pdy = p[1] - point.y,
-         distp = Math.sqrt(pdx * pdx + pdy * pdy),
-
-         qdx = q[0] - point.x,
-         qdy = q[1] - point.y,
-         distq = Math.sqrt(qdx * qdx + qdy * qdy);
-
-      return (distp < distq) ? -1 : (distp === distq ? 0 : 1);
-    });
-    return {x: intersections[0][0], y: intersections[0][1]};
-  } else {
-    console.log('NO INTERSECTION FOUND, RETURN NODE CENTER', node);
-    return node;
-  }
-}
-
-function isComposite(g, u) {
-  return 'children' in g && g.children(u).length;
-}
-
-function bind(func, thisArg) {
-  // For some reason PhantomJS occassionally fails when using the builtin bind,
-  // so we check if it is available and if not, use a degenerate polyfill.
-  if (func.bind) {
-    return func.bind(thisArg);
-  }
-
-  return function() {
-    return func.apply(thisArg, arguments);
-  };
-}
-
-function applyStyle(style, domNode) {
-  if (style) {
-    var currStyle = domNode.attr('style') || '';
-    domNode.attr('style', currStyle + '; ' + style);
-  }
-}
-
-},{"d3":10,"dagre":11}],4:[function(require,module,exports){
-module.exports = '0.2.9';
-
-},{}],5:[function(require,module,exports){
-exports.Set = require('./lib/Set');
-exports.PriorityQueue = require('./lib/PriorityQueue');
-exports.version = require('./lib/version');
-
-},{"./lib/PriorityQueue":6,"./lib/Set":7,"./lib/version":9}],6:[function(require,module,exports){
-module.exports = PriorityQueue;
-
-/**
- * A min-priority queue data structure. This algorithm is derived from Cormen,
- * et al., "Introduction to Algorithms". The basic idea of a min-priority
- * queue is that you can efficiently (in O(1) time) get the smallest key in
- * the queue. Adding and removing elements takes O(log n) time. A key can
- * have its priority decreased in O(log n) time.
- */
-function PriorityQueue() {
-  this._arr = [];
-  this._keyIndices = {};
-}
-
-/**
- * Returns the number of elements in the queue. Takes `O(1)` time.
- */
-PriorityQueue.prototype.size = function() {
-  return this._arr.length;
-};
-
-/**
- * Returns the keys that are in the queue. Takes `O(n)` time.
- */
-PriorityQueue.prototype.keys = function() {
-  return this._arr.map(function(x) { return x.key; });
-};
-
-/**
- * Returns `true` if **key** is in the queue and `false` if not.
- */
-PriorityQueue.prototype.has = function(key) {
-  return key in this._keyIndices;
-};
-
-/**
- * Returns the priority for **key**. If **key** is not present in the queue
- * then this function returns `undefined`. Takes `O(1)` time.
- *
- * @param {Object} key
- */
-PriorityQueue.prototype.priority = function(key) {
-  var index = this._keyIndices[key];
-  if (index !== undefined) {
-    return this._arr[index].priority;
-  }
-};
-
-/**
- * Returns the key for the minimum element in this queue. If the queue is
- * empty this function throws an Error. Takes `O(1)` time.
- */
-PriorityQueue.prototype.min = function() {
-  if (this.size() === 0) {
-    throw new Error("Queue underflow");
-  }
-  return this._arr[0].key;
-};
-
-/**
- * Inserts a new key into the priority queue. If the key already exists in
- * the queue this function returns `false`; otherwise it will return `true`.
- * Takes `O(n)` time.
- *
- * @param {Object} key the key to add
- * @param {Number} priority the initial priority for the key
- */
-PriorityQueue.prototype.add = function(key, priority) {
-  var keyIndices = this._keyIndices;
-  if (!(key in keyIndices)) {
-    var arr = this._arr;
-    var index = arr.length;
-    keyIndices[key] = index;
-    arr.push({key: key, priority: priority});
-    this._decrease(index);
-    return true;
-  }
-  return false;
-};
-
-/**
- * Removes and returns the smallest key in the queue. Takes `O(log n)` time.
- */
-PriorityQueue.prototype.removeMin = function() {
-  this._swap(0, this._arr.length - 1);
-  var min = this._arr.pop();
-  delete this._keyIndices[min.key];
-  this._heapify(0);
-  return min.key;
-};
-
-/**
- * Decreases the priority for **key** to **priority**. If the new priority is
- * greater than the previous priority, this function will throw an Error.
- *
- * @param {Object} key the key for which to raise priority
- * @param {Number} priority the new priority for the key
- */
-PriorityQueue.prototype.decrease = function(key, priority) {
-  var index = this._keyIndices[key];
-  if (priority > this._arr[index].priority) {
-    throw new Error("New priority is greater than current priority. " +
-        "Key: " + key + " Old: " + this._arr[index].priority + " New: " + priority);
-  }
-  this._arr[index].priority = priority;
-  this._decrease(index);
-};
-
-PriorityQueue.prototype._heapify = function(i) {
-  var arr = this._arr;
-  var l = 2 * i,
-      r = l + 1,
-      largest = i;
-  if (l < arr.length) {
-    largest = arr[l].priority < arr[largest].priority ? l : largest;
-    if (r < arr.length) {
-      largest = arr[r].priority < arr[largest].priority ? r : largest;
-    }
-    if (largest !== i) {
-      this._swap(i, largest);
-      this._heapify(largest);
-    }
-  }
-};
-
-PriorityQueue.prototype._decrease = function(index) {
-  var arr = this._arr;
-  var priority = arr[index].priority;
-  var parent;
-  while (index !== 0) {
-    parent = index >> 1;
-    if (arr[parent].priority < priority) {
-      break;
-    }
-    this._swap(index, parent);
-    index = parent;
-  }
-};
-
-PriorityQueue.prototype._swap = function(i, j) {
-  var arr = this._arr;
-  var keyIndices = this._keyIndices;
-  var origArrI = arr[i];
-  var origArrJ = arr[j];
-  arr[i] = origArrJ;
-  arr[j] = origArrI;
-  keyIndices[origArrJ.key] = i;
-  keyIndices[origArrI.key] = j;
-};
-
-},{}],7:[function(require,module,exports){
-var util = require('./util');
-
-module.exports = Set;
-
-/**
- * Constructs a new Set with an optional set of `initialKeys`.
- *
- * It is important to note that keys are coerced to String for most purposes
- * with this object, similar to the behavior of JavaScript's Object. For
- * example, the following will add only one key:
- *
- *     var s = new Set();
- *     s.add(1);
- *     s.add("1");
- *
- * However, the type of the key is preserved internally so that `keys` returns
- * the original key set uncoerced. For the above example, `keys` would return
- * `[1]`.
- */
-function Set(initialKeys) {
-  this._size = 0;
-  this._keys = {};
-
-  if (initialKeys) {
-    for (var i = 0, il = initialKeys.length; i < il; ++i) {
-      this.add(initialKeys[i]);
-    }
-  }
-}
-
-/**
- * Returns a new Set that represents the set intersection of the array of given
- * sets.
- */
-Set.intersect = function(sets) {
-  if (sets.length === 0) {
-    return new Set();
-  }
-
-  var result = new Set(!util.isArray(sets[0]) ? sets[0].keys() : sets[0]);
-  for (var i = 1, il = sets.length; i < il; ++i) {
-    var resultKeys = result.keys(),
-        other = !util.isArray(sets[i]) ? sets[i] : new Set(sets[i]);
-    for (var j = 0, jl = resultKeys.length; j < jl; ++j) {
-      var key = resultKeys[j];
-      if (!other.has(key)) {
-        result.remove(key);
-      }
-    }
-  }
-
-  return result;
-};
-
-/**
- * Returns a new Set that represents the set union of the array of given sets.
- */
-Set.union = function(sets) {
-  var totalElems = util.reduce(sets, function(lhs, rhs) {
-    return lhs + (rhs.size ? rhs.size() : rhs.length);
-  }, 0);
-  var arr = new Array(totalElems);
-
-  var k = 0;
-  for (var i = 0, il = sets.length; i < il; ++i) {
-    var cur = sets[i],
-        keys = !util.isArray(cur) ? cur.keys() : cur;
-    for (var j = 0, jl = keys.length; j < jl; ++j) {
-      arr[k++] = keys[j];
-    }
-  }
-
-  return new Set(arr);
-};
-
-/**
- * Returns the size of this set in `O(1)` time.
- */
-Set.prototype.size = function() {
-  return this._size;
-};
-
-/**
- * Returns the keys in this set. Takes `O(n)` time.
- */
-Set.prototype.keys = function() {
-  return values(this._keys);
-};
-
-/**
- * Tests if a key is present in this Set. Returns `true` if it is and `false`
- * if not. Takes `O(1)` time.
- */
-Set.prototype.has = function(key) {
-  return key in this._keys;
-};
-
-/**
- * Adds a new key to this Set if it is not already present. Returns `true` if
- * the key was added and `false` if it was already present. Takes `O(1)` time.
- */
-Set.prototype.add = function(key) {
-  if (!(key in this._keys)) {
-    this._keys[key] = key;
-    ++this._size;
-    return true;
-  }
-  return false;
-};
-
-/**
- * Removes a key from this Set. If the key was removed this function returns
- * `true`. If not, it returns `false`. Takes `O(1)` time.
- */
-Set.prototype.remove = function(key) {
-  if (key in this._keys) {
-    delete this._keys[key];
-    --this._size;
-    return true;
-  }
-  return false;
-};
-
-/*
- * Returns an array of all values for properties of **o**.
- */
-function values(o) {
-  var ks = Object.keys(o),
-      len = ks.length,
-      result = new Array(len),
-      i;
-  for (i = 0; i < len; ++i) {
-    result[i] = o[ks[i]];
-  }
-  return result;
-}
-
-},{"./util":8}],8:[function(require,module,exports){
-/*
- * This polyfill comes from
- * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray
- */
-if(!Array.isArray) {
-  exports.isArray = function (vArg) {
-    return Object.prototype.toString.call(vArg) === '[object Array]';
-  };
-} else {
-  exports.isArray = Array.isArray;
-}
-
-/*
- * Slightly adapted polyfill from
- * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/Reduce
- */
-if ('function' !== typeof Array.prototype.reduce) {
-  exports.reduce = function(array, callback, opt_initialValue) {
-    'use strict';
-    if (null === array || 'undefined' === typeof array) {
-      // At the moment all modern browsers, that support strict mode, have
-      // native implementation of Array.prototype.reduce. For instance, IE8
-      // does not support strict mode, so this check is actually useless.
-      throw new TypeError(
-          'Array.prototype.reduce called on null or undefined');
-    }
-    if ('function' !== typeof callback) {
-      throw new TypeError(callback + ' is not a function');
-    }
-    var index, value,
-        length = array.length >>> 0,
-        isValueSet = false;
-    if (1 < arguments.length) {
-      value = opt_initialValue;
-      isValueSet = true;
-    }
-    for (index = 0; length > index; ++index) {
-      if (array.hasOwnProperty(index)) {
-        if (isValueSet) {
-          value = callback(value, array[index], index, array);
-        }
-        else {
-          value = array[index];
-          isValueSet = true;
-        }
-      }
-    }
-    if (!isValueSet) {
-      throw new TypeError('Reduce of empty array with no initial value');
-    }
-    return value;
-  };
-} else {
-  exports.reduce = function(array, callback, opt_initialValue) {
-    return array.reduce(callback, opt_initialValue);
-  };
-}
-
-},{}],9:[function(require,module,exports){
-module.exports = '1.1.3';
-
-},{}],10:[function(require,module,exports){
-require("./d3");
-module.exports = d3;
-(function () { delete this.d3; })(); // unset global
-
-},{}],11:[function(require,module,exports){
-/*
-Copyright (c) 2012-2013 Chris Pettitt
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-exports.Digraph = require("graphlib").Digraph;
-exports.Graph = require("graphlib").Graph;
-exports.layout = require("./lib/layout");
-exports.version = require("./lib/version");
-exports.debug = require("./lib/debug");
-
-},{"./lib/debug":12,"./lib/layout":13,"./lib/version":28,"graphlib":29}],12:[function(require,module,exports){
-'use strict';
-
-var util = require('./util');
-
-/**
- * Renders a graph in a stringified DOT format that indicates the ordering of
- * nodes by layer. Circles represent normal nodes. Diamons represent dummy
- * nodes. While we try to put nodes in clusters, it appears that graphviz
- * does not respect this because we're later using subgraphs for ordering nodes
- * in each layer.
- */
-exports.dotOrdering = function(g) {
-  var ordering = util.ordering(g.filterNodes(util.filterNonSubgraphs(g)));
-  var result = 'digraph {';
-
-  function dfs(u) {
-    var children = g.children(u);
-    if (children.length) {
-      result += 'subgraph cluster_' + u + ' {';
-      result += 'label="' + u + '";';
-      children.forEach(function(v) {
-        dfs(v);
-      });
-      result += '}';
-    } else {
-      result += u;
-      if (g.node(u).dummy) {
-        result += ' [shape=diamond]';
-      }
-      result += ';';
-    }
-  }
-
-  g.children(null).forEach(dfs);
-
-  ordering.forEach(function(layer) {
-    result += 'subgraph { rank=same; edge [style="invis"];';
-    result += layer.join('->');
-    result += '}';
-  });
-
-  g.eachEdge(function(e, u, v) {
-    result += u + '->' + v + ';';
-  });
-
-  result += '}';
-
-  return result;
-};
-
-},{"./util":27}],13:[function(require,module,exports){
-'use strict';
-
-var util = require('./util'),
-    rank = require('./rank'),
-    order = require('./order'),
-    CGraph = require('graphlib').CGraph,
-    CDigraph = require('graphlib').CDigraph;
-
-module.exports = function() {
-  // External configuration
-  var config = {
-    // How much debug information to include?
-    debugLevel: 0,
-    // Max number of sweeps to perform in order phase
-    orderMaxSweeps: order.DEFAULT_MAX_SWEEPS,
-    // Use network simplex algorithm in ranking
-    rankSimplex: false,
-    // Rank direction. Valid values are (TB, LR)
-    rankDir: 'TB'
-  };
-
-  // Phase functions
-  var position = require('./position')();
-
-  // This layout object
-  var self = {};
-
-  self.orderIters = util.propertyAccessor(self, config, 'orderMaxSweeps');
-
-  self.rankSimplex = util.propertyAccessor(self, config, 'rankSimplex');
-
-  self.nodeSep = delegateProperty(position.nodeSep);
-  self.edgeSep = delegateProperty(position.edgeSep);
-  self.universalSep = delegateProperty(position.universalSep);
-  self.rankSep = delegateProperty(position.rankSep);
-  self.rankDir = util.propertyAccessor(self, config, 'rankDir');
-  self.debugAlignment = delegateProperty(position.debugAlignment);
-
-  self.debugLevel = util.propertyAccessor(self, config, 'debugLevel', function(x) {
-    util.log.level = x;
-    position.debugLevel(x);
-  });
-
-  self.run = util.time('Total layout', run);
-
-  self._normalize = normalize;
-
-  return self;
-
-  /*
-   * Constructs an adjacency graph using the nodes and edges specified through
-   * config. For each node and edge we add a property `dagre` that contains an
-   * object that will hold intermediate and final layout information. Some of
-   * the contents include:
-   *
-   *  1) A generated ID that uniquely identifies the object.
-   *  2) Dimension information for nodes (copied from the source node).
-   *  3) Optional dimension information for edges.
-   *
-   * After the adjacency graph is constructed the code no longer needs to use
-   * the original nodes and edges passed in via config.
-   */
-  function initLayoutGraph(inputGraph) {
-    var g = new CDigraph();
-
-    inputGraph.eachNode(function(u, value) {
-      if (value === undefined) value = {};
-      g.addNode(u, {
-        width: value.width,
-        height: value.height
-      });
-      if (value.hasOwnProperty('rank')) {
-        g.node(u).prefRank = value.rank;
-      }
-    });
-
-    // Set up subgraphs
-    if (inputGraph.parent) {
-      inputGraph.nodes().forEach(function(u) {
-        g.parent(u, inputGraph.parent(u));
-      });
-    }
-
-    inputGraph.eachEdge(function(e, u, v, value) {
-      if (value === undefined) value = {};
-      var newValue = {
-        e: e,
-        minLen: value.minLen || 1,
-        width: value.width || 0,
-        height: value.height || 0,
-        points: []
-      };
-
-      g.addEdge(null, u, v, newValue);
-    });
-
-    // Initial graph attributes
-    var graphValue = inputGraph.graph() || {};
-    g.graph({
-      rankDir: graphValue.rankDir || config.rankDir,
-      orderRestarts: graphValue.orderRestarts
-    });
-
-    return g;
-  }
-
-  function run(inputGraph) {
-    var rankSep = self.rankSep();
-    var g;
-    try {
-      // Build internal graph
-      g = util.time('initLayoutGraph', initLayoutGraph)(inputGraph);
-
-      if (g.order() === 0) {
-        return g;
-      }
-
-      // Make space for edge labels
-      g.eachEdge(function(e, s, t, a) {
-        a.minLen *= 2;
-      });
-      self.rankSep(rankSep / 2);
-
-      // Determine the rank for each node. Nodes with a lower rank will appear
-      // above nodes of higher rank.
-      util.time('rank.run', rank.run)(g, config.rankSimplex);
-
-      // Normalize the graph by ensuring that every edge is proper (each edge has
-      // a length of 1). We achieve this by adding dummy nodes to long edges,
-      // thus shortening them.
-      util.time('normalize', normalize)(g);
-
-      // Order the nodes so that edge crossings are minimized.
-      util.time('order', order)(g, config.orderMaxSweeps);
-
-      // Find the x and y coordinates for every node in the graph.
-      util.time('position', position.run)(g);
-
-      // De-normalize the graph by removing dummy nodes and augmenting the
-      // original long edges with coordinate information.
-      util.time('undoNormalize', undoNormalize)(g);
-
-      // Reverses points for edges that are in a reversed state.
-      util.time('fixupEdgePoints', fixupEdgePoints)(g);
-
-      // Restore delete edges and reverse edges that were reversed in the rank
-      // phase.
-      util.time('rank.restoreEdges', rank.restoreEdges)(g);
-
-      // Construct final result graph and return it
-      return util.time('createFinalGraph', createFinalGraph)(g, inputGraph.isDirected());
-    } finally {
-      self.rankSep(rankSep);
-    }
-  }
-
-  /*
-   * This function is responsible for 'normalizing' the graph. The process of
-   * normalization ensures that no edge in the graph has spans more than one
-   * rank. To do this it inserts dummy nodes as needed and links them by adding
-   * dummy edges. This function keeps enough information in the dummy nodes and
-   * edges to ensure that the original graph can be reconstructed later.
-   *
-   * This method assumes that the input graph is cycle free.
-   */
-  function normalize(g) {
-    var dummyCount = 0;
-    g.eachEdge(function(e, s, t, a) {
-      var sourceRank = g.node(s).rank;
-      var targetRank = g.node(t).rank;
-      if (sourceRank + 1 < targetRank) {
-        for (var u = s, rank = sourceRank + 1, i = 0; rank < targetRank; ++rank, ++i) {
-          var v = '_D' + (++dummyCount);
-          var node = {
-            width: a.width,
-            height: a.height,
-            edge: { id: e, source: s, target: t, attrs: a },
-            rank: rank,
-            dummy: true
-          };
-
-          // If this node represents a bend then we will use it as a control
-          // point. For edges with 2 segments this will be the center dummy
-          // node. For edges with more than two segments, this will be the
-          // first and last dummy node.
-          if (i === 0) node.index = 0;
-          else if (rank + 1 === targetRank) node.index = 1;
-
-          g.addNode(v, node);
-          g.addEdge(null, u, v, {});
-          u = v;
-        }
-        g.addEdge(null, u, t, {});
-        g.delEdge(e);
-      }
-    });
-  }
-
-  /*
-   * Reconstructs the graph as it was before normalization. The positions of
-   * dummy nodes are used to build an array of points for the original 'long'
-   * edge. Dummy nodes and edges are removed.
-   */
-  function undoNormalize(g) {
-    g.eachNode(function(u, a) {
-      if (a.dummy) {
-        if ('index' in a) {
-          var edge = a.edge;
-          if (!g.hasEdge(edge.id)) {
-            g.addEdge(edge.id, edge.source, edge.target, edge.attrs);
-          }
-          var points = g.edge(edge.id).points;
-          points[a.index] = { x: a.x, y: a.y, ul: a.ul, ur: a.ur, dl: a.dl, dr: a.dr };
-        }
-        g.delNode(u);
-      }
-    });
-  }
-
-  /*
-   * For each edge that was reversed during the `acyclic` step, reverse its
-   * array of points.
-   */
-  function fixupEdgePoints(g) {
-    g.eachEdge(function(e, s, t, a) { if (a.reversed) a.points.reverse(); });
-  }
-
-  function createFinalGraph(g, isDirected) {
-    var out = isDirected ? new CDigraph() : new CGraph();
-    out.graph(g.graph());
-    g.eachNode(function(u, value) { out.addNode(u, value); });
-    g.eachNode(function(u) { out.parent(u, g.parent(u)); });
-    g.eachEdge(function(e, u, v, value) {
-      out.addEdge(value.e, u, v, value);
-    });
-
-    // Attach bounding box information
-    var maxX = 0, maxY = 0;
-    g.eachNode(function(u, value) {
-      if (!g.children(u).length) {
-        maxX = Math.max(maxX, value.x + value.width / 2);
-        maxY = Math.max(maxY, value.y + value.height / 2);
-      }
-    });
-    g.eachEdge(function(e, u, v, value) {
-      var maxXPoints = Math.max.apply(Math, value.points.map(function(p) { return p.x; }));
-      var maxYPoints = Math.max.apply(Math, value.points.map(function(p) { return p.y; }));
-      maxX = Math.max(maxX, maxXPoints + value.width / 2);
-      maxY = Math.max(maxY, maxYPoints + value.height / 2);
-    });
-    out.graph().width = maxX;
-    out.graph().height = maxY;
-
-    return out;
-  }
-
-  /*
-   * Given a function, a new function is returned that invokes the given
-   * function. The return value from the function is always the `self` object.
-   */
-  function delegateProperty(f) {
-    return function() {
-      if (!arguments.length) return f();
-      f.apply(null, arguments);
-      return self;
-    };
-  }
-};
-
-
-},{"./order":14,"./position":19,"./rank":20,"./util":27,"graphlib":29}],14:[function(require,module,exports){
-'use strict';
-
-var util = require('./util'),
-    crossCount = require('./order/crossCount'),
-    initLayerGraphs = require('./order/initLayerGraphs'),
-    initOrder = require('./order/initOrder'),
-    sortLayer = require('./order/sortLayer');
-
-module.exports = order;
-
-// The maximum number of sweeps to perform before finishing the order phase.
-var DEFAULT_MAX_SWEEPS = 24;
-order.DEFAULT_MAX_SWEEPS = DEFAULT_MAX_SWEEPS;
-
-/*
- * Runs the order phase with the specified `graph, `maxSweeps`, and
- * `debugLevel`. If `maxSweeps` is not specified we use `DEFAULT_MAX_SWEEPS`.
- * If `debugLevel` is not set we assume 0.
- */
-function order(g, maxSweeps) {
-  if (arguments.length < 2) {
-    maxSweeps = DEFAULT_MAX_SWEEPS;
-  }
-
-  var restarts = g.graph().orderRestarts || 0;
-
-  var layerGraphs = initLayerGraphs(g);
-  // TODO: remove this when we add back support for ordering clusters
-  layerGraphs.forEach(function(lg) {
-    lg = lg.filterNodes(function(u) { return !g.children(u).length; });
-  });
-
-  var iters = 0,
-      currentBestCC,
-      allTimeBestCC = Number.MAX_VALUE,
-      allTimeBest = {};
-
-  function saveAllTimeBest() {
-    g.eachNode(function(u, value) { allTimeBest[u] = value.order; });
-  }
-
-  for (var j = 0; j < Number(restarts) + 1 && allTimeBestCC !== 0; ++j) {
-    currentBestCC = Number.MAX_VALUE;
-    initOrder(g, restarts > 0);
-
-    util.log(2, 'Order phase start cross count: ' + g.graph().orderInitCC);
-
-    var i, lastBest, cc;
-    for (i = 0, lastBest = 0; lastBest < 4 && i < maxSweeps && currentBestCC > 0; ++i, ++lastBest, ++iters) {
-      sweep(g, layerGraphs, i);
-      cc = crossCount(g);
-      if (cc < currentBestCC) {
-        lastBest = 0;
-        currentBestCC = cc;
-        if (cc < allTimeBestCC) {
-          saveAllTimeBest();
-          allTimeBestCC = cc;
-        }
-      }
-      util.log(3, 'Order phase start ' + j + ' iter ' + i + ' cross count: ' + cc);
-    }
-  }
-
-  Object.keys(allTimeBest).forEach(function(u) {
-    if (!g.children || !g.children(u).length) {
-      g.node(u).order = allTimeBest[u];
-    }
-  });
-  g.graph().orderCC = allTimeBestCC;
-
-  util.log(2, 'Order iterations: ' + iters);
-  util.log(2, 'Order phase best cross count: ' + g.graph().orderCC);
-}
-
-function predecessorWeights(g, nodes) {
-  var weights = {};
-  nodes.forEach(function(u) {
-    weights[u] = g.inEdges(u).map(function(e) {
-      return g.node(g.source(e)).order;
-    });
-  });
-  return weights;
-}
-
-function successorWeights(g, nodes) {
-  var weights = {};
-  nodes.forEach(function(u) {
-    weights[u] = g.outEdges(u).map(function(e) {
-      return g.node(g.target(e)).order;
-    });
-  });
-  return weights;
-}
-
-function sweep(g, layerGraphs, iter) {
-  if (iter % 2 === 0) {
-    sweepDown(g, layerGraphs, iter);
-  } else {
-    sweepUp(g, layerGraphs, iter);
-  }
-}
-
-function sweepDown(g, layerGraphs) {
-  var cg;
-  for (var i = 1; i < layerGraphs.length; ++i) {
-    cg = sortLayer(layerGraphs[i], cg, predecessorWeights(g, layerGraphs[i].nodes()));
-  }
-}
-
-function sweepUp(g, layerGraphs) {
-  var cg;
-  for (var i = layerGraphs.length - 2; i >= 0; --i) {
-    sortLayer(layerGraphs[i], cg, successorWeights(g, layerGraphs[i].nodes()));
-  }
-}
-
-},{"./order/crossCount":15,"./order/initLayerGraphs":16,"./order/initOrder":17,"./order/sortLayer":18,"./util":27}],15:[function(require,module,exports){
-'use strict';
-
-var util = require('../util');
-
-module.exports = crossCount;
-
-/*
- * Returns the cross count for the given graph.
- */
-function crossCount(g) {
-  var cc = 0;
-  var ordering = util.ordering(g);
-  for (var i = 1; i < ordering.length; ++i) {
-    cc += twoLayerCrossCount(g, ordering[i-1], ordering[i]);
-  }
-  return cc;
-}
-
-/*
- * This function searches through a ranked and ordered graph and counts the
- * number of edges that cross. This algorithm is derived from:
- *
- *    W. Barth et al., Bilayer Cross Counting, JGAA, 8(2) 179–194 (2004)
- */
-function twoLayerCrossCount(g, layer1, layer2) {
-  var indices = [];
-  layer1.forEach(function(u) {
-    var nodeIndices = [];
-    g.outEdges(u).forEach(function(e) { nodeIndices.push(g.node(g.target(e)).order); });
-    nodeIndices.sort(function(x, y) { return x - y; });
-    indices = indices.concat(nodeIndices);
-  });
-
-  var firstIndex = 1;
-  while (firstIndex < layer2.length) firstIndex <<= 1;
-
-  var treeSize = 2 * firstIndex - 1;
-  firstIndex -= 1;
-
-  var tree = [];
-  for (var i = 0; i < treeSize; ++i) { tree[i] = 0; }
-
-  var cc = 0;
-  indices.forEach(function(i) {
-    var treeIndex = i + firstIndex;
-    ++tree[treeIndex];
-    while (treeIndex > 0) {
-      if (treeIndex % 2) {
-        cc += tree[treeIndex + 1];
-      }
-      treeIndex = (treeIndex - 1) >> 1;
-      ++tree[treeIndex];
-    }
-  });
-
-  return cc;
-}
-
-},{"../util":27}],16:[function(require,module,exports){
-'use strict';
-
-var nodesFromList = require('graphlib').filter.nodesFromList,
-    /* jshint -W079 */
-    Set = require('cp-data').Set;
-
-module.exports = initLayerGraphs;
-
-/*
- * This function takes a compound layered graph, g, and produces an array of
- * layer graphs. Each entry in the array represents a subgraph of nodes
- * relevant for performing crossing reduction on that layer.
- */
-function initLayerGraphs(g) {
-  var ranks = [];
-
-  function dfs(u) {
-    if (u === null) {
-      g.children(u).forEach(function(v) { dfs(v); });
-      return;
-    }
-
-    var value = g.node(u);
-    value.minRank = ('rank' in value) ? value.rank : Number.MAX_VALUE;
-    value.maxRank = ('rank' in value) ? value.rank : Number.MIN_VALUE;
-    var uRanks = new Set();
-    g.children(u).forEach(function(v) {
-      var rs = dfs(v);
-      uRanks = Set.union([uRanks, rs]);
-      value.minRank = Math.min(value.minRank, g.node(v).minRank);
-      value.maxRank = Math.max(value.maxRank, g.node(v).maxRank);
-    });
-
-    if ('rank' in value) uRanks.add(value.rank);
-
-    uRanks.keys().forEach(function(r) {
-      if (!(r in ranks)) ranks[r] = [];
-      ranks[r].push(u);
-    });
-
-    return uRanks;
-  }
-  dfs(null);
-
-  var layerGraphs = [];
-  ranks.forEach(function(us, rank) {
-    layerGraphs[rank] = g.filterNodes(nodesFromList(us));
-  });
-
-  return layerGraphs;
-}
-
-},{"cp-data":5,"graphlib":29}],17:[function(require,module,exports){
-'use strict';
-
-var crossCount = require('./crossCount'),
-    util = require('../util');
-
-module.exports = initOrder;
-
-/*
- * Given a graph with a set of layered nodes (i.e. nodes that have a `rank`
- * attribute) this function attaches an `order` attribute that uniquely
- * arranges each node of each rank. If no constraint graph is provided the
- * order of the nodes in each rank is entirely arbitrary.
- */
-function initOrder(g, random) {
-  var layers = [];
-
-  g.eachNode(function(u, value) {
-    var layer = layers[value.rank];
-    if (g.children && g.children(u).length > 0) return;
-    if (!layer) {
-      layer = layers[value.rank] = [];
-    }
-    layer.push(u);
-  });
-
-  layers.forEach(function(layer) {
-    if (random) {
-      util.shuffle(layer);
-    }
-    layer.forEach(function(u, i) {
-      g.node(u).order = i;
-    });
-  });
-
-  var cc = crossCount(g);
-  g.graph().orderInitCC = cc;
-  g.graph().orderCC = Number.MAX_VALUE;
-}
-
-},{"../util":27,"./crossCount":15}],18:[function(require,module,exports){
-'use strict';
-
-var util = require('../util'),
-    Digraph = require('graphlib').Digraph,
-    topsort = require('graphlib').alg.topsort,
-    nodesFromList = require('graphlib').filter.nodesFromList;
-
-module.exports = sortLayer;
-
-function sortLayer(g, cg, weights) {
-  weights = adjustWeights(g, weights);
-  var result = sortLayerSubgraph(g, null, cg, weights);
-
-  result.list.forEach(function(u, i) {
-    g.node(u).order = i;
-  });
-  return result.constraintGraph;
-}
-
-function sortLayerSubgraph(g, sg, cg, weights) {
-  cg = cg ? cg.filterNodes(nodesFromList(g.children(sg))) : new Digraph();
-
-  var nodeData = {};
-  g.children(sg).forEach(function(u) {
-    if (g.children(u).length) {
-      nodeData[u] = sortLayerSubgraph(g, u, cg, weights);
-      nodeData[u].firstSG = u;
-      nodeData[u].lastSG = u;
-    } else {
-      var ws = weights[u];
-      nodeData[u] = {
-        degree: ws.length,
-        barycenter: util.sum(ws) / ws.length,
-        order: g.node(u).order,
-        orderCount: 1,
-        list: [u]
-      };
-    }
-  });
-
-  resolveViolatedConstraints(g, cg, nodeData);
-
-  var keys = Object.keys(nodeData);
-  keys.sort(function(x, y) {
-    return nodeData[x].barycenter - nodeData[y].barycenter ||
-           nodeData[x].order - nodeData[y].order;
-  });
-
-  var result =  keys.map(function(u) { return nodeData[u]; })
-                    .reduce(function(lhs, rhs) { return mergeNodeData(g, lhs, rhs); });
-  return result;
-}
-
-function mergeNodeData(g, lhs, rhs) {
-  var cg = mergeDigraphs(lhs.constraintGraph, rhs.constraintGraph);
-
-  if (lhs.lastSG !== undefined && rhs.firstSG !== undefined) {
-    if (cg === undefined) {
-      cg = new Digraph();
-    }
-    if (!cg.hasNode(lhs.lastSG)) { cg.addNode(lhs.lastSG); }
-    cg.addNode(rhs.firstSG);
-    cg.addEdge(null, lhs.lastSG, rhs.firstSG);
-  }
-
-  return {
-    degree: lhs.degree + rhs.degree,
-    barycenter: (lhs.barycenter * lhs.degree + rhs.barycenter * rhs.degree) /
-                (lhs.degree + rhs.degree),
-    order: (lhs.order * lhs.orderCount + rhs.order * rhs.orderCount) /
-           (lhs.orderCount + rhs.orderCount),
-    orderCount: lhs.orderCount + rhs.orderCount,
-    list: lhs.list.concat(rhs.list),
-    firstSG: lhs.firstSG !== undefined ? lhs.firstSG : rhs.firstSG,
-    lastSG: rhs.lastSG !== undefined ? rhs.lastSG : lhs.lastSG,
-    constraintGraph: cg
-  };
-}
-
-function mergeDigraphs(lhs, rhs) {
-  if (lhs === undefined) return rhs;
-  if (rhs === undefined) return lhs;
-
-  lhs = lhs.copy();
-  rhs.nodes().forEach(function(u) { lhs.addNode(u); });
-  rhs.edges().forEach(function(e, u, v) { lhs.addEdge(null, u, v); });
-  return lhs;
-}
-
-function resolveViolatedConstraints(g, cg, nodeData) {
-  // Removes nodes `u` and `v` from `cg` and makes any edges incident on them
-  // incident on `w` instead.
-  function collapseNodes(u, v, w) {
-    // TODO original paper removes self loops, but it is not obvious when this would happen
-    cg.inEdges(u).forEach(function(e) {
-      cg.delEdge(e);
-      cg.addEdge(null, cg.source(e), w);
-    });
-
-    cg.outEdges(v).forEach(function(e) {
-      cg.delEdge(e);
-      cg.addEdge(null, w, cg.target(e));
-    });
-
-    cg.delNode(u);
-    cg.delNode(v);
-  }
-
-  var violated;
-  while ((violated = findViolatedConstraint(cg, nodeData)) !== undefined) {
-    var source = cg.source(violated),
-        target = cg.target(violated);
-
-    var v;
-    while ((v = cg.addNode(null)) && g.hasNode(v)) {
-      cg.delNode(v);
-    }
-
-    // Collapse barycenter and list
-    nodeData[v] = mergeNodeData(g, nodeData[source], nodeData[target]);
-    delete nodeData[source];
-    delete nodeData[target];
-
-    collapseNodes(source, target, v);
-    if (cg.incidentEdges(v).length === 0) { cg.delNode(v); }
-  }
-}
-
-function findViolatedConstraint(cg, nodeData) {
-  var us = topsort(cg);
-  for (var i = 0; i < us.length; ++i) {
-    var u = us[i];
-    var inEdges = cg.inEdges(u);
-    for (var j = 0; j < inEdges.length; ++j) {
-      var e = inEdges[j];
-      if (nodeData[cg.source(e)].barycenter >= nodeData[u].barycenter) {
-        return e;
-      }
-    }
-  }
-}
-
-// Adjust weights so that they fall in the range of 0..|N|-1. If a node has no
-// weight assigned then set its adjusted weight to its current position. This
-// allows us to better retain the origiinal position of nodes without neighbors.
-function adjustWeights(g, weights) {
-  var minW = Number.MAX_VALUE,
-      maxW = 0,
-      adjusted = {};
-  g.eachNode(function(u) {
-    if (g.children(u).length) return;
-
-    var ws = weights[u];
-    if (ws.length) {
-      minW = Math.min(minW, util.min(ws));
-      maxW = Math.max(maxW, util.max(ws));
-    }
-  });
-
-  var rangeW = (maxW - minW);
-  g.eachNode(function(u) {
-    if (g.children(u).length) return;
-
-    var ws = weights[u];
-    if (!ws.length) {
-      adjusted[u] = [g.node(u).order];
-    } else {
-      adjusted[u] = ws.map(function(w) {
-        if (rangeW) {
-          return (w - minW) * (g.order() - 1) / rangeW;
-        } else {
-          return g.order() - 1 / 2;
-        }
-      });
-    }
-  });
-
-  return adjusted;
-}
-
-},{"../util":27,"graphlib":29}],19:[function(require,module,exports){
-'use strict';
-
-var util = require('./util');
-
-/*
- * The algorithms here are based on Brandes and Köpf, "Fast and Simple
- * Horizontal Coordinate Assignment".
- */
-module.exports = function() {
-  // External configuration
-  var config = {
-    nodeSep: 50,
-    edgeSep: 10,
-    universalSep: null,
-    rankSep: 30
-  };
-
-  var self = {};
-
-  self.nodeSep = util.propertyAccessor(self, config, 'nodeSep');
-  self.edgeSep = util.propertyAccessor(self, config, 'edgeSep');
-  // If not null this separation value is used for all nodes and edges
-  // regardless of their widths. `nodeSep` and `edgeSep` are ignored with this
-  // option.
-  self.universalSep = util.propertyAccessor(self, config, 'universalSep');
-  self.rankSep = util.propertyAccessor(self, config, 'rankSep');
-  self.debugLevel = util.propertyAccessor(self, config, 'debugLevel');
-
-  self.run = run;
-
-  return self;
-
-  function run(g) {
-    g = g.filterNodes(util.filterNonSubgraphs(g));
-
-    var layering = util.ordering(g);
-
-    var conflicts = findConflicts(g, layering);
-
-    var xss = {};
-    ['u', 'd'].forEach(function(vertDir) {
-      if (vertDir === 'd') layering.reverse();
-
-      ['l', 'r'].forEach(function(horizDir) {
-        if (horizDir === 'r') reverseInnerOrder(layering);
-
-        var dir = vertDir + horizDir;
-        var align = verticalAlignment(g, layering, conflicts, vertDir === 'u' ? 'predecessors' : 'successors');
-        xss[dir]= horizontalCompaction(g, layering, align.pos, align.root, align.align);
-
-        if (config.debugLevel >= 3)
-          debugPositioning(vertDir + horizDir, g, layering, xss[dir]);
-
-        if (horizDir === 'r') flipHorizontally(xss[dir]);
-
-        if (horizDir === 'r') reverseInnerOrder(layering);
-      });
-
-      if (vertDir === 'd') layering.reverse();
-    });
-
-    balance(g, layering, xss);
-
-    g.eachNode(function(v) {
-      var xs = [];
-      for (var alignment in xss) {
-        var alignmentX = xss[alignment][v];
-        posXDebug(alignment, g, v, alignmentX);
-        xs.push(alignmentX);
-      }
-      xs.sort(function(x, y) { return x - y; });
-      posX(g, v, (xs[1] + xs[2]) / 2);
-    });
-
-    // Align y coordinates with ranks
-    var y = 0, reverseY = g.graph().rankDir === 'BT' || g.graph().rankDir === 'RL';
-    layering.forEach(function(layer) {
-      var maxHeight = util.max(layer.map(function(u) { return height(g, u); }));
-      y += maxHeight / 2;
-      layer.forEach(function(u) {
-        posY(g, u, reverseY ? -y : y);
-      });
-      y += maxHeight / 2 + config.rankSep;
-    });
-
-    // Translate layout so that top left corner of bounding rectangle has
-    // coordinate (0, 0).
-    var minX = util.min(g.nodes().map(function(u) { return posX(g, u) - width(g, u) / 2; }));
-    var minY = util.min(g.nodes().map(function(u) { return posY(g, u) - height(g, u) / 2; }));
-    g.eachNode(function(u) {
-      posX(g, u, posX(g, u) - minX);
-      posY(g, u, posY(g, u) - minY);
-    });
-  }
-
-  /*
-   * Generate an ID that can be used to represent any undirected edge that is
-   * incident on `u` and `v`.
-   */
-  function undirEdgeId(u, v) {
-    return u < v
-      ? u.toString().length + ':' + u + '-' + v
-      : v.toString().length + ':' + v + '-' + u;
-  }
-
-  function findConflicts(g, layering) {
-    var conflicts = {}, // Set of conflicting edge ids
-        pos = {},       // Position of node in its layer
-        prevLayer,
-        currLayer,
-        k0,     // Position of the last inner segment in the previous layer
-        l,      // Current position in the current layer (for iteration up to `l1`)
-        k1;     // Position of the next inner segment in the previous layer or
-                // the position of the last element in the previous layer
-
-    if (layering.length <= 2) return conflicts;
-
-    function updateConflicts(v) {
-      var k = pos[v];
-      if (k < k0 || k > k1) {
-        conflicts[undirEdgeId(currLayer[l], v)] = true;
-      }
-    }
-
-    layering[1].forEach(function(u, i) { pos[u] = i; });
-    for (var i = 1; i < layering.length - 1; ++i) {
-      prevLayer = layering[i];
-      currLayer = layering[i+1];
-      k0 = 0;
-      l = 0;
-
-      // Scan current layer for next node that is incident to an inner segement
-      // between layering[i+1] and layering[i].
-      for (var l1 = 0; l1 < currLayer.length; ++l1) {
-        var u = currLayer[l1]; // Next inner segment in the current layer or
-                               // last node in the current layer
-        pos[u] = l1;
-        k1 = undefined;
-
-        if (g.node(u).dummy) {
-          var uPred = g.predecessors(u)[0];
-          // Note: In the case of self loops and sideways edges it is possible
-          // for a dummy not to have a predecessor.
-          if (uPred !== undefined && g.node(uPred).dummy)
-            k1 = pos[uPred];
-        }
-        if (k1 === undefined && l1 === currLayer.length - 1)
-          k1 = prevLayer.length - 1;
-
-        if (k1 !== undefined) {
-          for (; l <= l1; ++l) {
-            g.predecessors(currLayer[l]).forEach(updateConflicts);
-          }
-          k0 = k1;
-        }
-      }
-    }
-
-    return conflicts;
-  }
-
-  function verticalAlignment(g, layering, conflicts, relationship) {
-    var pos = {},   // Position for a node in its layer
-        root = {},  // Root of the block that the node participates in
-        align = {}; // Points to the next node in the block or, if the last
-                    // element in the block, points to the first block's root
-
-    layering.forEach(function(layer) {
-      layer.forEach(function(u, i) {
-        root[u] = u;
-        align[u] = u;
-        pos[u] = i;
-      });
-    });
-
-    layering.forEach(function(layer) {
-      var prevIdx = -1;
-      layer.forEach(function(v) {
-        var related = g[relationship](v), // Adjacent nodes from the previous layer
-            mid;                          // The mid point in the related array
-
-        if (related.length > 0) {
-          related.sort(function(x, y) { return pos[x] - pos[y]; });
-          mid = (related.length - 1) / 2;
-          related.slice(Math.floor(mid), Math.ceil(mid) + 1).forEach(function(u) {
-            if (align[v] === v) {
-              if (!conflicts[undirEdgeId(u, v)] && prevIdx < pos[u]) {
-                align[u] = v;
-                align[v] = root[v] = root[u];
-                prevIdx = pos[u];
-              }
-            }
-          });
-        }
-      });
-    });
-
-    return { pos: pos, root: root, align: align };
-  }
-
-  // This function deviates from the standard BK algorithm in two ways. First
-  // it takes into account the size of the nodes. Second it includes a fix to
-  // the original algorithm that is described in Carstens, "Node and Label
-  // Placement in a Layered Layout Algorithm".
-  function horizontalCompaction(g, layering, pos, root, align) {
-    var sink = {},       // Mapping of node id -> sink node id for class
-        maybeShift = {}, // Mapping of sink node id -> { class node id, min shift }
-        shift = {},      // Mapping of sink node id -> shift
-        pred = {},       // Mapping of node id -> predecessor node (or null)
-        xs = {};         // Calculated X positions
-
-    layering.forEach(function(layer) {
-      layer.forEach(function(u, i) {
-        sink[u] = u;
-        maybeShift[u] = {};
-        if (i > 0)
-          pred[u] = layer[i - 1];
-      });
-    });
-
-    function updateShift(toShift, neighbor, delta) {
-      if (!(neighbor in maybeShift[toShift])) {
-        maybeShift[toShift][neighbor] = delta;
-      } else {
-        maybeShift[toShift][neighbor] = Math.min(maybeShift[toShift][neighbor], delta);
-      }
-    }
-
-    function placeBlock(v) {
-      if (!(v in xs)) {
-        xs[v] = 0;
-        var w = v;
-        do {
-          if (pos[w] > 0) {
-            var u = root[pred[w]];
-            placeBlock(u);
-            if (sink[v] === v) {
-              sink[v] = sink[u];
-            }
-            var delta = sep(g, pred[w]) + sep(g, w);
-            if (sink[v] !== sink[u]) {
-              updateShift(sink[u], sink[v], xs[v] - xs[u] - delta);
-            } else {
-              xs[v] = Math.max(xs[v], xs[u] + delta);
-            }
-          }
-          w = align[w];
-        } while (w !== v);
-      }
-    }
-
-    // Root coordinates relative to sink
-    util.values(root).forEach(function(v) {
-      placeBlock(v);
-    });
-
-    // Absolute coordinates
-    // There is an assumption here that we've resolved shifts for any classes
-    // that begin at an earlier layer. We guarantee this by visiting layers in
-    // order.
-    layering.forEach(function(layer) {
-      layer.forEach(function(v) {
-        xs[v] = xs[root[v]];
-        if (v === root[v] && v === sink[v]) {
-          var minShift = 0;
-          if (v in maybeShift && Object.keys(maybeShift[v]).length > 0) {
-            minShift = util.min(Object.keys(maybeShift[v])
-                                 .map(function(u) {
-                                      return maybeShift[v][u] + (u in shift ? shift[u] : 0);
-                                      }
-                                 ));
-          }
-          shift[v] = minShift;
-        }
-      });
-    });
-
-    layering.forEach(function(layer) {
-      layer.forEach(function(v) {
-        xs[v] += shift[sink[root[v]]] || 0;
-      });
-    });
-
-    return xs;
-  }
-
-  function findMinCoord(g, layering, xs) {
-    return util.min(layering.map(function(layer) {
-      var u = layer[0];
-      return xs[u];
-    }));
-  }
-
-  function findMaxCoord(g, layering, xs) {
-    return util.max(layering.map(function(layer) {
-      var u = layer[layer.length - 1];
-      return xs[u];
-    }));
-  }
-
-  function balance(g, layering, xss) {
-    var min = {},                            // Min coordinate for the alignment
-        max = {},                            // Max coordinate for the alginment
-        smallestAlignment,
-        shift = {};                          // Amount to shift a given alignment
-
-    function updateAlignment(v) {
-      xss[alignment][v] += shift[alignment];
-    }
-
-    var smallest = Number.POSITIVE_INFINITY;
-    for (var alignment in xss) {
-      var xs = xss[alignment];
-      min[alignment] = findMinCoord(g, layering, xs);
-      max[alignment] = findMaxCoord(g, layering, xs);
-      var w = max[alignment] - min[alignment];
-      if (w < smallest) {
-        smallest = w;
-        smallestAlignment = alignment;
-      }
-    }
-
-    // Determine how much to adjust positioning for each alignment
-    ['u', 'd'].forEach(function(vertDir) {
-      ['l', 'r'].forEach(function(horizDir) {
-        var alignment = vertDir + horizDir;
-        shift[alignment] = horizDir === 'l'
-            ? min[smallestAlignment] - min[alignment]
-            : max[smallestAlignment] - max[alignment];
-      });
-    });
-
-    // Find average of medians for xss array
-    for (alignment in xss) {
-      g.eachNode(updateAlignment);
-    }
-  }
-
-  function flipHorizontally(xs) {
-    for (var u in xs) {
-      xs[u] = -xs[u];
-    }
-  }
-
-  function reverseInnerOrder(layering) {
-    layering.forEach(function(layer) {
-      layer.reverse();
-    });
-  }
-
-  function width(g, u) {
-    switch (g.graph().rankDir) {
-      case 'LR': return g.node(u).height;
-      case 'RL': return g.node(u).height;
-      default:   return g.node(u).width;
-    }
-  }
-
-  function height(g, u) {
-    switch(g.graph().rankDir) {
-      case 'LR': return g.node(u).width;
-      case 'RL': return g.node(u).width;
-      default:   return g.node(u).height;
-    }
-  }
-
-  function sep(g, u) {
-    if (config.universalSep !== null) {
-      return config.universalSep;
-    }
-    var w = width(g, u);
-    var s = g.node(u).dummy ? config.edgeSep : config.nodeSep;
-    return (w + s) / 2;
-  }
-
-  function posX(g, u, x) {
-    if (g.graph().rankDir === 'LR' || g.graph().rankDir === 'RL') {
-      if (arguments.length < 3) {
-        return g.node(u).y;
-      } else {
-        g.node(u).y = x;
-      }
-    } else {
-      if (arguments.length < 3) {
-        return g.node(u).x;
-      } else {
-        g.node(u).x = x;
-      }
-    }
-  }
-
-  function posXDebug(name, g, u, x) {
-    if (g.graph().rankDir === 'LR' || g.graph().rankDir === 'RL') {
-      if (arguments.length < 3) {
-        return g.node(u)[name];
-      } else {
-        g.node(u)[name] = x;
-      }
-    } else {
-      if (arguments.length < 3) {
-        return g.node(u)[name];
-      } else {
-        g.node(u)[name] = x;
-      }
-    }
-  }
-
-  function posY(g, u, y) {
-    if (g.graph().rankDir === 'LR' || g.graph().rankDir === 'RL') {
-      if (arguments.length < 3) {
-        return g.node(u).x;
-      } else {
-        g.node(u).x = y;
-      }
-    } else {
-      if (arguments.length < 3) {
-        return g.node(u).y;
-      } else {
-        g.node(u).y = y;
-      }
-    }
-  }
-
-  function debugPositioning(align, g, layering, xs) {
-    layering.forEach(function(l, li) {
-      var u, xU;
-      l.forEach(function(v) {
-        var xV = xs[v];
-        if (u) {
-          var s = sep(g, u) + sep(g, v);
-          if (xV - xU < s)
-            console.log('Position phase: sep violation. Align: ' + align + '. Layer: ' + li + '. ' +
-              'U: ' + u + ' V: ' + v + '. Actual sep: ' + (xV - xU) + ' Expected sep: ' + s);
-        }
-        u = v;
-        xU = xV;
-      });
-    });
-  }
-};
-
-},{"./util":27}],20:[function(require,module,exports){
-'use strict';
-
-var util = require('./util'),
-    acyclic = require('./rank/acyclic'),
-    initRank = require('./rank/initRank'),
-    feasibleTree = require('./rank/feasibleTree'),
-    constraints = require('./rank/constraints'),
-    simplex = require('./rank/simplex'),
-    components = require('graphlib').alg.components,
-    filter = require('graphlib').filter;
-
-exports.run = run;
-exports.restoreEdges = restoreEdges;
-
-/*
- * Heuristic function that assigns a rank to each node of the input graph with
- * the intent of minimizing edge lengths, while respecting the `minLen`
- * attribute of incident edges.
- *
- * Prerequisites:
- *
- *  * Each edge in the input graph must have an assigned 'minLen' attribute
- */
-function run(g, useSimplex) {
-  expandSelfLoops(g);
-
-  // If there are rank constraints on nodes, then build a new graph that
-  // encodes the constraints.
-  util.time('constraints.apply', constraints.apply)(g);
-
-  expandSidewaysEdges(g);
-
-  // Reverse edges to get an acyclic graph, we keep the graph in an acyclic
-  // state until the very end.
-  util.time('acyclic', acyclic)(g);
-
-  // Convert the graph into a flat graph for ranking
-  var flatGraph = g.filterNodes(util.filterNonSubgraphs(g));
-
-  // Assign an initial ranking using DFS.
-  initRank(flatGraph);
-
-  // For each component improve the assigned ranks.
-  components(flatGraph).forEach(function(cmpt) {
-    var subgraph = flatGraph.filterNodes(filter.nodesFromList(cmpt));
-    rankComponent(subgraph, useSimplex);
-  });
-
-  // Relax original constraints
-  util.time('constraints.relax', constraints.relax(g));
-
-  // When handling nodes with constrained ranks it is possible to end up with
-  // edges that point to previous ranks. Most of the subsequent algorithms assume
-  // that edges are pointing to successive ranks only. Here we reverse any "back
-  // edges" and mark them as such. The acyclic algorithm will reverse them as a
-  // post processing step.
-  util.time('reorientEdges', reorientEdges)(g);
-}
-
-function restoreEdges(g) {
-  acyclic.undo(g);
-}
-
-/*
- * Expand self loops into three dummy nodes. One will sit above the incident
- * node, one will be at the same level, and one below. The result looks like:
- *
- *         /--<--x--->--\
- *     node              y
- *         \--<--z--->--/
- *
- * Dummy nodes x, y, z give us the shape of a loop and node y is where we place
- * the label.
- *
- * TODO: consolidate knowledge of dummy node construction.
- * TODO: support minLen = 2
- */
-function expandSelfLoops(g) {
-  g.eachEdge(function(e, u, v, a) {
-    if (u === v) {
-      var x = addDummyNode(g, e, u, v, a, 0, false),
-          y = addDummyNode(g, e, u, v, a, 1, true),
-          z = addDummyNode(g, e, u, v, a, 2, false);
-      g.addEdge(null, x, u, {minLen: 1, selfLoop: true});
-      g.addEdge(null, x, y, {minLen: 1, selfLoop: true});
-      g.addEdge(null, u, z, {minLen: 1, selfLoop: true});
-      g.addEdge(null, y, z, {minLen: 1, selfLoop: true});
-      g.delEdge(e);
-    }
-  });
-}
-
-function expandSidewaysEdges(g) {
-  g.eachEdge(function(e, u, v, a) {
-    if (u === v) {
-      var origEdge = a.originalEdge,
-          dummy = addDummyNode(g, origEdge.e, origEdge.u, origEdge.v, origEdge.value, 0, true);
-      g.addEdge(null, u, dummy, {minLen: 1});
-      g.addEdge(null, dummy, v, {minLen: 1});
-      g.delEdge(e);
-    }
-  });
-}
-
-function addDummyNode(g, e, u, v, a, index, isLabel) {
-  return g.addNode(null, {
-    width: isLabel ? a.width : 0,
-    height: isLabel ? a.height : 0,
-    edge: { id: e, source: u, target: v, attrs: a },
-    dummy: true,
-    index: index
-  });
-}
-
-function reorientEdges(g) {
-  g.eachEdge(function(e, u, v, value) {
-    if (g.node(u).rank > g.node(v).rank) {
-      g.delEdge(e);
-      value.reversed = true;
-      g.addEdge(e, v, u, value);
-    }
-  });
-}
-
-function rankComponent(subgraph, useSimplex) {
-  var spanningTree = feasibleTree(subgraph);
-
-  if (useSimplex) {
-    util.log(1, 'Using network simplex for ranking');
-    simplex(subgraph, spanningTree);
-  }
-  normalize(subgraph);
-}
-
-function normalize(g) {
-  var m = util.min(g.nodes().map(function(u) { return g.node(u).rank; }));
-  g.eachNode(function(u, node) { node.rank -= m; });
-}
-
-},{"./rank/acyclic":21,"./rank/constraints":22,"./rank/feasibleTree":23,"./rank/initRank":24,"./rank/simplex":26,"./util":27,"graphlib":29}],21:[function(require,module,exports){
-'use strict';
-
-var util = require('../util');
-
-module.exports = acyclic;
-module.exports.undo = undo;
-
-/*
- * This function takes a directed graph that may have cycles and reverses edges
- * as appropriate to break these cycles. Each reversed edge is assigned a
- * `reversed` attribute with the value `true`.
- *
- * There should be no self loops in the graph.
- */
-function acyclic(g) {
-  var onStack = {},
-      visited = {},
-      reverseCount = 0;
-
-  function dfs(u) {
-    if (u in visited) return;
-    visited[u] = onStack[u] = true;
-    g.outEdges(u).forEach(function(e) {
-      var t = g.target(e),
-          value;
-
-      if (u === t) {
-        console.error('Warning: found self loop "' + e + '" for node "' + u + '"');
-      } else if (t in onStack) {
-        value = g.edge(e);
-        g.delEdge(e);
-        value.reversed = true;
-        ++reverseCount;
-        g.addEdge(e, t, u, value);
-      } else {
-        dfs(t);
-      }
-    });
-
-    delete onStack[u];
-  }
-
-  g.eachNode(function(u) { dfs(u); });
-
-  util.log(2, 'Acyclic Phase: reversed ' + reverseCount + ' edge(s)');
-
-  return reverseCount;
-}
-
-/*
- * Given a graph that has had the acyclic operation applied, this function
- * undoes that operation. More specifically, any edge with the `reversed`
- * attribute is again reversed to restore the original direction of the edge.
- */
-function undo(g) {
-  g.eachEdge(function(e, s, t, a) {
-    if (a.reversed) {
-      delete a.reversed;
-      g.delEdge(e);
-      g.addEdge(e, t, s, a);
-    }
-  });
-}
-
-},{"../util":27}],22:[function(require,module,exports){
-'use strict';
-
-exports.apply = function(g) {
-  function dfs(sg) {
-    var rankSets = {};
-    g.children(sg).forEach(function(u) {
-      if (g.children(u).length) {
-        dfs(u);
-        return;
-      }
-
-      var value = g.node(u),
-          prefRank = value.prefRank;
-      if (prefRank !== undefined) {
-        if (!checkSupportedPrefRank(prefRank)) { return; }
-
-        if (!(prefRank in rankSets)) {
-          rankSets.prefRank = [u];
-        } else {
-          rankSets.prefRank.push(u);
-        }
-
-        var newU = rankSets[prefRank];
-        if (newU === undefined) {
-          newU = rankSets[prefRank] = g.addNode(null, { originalNodes: [] });
-          g.parent(newU, sg);
-        }
-
-        redirectInEdges(g, u, newU, prefRank === 'min');
-        redirectOutEdges(g, u, newU, prefRank === 'max');
-
-        // Save original node and remove it from reduced graph
-        g.node(newU).originalNodes.push({ u: u, value: value, parent: sg });
-        g.delNode(u);
-      }
-    });
-
-    addLightEdgesFromMinNode(g, sg, rankSets.min);
-    addLightEdgesToMaxNode(g, sg, rankSets.max);
-  }
-
-  dfs(null);
-};
-
-function checkSupportedPrefRank(prefRank) {
-  if (prefRank !== 'min' && prefRank !== 'max' && prefRank.indexOf('same_') !== 0) {
-    console.error('Unsupported rank type: ' + prefRank);
-    return false;
-  }
-  return true;
-}
-
-function redirectInEdges(g, u, newU, reverse) {
-  g.inEdges(u).forEach(function(e) {
-    var origValue = g.edge(e),
-        value;
-    if (origValue.originalEdge) {
-      value = origValue;
-    } else {
-      value =  {
-        originalEdge: { e: e, u: g.source(e), v: g.target(e), value: origValue },
-        minLen: g.edge(e).minLen
-      };
-    }
-
-    // Do not reverse edges for self-loops.
-    if (origValue.selfLoop) {
-      reverse = false;
-    }
-
-    if (reverse) {
-      // Ensure that all edges to min are reversed
-      g.addEdge(null, newU, g.source(e), value);
-      value.reversed = true;
-    } else {
-      g.addEdge(null, g.source(e), newU, value);
-    }
-  });
-}
-
-function redirectOutEdges(g, u, newU, reverse) {
-  g.outEdges(u).forEach(function(e) {
-    var origValue = g.edge(e),
-        value;
-    if (origValue.originalEdge) {
-      value = origValue;
-    } else {
-      value =  {
-        originalEdge: { e: e, u: g.source(e), v: g.target(e), value: origValue },
-        minLen: g.edge(e).minLen
-      };
-    }
-
-    // Do not reverse edges for self-loops.
-    if (origValue.selfLoop) {
-      reverse = false;
-    }
-
-    if (reverse) {
-      // Ensure that all edges from max are reversed
-      g.addEdge(null, g.target(e), newU, value);
-      value.reversed = true;
-    } else {
-      g.addEdge(null, newU, g.target(e), value);
-    }
-  });
-}
-
-function addLightEdgesFromMinNode(g, sg, minNode) {
-  if (minNode !== undefined) {
-    g.children(sg).forEach(function(u) {
-      // The dummy check ensures we don't add an edge if the node is involved
-      // in a self loop or sideways edge.
-      if (u !== minNode && !g.outEdges(minNode, u).length && !g.node(u).dummy) {
-        g.addEdge(null, minNode, u, { minLen: 0 });
-      }
-    });
-  }
-}
-
-function addLightEdgesToMaxNode(g, sg, maxNode) {
-  if (maxNode !== undefined) {
-    g.children(sg).forEach(function(u) {
-      // The dummy check ensures we don't add an edge if the node is involved
-      // in a self loop or sideways edge.
-      if (u !== maxNode && !g.outEdges(u, maxNode).length && !g.node(u).dummy) {
-        g.addEdge(null, u, maxNode, { minLen: 0 });
-      }
-    });
-  }
-}
-
-/*
- * This function "relaxes" the constraints applied previously by the "apply"
- * function. It expands any nodes that were collapsed and assigns the rank of
- * the collapsed node to each of the expanded nodes. It also restores the
- * original edges and removes any dummy edges pointing at the collapsed nodes.
- *
- * Note that the process of removing collapsed nodes also removes dummy edges
- * automatically.
- */
-exports.relax = function(g) {
-  // Save original edges
-  var originalEdges = [];
-  g.eachEdge(function(e, u, v, value) {
-    var originalEdge = value.originalEdge;
-    if (originalEdge) {
-      originalEdges.push(originalEdge);
-    }
-  });
-
-  // Expand collapsed nodes
-  g.eachNode(function(u, value) {
-    var originalNodes = value.originalNodes;
-    if (originalNodes) {
-      originalNodes.forEach(function(originalNode) {
-        originalNode.value.rank = value.rank;
-        g.addNode(originalNode.u, originalNode.value);
-        g.parent(originalNode.u, originalNode.parent);
-      });
-      g.delNode(u);
-    }
-  });
-
-  // Restore original edges
-  originalEdges.forEach(function(edge) {
-    g.addEdge(edge.e, edge.u, edge.v, edge.value);
-  });
-};
-
-},{}],23:[function(require,module,exports){
-'use strict';
-
-/* jshint -W079 */
-var Set = require('cp-data').Set,
-/* jshint +W079 */
-    Digraph = require('graphlib').Digraph,
-    util = require('../util');
-
-module.exports = feasibleTree;
-
-/*
- * Given an acyclic graph with each node assigned a `rank` attribute, this
- * function constructs and returns a spanning tree. This function may reduce
- * the length of some edges from the initial rank assignment while maintaining
- * the `minLen` specified by each edge.
- *
- * Prerequisites:
- *
- * * The input graph is acyclic
- * * Each node in the input graph has an assigned `rank` attribute
- * * Each edge in the input graph has an assigned `minLen` attribute
- *
- * Outputs:
- *
- * A feasible spanning tree for the input graph (i.e. a spanning tree that
- * respects each graph edge's `minLen` attribute) represented as a Digraph with
- * a `root` attribute on graph.
- *
- * Nodes have the same id and value as that in the input graph.
- *
- * Edges in the tree have arbitrarily assigned ids. The attributes for edges
- * include `reversed`. `reversed` indicates that the edge is a
- * back edge in the input graph.
- */
-function feasibleTree(g) {
-  var remaining = new Set(g.nodes()),
-      tree = new Digraph();
-
-  if (remaining.size() === 1) {
-    var root = g.nodes()[0];
-    tree.addNode(root, {});
-    tree.graph({ root: root });
-    return tree;
-  }
-
-  function addTightEdges(v) {
-    var continueToScan = true;
-    g.predecessors(v).forEach(function(u) {
-      if (remaining.has(u) && !slack(g, u, v)) {
-        if (remaining.has(v)) {
-          tree.addNode(v, {});
-          remaining.remove(v);
-          tree.graph({ root: v });
-        }
-
-        tree.addNode(u, {});
-        tree.addEdge(null, u, v, { reversed: true });
-        remaining.remove(u);
-        addTightEdges(u);
-        continueToScan = false;
-      }
-    });
-
-    g.successors(v).forEach(function(w)  {
-      if (remaining.has(w) && !slack(g, v, w)) {
-        if (remaining.has(v)) {
-          tree.addNode(v, {});
-          remaining.remove(v);
-          tree.graph({ root: v });
-        }
-
-        tree.addNode(w, {});
-        tree.addEdge(null, v, w, {});
-        remaining.remove(w);
-        addTightEdges(w);
-        continueToScan = false;
-      }
-    });
-    return continueToScan;
-  }
-
-  function createTightEdge() {
-    var minSlack = Number.MAX_VALUE;
-    remaining.keys().forEach(function(v) {
-      g.predecessors(v).forEach(function(u) {
-        if (!remaining.has(u)) {
-          var edgeSlack = slack(g, u, v);
-          if (Math.abs(edgeSlack) < Math.abs(minSlack)) {
-            minSlack = -edgeSlack;
-          }
-        }
-      });
-
-      g.successors(v).forEach(function(w) {
-        if (!remaining.has(w)) {
-          var edgeSlack = slack(g, v, w);
-          if (Math.abs(edgeSlack) < Math.abs(minSlack)) {
-            minSlack = edgeSlack;
-          }
-        }
-      });
-    });
-
-    tree.eachNode(function(u) { g.node(u).rank -= minSlack; });
-  }
-
-  while (remaining.size()) {
-    var nodesToSearch = !tree.order() ? remaining.keys() : tree.nodes();
-    for (var i = 0, il = nodesToSearch.length;
-         i < il && addTightEdges(nodesToSearch[i]);
-         ++i);
-    if (remaining.size()) {
-      createTightEdge();
-    }
-  }
-
-  return tree;
-}
-
-function slack(g, u, v) {
-  var rankDiff = g.node(v).rank - g.node(u).rank;
-  var maxMinLen = util.max(g.outEdges(u, v)
-                            .map(function(e) { return g.edge(e).minLen; }));
-  return rankDiff - maxMinLen;
-}
-
-},{"../util":27,"cp-data":5,"graphlib":29}],24:[function(require,module,exports){
-'use strict';
-
-var util = require('../util'),
-    topsort = require('graphlib').alg.topsort;
-
-module.exports = initRank;
-
-/*
- * Assigns a `rank` attribute to each node in the input graph and ensures that
- * this rank respects the `minLen` attribute of incident edges.
- *
- * Prerequisites:
- *
- *  * The input graph must be acyclic
- *  * Each edge in the input graph must have an assigned 'minLen' attribute
- */
-function initRank(g) {
-  var sorted = topsort(g);
-
-  sorted.forEach(function(u) {
-    var inEdges = g.inEdges(u);
-    if (inEdges.length === 0) {
-      g.node(u).rank = 0;
-      return;
-    }
-
-    var minLens = inEdges.map(function(e) {
-      return g.node(g.source(e)).rank + g.edge(e).minLen;
-    });
-    g.node(u).rank = util.max(minLens);
-  });
-}
-
-},{"../util":27,"graphlib":29}],25:[function(require,module,exports){
-'use strict';
-
-module.exports = {
-  slack: slack
-};
-
-/*
- * A helper to calculate the slack between two nodes (`u` and `v`) given a
- * `minLen` constraint. The slack represents how much the distance between `u`
- * and `v` could shrink while maintaining the `minLen` constraint. If the value
- * is negative then the constraint is currently violated.
- *
-  This function requires that `u` and `v` are in `graph` and they both have a
-  `rank` attribute.
- */
-function slack(graph, u, v, minLen) {
-  return Math.abs(graph.node(u).rank - graph.node(v).rank) - minLen;
-}
-
-},{}],26:[function(require,module,exports){
-'use strict';
-
-var util = require('../util'),
-    rankUtil = require('./rankUtil');
-
-module.exports = simplex;
-
-function simplex(graph, spanningTree) {
-  // The network simplex algorithm repeatedly replaces edges of
-  // the spanning tree with negative cut values until no such
-  // edge exists.
-  initCutValues(graph, spanningTree);
-  while (true) {
-    var e = leaveEdge(spanningTree);
-    if (e === null) break;
-    var f = enterEdge(graph, spanningTree, e);
-    exchange(graph, spanningTree, e, f);
-  }
-}
-
-/*
- * Set the cut values of edges in the spanning tree by a depth-first
- * postorder traversal.  The cut value corresponds to the cost, in
- * terms of a ranking's edge length sum, of lengthening an edge.
- * Negative cut values typically indicate edges that would yield a
- * smaller edge length sum if they were lengthened.
- */
-function initCutValues(graph, spanningTree) {
-  computeLowLim(spanningTree);
-
-  spanningTree.eachEdge(function(id, u, v, treeValue) {
-    treeValue.cutValue = 0;
-  });
-
-  // Propagate cut values up the tree.
-  function dfs(n) {
-    var children = spanningTree.successors(n);
-    for (var c in children) {
-      var child = children[c];
-      dfs(child);
-    }
-    if (n !== spanningTree.graph().root) {
-      setCutValue(graph, spanningTree, n);
-    }
-  }
-  dfs(spanningTree.graph().root);
-}
-
-/*
- * Perform a DFS postorder traversal, labeling each node v with
- * its traversal order 'lim(v)' and the minimum traversal number
- * of any of its descendants 'low(v)'.  This provides an efficient
- * way to test whether u is an ancestor of v since
- * low(u) <= lim(v) <= lim(u) if and only if u is an ancestor.
- */
-function computeLowLim(tree) {
-  var postOrderNum = 0;
-
-  function dfs(n) {
-    var children = tree.successors(n);
-    var low = postOrderNum;
-    for (var c in children) {
-      var child = children[c];
-      dfs(child);
-      low = Math.min(low, tree.node(child).low);
-    }
-    tree.node(n).low = low;
-    tree.node(n).lim = postOrderNum++;
-  }
-
-  dfs(tree.graph().root);
-}
-
-/*
- * To compute the cut value of the edge parent -> child, we consider
- * it and any other graph edges to or from the child.
- *          parent
- *             |
- *           child
- *          /      \
- *         u        v
- */
-function setCutValue(graph, tree, child) {
-  var parentEdge = tree.inEdges(child)[0];
-
-  // List of child's children in the spanning tree.
-  var grandchildren = [];
-  var grandchildEdges = tree.outEdges(child);
-  for (var gce in grandchildEdges) {
-    grandchildren.push(tree.target(grandchildEdges[gce]));
-  }
-
-  var cutValue = 0;
-
-  // TODO: Replace unit increment/decrement with edge weights.
-  var E = 0;    // Edges from child to grandchild's subtree.
-  var F = 0;    // Edges to child from grandchild's subtree.
-  var G = 0;    // Edges from child to nodes outside of child's subtree.
-  var H = 0;    // Edges from nodes outside of child's subtree to child.
-
-  // Consider all graph edges from child.
-  var outEdges = graph.outEdges(child);
-  var gc;
-  for (var oe in outEdges) {
-    var succ = graph.target(outEdges[oe]);
-    for (gc in grandchildren) {
-      if (inSubtree(tree, succ, grandchildren[gc])) {
-        E++;
-      }
-    }
-    if (!inSubtree(tree, succ, child)) {
-      G++;
-    }
-  }
-
-  // Consider all graph edges to child.
-  var inEdges = graph.inEdges(child);
-  for (var ie in inEdges) {
-    var pred = graph.source(inEdges[ie]);
-    for (gc in grandchildren) {
-      if (inSubtree(tree, pred, grandchildren[gc])) {
-        F++;
-      }
-    }
-    if (!inSubtree(tree, pred, child)) {
-      H++;
-    }
-  }
-
-  // Contributions depend on the alignment of the parent -> child edge
-  // and the child -> u or v edges.
-  var grandchildCutSum = 0;
-  for (gc in grandchildren) {
-    var cv = tree.edge(grandchildEdges[gc]).cutValue;
-    if (!tree.edge(grandchildEdges[gc]).reversed) {
-      grandchildCutSum += cv;
-    } else {
-      grandchildCutSum -= cv;
-    }
-  }
-
-  if (!tree.edge(parentEdge).reversed) {
-    cutValue += grandchildCutSum - E + F - G + H;
-  } else {
-    cutValue -= grandchildCutSum - E + F - G + H;
-  }
-
-  tree.edge(parentEdge).cutValue = cutValue;
-}
-
-/*
- * Return whether n is a node in the subtree with the given
- * root.
- */
-function inSubtree(tree, n, root) {
-  return (tree.node(root).low <= tree.node(n).lim &&
-          tree.node(n).lim <= tree.node(root).lim);
-}
-
-/*
- * Return an edge from the tree with a negative cut value, or null if there
- * is none.
- */
-function leaveEdge(tree) {
-  var edges = tree.edges();
-  for (var n in edges) {
-    var e = edges[n];
-    var treeValue = tree.edge(e);
-    if (treeValue.cutValue < 0) {
-      return e;
-    }
-  }
-  return null;
-}
-
-/*
- * The edge e should be an edge in the tree, with an underlying edge
- * in the graph, with a negative cut value.  Of the two nodes incident
- * on the edge, take the lower one.  enterEdge returns an edge with
- * minimum slack going from outside of that node's subtree to inside
- * of that node's subtree.
- */
-function enterEdge(graph, tree, e) {
-  var source = tree.source(e);
-  var target = tree.target(e);
-  var lower = tree.node(target).lim < tree.node(source).lim ? target : source;
-
-  // Is the tree edge aligned with the graph edge?
-  var aligned = !tree.edge(e).reversed;
-
-  var minSlack = Number.POSITIVE_INFINITY;
-  var minSlackEdge;
-  if (aligned) {
-    graph.eachEdge(function(id, u, v, value) {
-      if (id !== e && inSubtree(tree, u, lower) && !inSubtree(tree, v, lower)) {
-        var slack = rankUtil.slack(graph, u, v, value.minLen);
-        if (slack < minSlack) {
-          minSlack = slack;
-          minSlackEdge = id;
-        }
-      }
-    });
-  } else {
-    graph.eachEdge(function(id, u, v, value) {
-      if (id !== e && !inSubtree(tree, u, lower) && inSubtree(tree, v, lower)) {
-        var slack = rankUtil.slack(graph, u, v, value.minLen);
-        if (slack < minSlack) {
-          minSlack = slack;
-          minSlackEdge = id;
-        }
-      }
-    });
-  }
-
-  if (minSlackEdge === undefined) {
-    var outside = [];
-    var inside = [];
-    graph.eachNode(function(id) {
-      if (!inSubtree(tree, id, lower)) {
-        outside.push(id);
-      } else {
-        inside.push(id);
-      }
-    });
-    throw new Error('No edge found from outside of tree to inside');
-  }
-
-  return minSlackEdge;
-}
-
-/*
- * Replace edge e with edge f in the tree, recalculating the tree root,
- * the nodes' low and lim properties and the edges' cut values.
- */
-function exchange(graph, tree, e, f) {
-  tree.delEdge(e);
-  var source = graph.source(f);
-  var target = graph.target(f);
-
-  // Redirect edges so that target is the root of its subtree.
-  function redirect(v) {
-    var edges = tree.inEdges(v);
-    for (var i in edges) {
-      var e = edges[i];
-      var u = tree.source(e);
-      var value = tree.edge(e);
-      redirect(u);
-      tree.delEdge(e);
-      value.reversed = !value.reversed;
-      tree.addEdge(e, v, u, value);
-    }
-  }
-
-  redirect(target);
-
-  var root = source;
-  var edges = tree.inEdges(root);
-  while (edges.length > 0) {
-    root = tree.source(edges[0]);
-    edges = tree.inEdges(root);
-  }
-
-  tree.graph().root = root;
-
-  tree.addEdge(null, source, target, {cutValue: 0});
-
-  initCutValues(graph, tree);
-
-  adjustRanks(graph, tree);
-}
-
-/*
- * Reset the ranks of all nodes based on the current spanning tree.
- * The rank of the tree's root remains unchanged, while all other
- * nodes are set to the sum of minimum length constraints along
- * the path from the root.
- */
-function adjustRanks(graph, tree) {
-  function dfs(p) {
-    var children = tree.successors(p);
-    children.forEach(function(c) {
-      var minLen = minimumLength(graph, p, c);
-      graph.node(c).rank = graph.node(p).rank + minLen;
-      dfs(c);
-    });
-  }
-
-  dfs(tree.graph().root);
-}
-
-/*
- * If u and v are connected by some edges in the graph, return the
- * minimum length of those edges, as a positive number if v succeeds
- * u and as a negative number if v precedes u.
- */
-function minimumLength(graph, u, v) {
-  var outEdges = graph.outEdges(u, v);
-  if (outEdges.length > 0) {
-    return util.max(outEdges.map(function(e) {
-      return graph.edge(e).minLen;
-    }));
-  }
-
-  var inEdges = graph.inEdges(u, v);
-  if (inEdges.length > 0) {
-    return -util.max(inEdges.map(function(e) {
-      return graph.edge(e).minLen;
-    }));
-  }
-}
-
-},{"../util":27,"./rankUtil":25}],27:[function(require,module,exports){
-'use strict';
-
-/*
- * Returns the smallest value in the array.
- */
-exports.min = function(values) {
-  return Math.min.apply(Math, values);
-};
-
-/*
- * Returns the largest value in the array.
- */
-exports.max = function(values) {
-  return Math.max.apply(Math, values);
-};
-
-/*
- * Returns `true` only if `f(x)` is `true` for all `x` in `xs`. Otherwise
- * returns `false`. This function will return immediately if it finds a
- * case where `f(x)` does not hold.
- */
-exports.all = function(xs, f) {
-  for (var i = 0; i < xs.length; ++i) {
-    if (!f(xs[i])) {
-      return false;
-    }
-  }
-  return true;
-};
-
-/*
- * Accumulates the sum of elements in the given array using the `+` operator.
- */
-exports.sum = function(values) {
-  return values.reduce(function(acc, x) { return acc + x; }, 0);
-};
-
-/*
- * Returns an array of all values in the given object.
- */
-exports.values = function(obj) {
-  return Object.keys(obj).map(function(k) { return obj[k]; });
-};
-
-exports.shuffle = function(array) {
-  for (var i = array.length - 1; i > 0; --i) {
-    var j = Math.floor(Math.random() * (i + 1));
-    var aj = array[j];
-    array[j] = array[i];
-    array[i] = aj;
-  }
-};
-
-exports.propertyAccessor = function(self, config, field, setHook) {
-  return function(x) {
-    if (!arguments.length) return config[field];
-    config[field] = x;
-    if (setHook) setHook(x);
-    return self;
-  };
-};
-
-/*
- * Given a layered, directed graph with `rank` and `order` node attributes,
- * this function returns an array of ordered ranks. Each rank contains an array
- * of the ids of the nodes in that rank in the order specified by the `order`
- * attribute.
- */
-exports.ordering = function(g) {
-  var ordering = [];
-  g.eachNode(function(u, value) {
-    var rank = ordering[value.rank] || (ordering[value.rank] = []);
-    rank[value.order] = u;
-  });
-  return ordering;
-};
-
-/*
- * A filter that can be used with `filterNodes` to get a graph that only
- * includes nodes that do not contain others nodes.
- */
-exports.filterNonSubgraphs = function(g) {
-  return function(u) {
-    return g.children(u).length === 0;
-  };
-};
-
-/*
- * Returns a new function that wraps `func` with a timer. The wrapper logs the
- * time it takes to execute the function.
- *
- * The timer will be enabled provided `log.level >= 1`.
- */
-function time(name, func) {
-  return function() {
-    var start = new Date().getTime();
-    try {
-      return func.apply(null, arguments);
-    } finally {
-      log(1, name + ' time: ' + (new Date().getTime() - start) + 'ms');
-    }
-  };
-}
-time.enabled = false;
-
-exports.time = time;
-
-/*
- * A global logger with the specification `log(level, message, ...)` that
- * will log a message to the console if `log.level >= level`.
- */
-function log(level) {
-  if (log.level >= level) {
-    console.log.apply(console, Array.prototype.slice.call(arguments, 1));
-  }
-}
-log.level = 0;
-
-exports.log = log;
-
-},{}],28:[function(require,module,exports){
-module.exports = '0.4.6';
-
-},{}],29:[function(require,module,exports){
-exports.Graph = require("./lib/Graph");
-exports.Digraph = require("./lib/Digraph");
-exports.CGraph = require("./lib/CGraph");
-exports.CDigraph = require("./lib/CDigraph");
-require("./lib/graph-converters");
-
-exports.alg = {
-  isAcyclic: require("./lib/alg/isAcyclic"),
-  components: require("./lib/alg/components"),
-  dijkstra: require("./lib/alg/dijkstra"),
-  dijkstraAll: require("./lib/alg/dijkstraAll"),
-  findCycles: require("./lib/alg/findCycles"),
-  floydWarshall: require("./lib/alg/floydWarshall"),
-  postorder: require("./lib/alg/postorder"),
-  preorder: require("./lib/alg/preorder"),
-  prim: require("./lib/alg/prim"),
-  tarjan: require("./lib/alg/tarjan"),
-  topsort: require("./lib/alg/topsort")
-};
-
-exports.converter = {
-  json: require("./lib/converter/json.js")
-};
-
-var filter = require("./lib/filter");
-exports.filter = {
-  all: filter.all,
-  nodesFromList: filter.nodesFromList
-};
-
-exports.version = require("./lib/version");
-
-},{"./lib/CDigraph":31,"./lib/CGraph":32,"./lib/Digraph":33,"./lib/Graph":34,"./lib/alg/components":35,"./lib/alg/dijkstra":36,"./lib/alg/dijkstraAll":37,"./lib/alg/findCycles":38,"./lib/alg/floydWarshall":39,"./lib/alg/isAcyclic":40,"./lib/alg/postorder":41,"./lib/alg/preorder":42,"./lib/alg/prim":43,"./lib/alg/tarjan":44,"./lib/alg/topsort":45,"./lib/converter/json.js":47,"./lib/filter":48,"./lib/graph-converters":49,"./lib/version":51}],30:[function(require,module,exports){
-/* jshint -W079 */
-var Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = BaseGraph;
-
-function BaseGraph() {
-  // The value assigned to the graph itself.
-  this._value = undefined;
-
-  // Map of node id -> { id, value }
-  this._nodes = {};
-
-  // Map of edge id -> { id, u, v, value }
-  this._edges = {};
-
-  // Used to generate a unique id in the graph
-  this._nextId = 0;
-}
-
-// Number of nodes
-BaseGraph.prototype.order = function() {
-  return Object.keys(this._nodes).length;
-};
-
-// Number of edges
-BaseGraph.prototype.size = function() {
-  return Object.keys(this._edges).length;
-};
-
-// Accessor for graph level value
-BaseGraph.prototype.graph = function(value) {
-  if (arguments.length === 0) {
-    return this._value;
-  }
-  this._value = value;
-};
-
-BaseGraph.prototype.hasNode = function(u) {
-  return u in this._nodes;
-};
-
-BaseGraph.prototype.node = function(u, value) {
-  var node = this._strictGetNode(u);
-  if (arguments.length === 1) {
-    return node.value;
-  }
-  node.value = value;
-};
-
-BaseGraph.prototype.nodes = function() {
-  var nodes = [];
-  this.eachNode(function(id) { nodes.push(id); });
-  return nodes;
-};
-
-BaseGraph.prototype.eachNode = function(func) {
-  for (var k in this._nodes) {
-    var node = this._nodes[k];
-    func(node.id, node.value);
-  }
-};
-
-BaseGraph.prototype.hasEdge = function(e) {
-  return e in this._edges;
-};
-
-BaseGraph.prototype.edge = function(e, value) {
-  var edge = this._strictGetEdge(e);
-  if (arguments.length === 1) {
-    return edge.value;
-  }
-  edge.value = value;
-};
-
-BaseGraph.prototype.edges = function() {
-  var es = [];
-  this.eachEdge(function(id) { es.push(id); });
-  return es;
-};
-
-BaseGraph.prototype.eachEdge = function(func) {
-  for (var k in this._edges) {
-    var edge = this._edges[k];
-    func(edge.id, edge.u, edge.v, edge.value);
-  }
-};
-
-BaseGraph.prototype.incidentNodes = function(e) {
-  var edge = this._strictGetEdge(e);
-  return [edge.u, edge.v];
-};
-
-BaseGraph.prototype.addNode = function(u, value) {
-  if (u === undefined || u === null) {
-    do {
-      u = "_" + (++this._nextId);
-    } while (this.hasNode(u));
-  } else if (this.hasNode(u)) {
-    throw new Error("Graph already has node '" + u + "'");
-  }
-  this._nodes[u] = { id: u, value: value };
-  return u;
-};
-
-BaseGraph.prototype.delNode = function(u) {
-  this._strictGetNode(u);
-  this.incidentEdges(u).forEach(function(e) { this.delEdge(e); }, this);
-  delete this._nodes[u];
-};
-
-// inMap and outMap are opposite sides of an incidence map. For example, for
-// Graph these would both come from the _incidentEdges map, while for Digraph
-// they would come from _inEdges and _outEdges.
-BaseGraph.prototype._addEdge = function(e, u, v, value, inMap, outMap) {
-  this._strictGetNode(u);
-  this._strictGetNode(v);
-
-  if (e === undefined || e === null) {
-    do {
-      e = "_" + (++this._nextId);
-    } while (this.hasEdge(e));
-  }
-  else if (this.hasEdge(e)) {
-    throw new Error("Graph already has edge '" + e + "'");
-  }
-
-  this._edges[e] = { id: e, u: u, v: v, value: value };
-  addEdgeToMap(inMap[v], u, e);
-  addEdgeToMap(outMap[u], v, e);
-
-  return e;
-};
-
-// See note for _addEdge regarding inMap and outMap.
-BaseGraph.prototype._delEdge = function(e, inMap, outMap) {
-  var edge = this._strictGetEdge(e);
-  delEdgeFromMap(inMap[edge.v], edge.u, e);
-  delEdgeFromMap(outMap[edge.u], edge.v, e);
-  delete this._edges[e];
-};
-
-BaseGraph.prototype.copy = function() {
-  var copy = new this.constructor();
-  copy.graph(this.graph());
-  this.eachNode(function(u, value) { copy.addNode(u, value); });
-  this.eachEdge(function(e, u, v, value) { copy.addEdge(e, u, v, value); });
-  copy._nextId = this._nextId;
-  return copy;
-};
-
-BaseGraph.prototype.filterNodes = function(filter) {
-  var copy = new this.constructor();
-  copy.graph(this.graph());
-  this.eachNode(function(u, value) {
-    if (filter(u)) {
-      copy.addNode(u, value);
-    }
-  });
-  this.eachEdge(function(e, u, v, value) {
-    if (copy.hasNode(u) && copy.hasNode(v)) {
-      copy.addEdge(e, u, v, value);
-    }
-  });
-  return copy;
-};
-
-BaseGraph.prototype._strictGetNode = function(u) {
-  var node = this._nodes[u];
-  if (node === undefined) {
-    throw new Error("Node '" + u + "' is not in graph");
-  }
-  return node;
-};
-
-BaseGraph.prototype._strictGetEdge = function(e) {
-  var edge = this._edges[e];
-  if (edge === undefined) {
-    throw new Error("Edge '" + e + "' is not in graph");
-  }
-  return edge;
-};
-
-function addEdgeToMap(map, v, e) {
-  (map[v] || (map[v] = new Set())).add(e);
-}
-
-function delEdgeFromMap(map, v, e) {
-  var vEntry = map[v];
-  vEntry.remove(e);
-  if (vEntry.size() === 0) {
-    delete map[v];
-  }
-}
-
-
-},{"cp-data":5}],31:[function(require,module,exports){
-var Digraph = require("./Digraph"),
-    compoundify = require("./compoundify");
-
-var CDigraph = compoundify(Digraph);
-
-module.exports = CDigraph;
-
-CDigraph.fromDigraph = function(src) {
-  var g = new CDigraph(),
-      graphValue = src.graph();
-
-  if (graphValue !== undefined) {
-    g.graph(graphValue);
-  }
-
-  src.eachNode(function(u, value) {
-    if (value === undefined) {
-      g.addNode(u);
-    } else {
-      g.addNode(u, value);
-    }
-  });
-  src.eachEdge(function(e, u, v, value) {
-    if (value === undefined) {
-      g.addEdge(null, u, v);
-    } else {
-      g.addEdge(null, u, v, value);
-    }
-  });
-  return g;
-};
-
-CDigraph.prototype.toString = function() {
-  return "CDigraph " + JSON.stringify(this, null, 2);
-};
-
-},{"./Digraph":33,"./compoundify":46}],32:[function(require,module,exports){
-var Graph = require("./Graph"),
-    compoundify = require("./compoundify");
-
-var CGraph = compoundify(Graph);
-
-module.exports = CGraph;
-
-CGraph.fromGraph = function(src) {
-  var g = new CGraph(),
-      graphValue = src.graph();
-
-  if (graphValue !== undefined) {
-    g.graph(graphValue);
-  }
-
-  src.eachNode(function(u, value) {
-    if (value === undefined) {
-      g.addNode(u);
-    } else {
-      g.addNode(u, value);
-    }
-  });
-  src.eachEdge(function(e, u, v, value) {
-    if (value === undefined) {
-      g.addEdge(null, u, v);
-    } else {
-      g.addEdge(null, u, v, value);
-    }
-  });
-  return g;
-};
-
-CGraph.prototype.toString = function() {
-  return "CGraph " + JSON.stringify(this, null, 2);
-};
-
-},{"./Graph":34,"./compoundify":46}],33:[function(require,module,exports){
-/*
- * This file is organized with in the following order:
- *
- * Exports
- * Graph constructors
- * Graph queries (e.g. nodes(), edges()
- * Graph mutators
- * Helper functions
- */
-
-var util = require("./util"),
-    BaseGraph = require("./BaseGraph"),
-/* jshint -W079 */
-    Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = Digraph;
-
-/*
- * Constructor to create a new directed multi-graph.
- */
-function Digraph() {
-  BaseGraph.call(this);
-
-  /*! Map of sourceId -> {targetId -> Set of edge ids} */
-  this._inEdges = {};
-
-  /*! Map of targetId -> {sourceId -> Set of edge ids} */
-  this._outEdges = {};
-}
-
-Digraph.prototype = new BaseGraph();
-Digraph.prototype.constructor = Digraph;
-
-/*
- * Always returns `true`.
- */
-Digraph.prototype.isDirected = function() {
-  return true;
-};
-
-/*
- * Returns all successors of the node with the id `u`. That is, all nodes
- * that have the node `u` as their source are returned.
- *
- * If no node `u` exists in the graph this function throws an Error.
- *
- * @param {String} u a node id
- */
-Digraph.prototype.successors = function(u) {
-  this._strictGetNode(u);
-  return Object.keys(this._outEdges[u])
-               .map(function(v) { return this._nodes[v].id; }, this);
-};
-
-/*
- * Returns all predecessors of the node with the id `u`. That is, all nodes
- * that have the node `u` as their target are returned.
- *
- * If no node `u` exists in the graph this function throws an Error.
- *
- * @param {String} u a node id
- */
-Digraph.prototype.predecessors = function(u) {
-  this._strictGetNode(u);
-  return Object.keys(this._inEdges[u])
-               .map(function(v) { return this._nodes[v].id; }, this);
-};
-
-/*
- * Returns all nodes that are adjacent to the node with the id `u`. In other
- * words, this function returns the set of all successors and predecessors of
- * node `u`.
- *
- * @param {String} u a node id
- */
-Digraph.prototype.neighbors = function(u) {
-  return Set.union([this.successors(u), this.predecessors(u)]).keys();
-};
-
-/*
- * Returns all nodes in the graph that have no in-edges.
- */
-Digraph.prototype.sources = function() {
-  var self = this;
-  return this._filterNodes(function(u) {
-    // This could have better space characteristics if we had an inDegree function.
-    return self.inEdges(u).length === 0;
-  });
-};
-
-/*
- * Returns all nodes in the graph that have no out-edges.
- */
-Digraph.prototype.sinks = function() {
-  var self = this;
-  return this._filterNodes(function(u) {
-    // This could have better space characteristics if we have an outDegree function.
-    return self.outEdges(u).length === 0;
-  });
-};
-
-/*
- * Returns the source node incident on the edge identified by the id `e`. If no
- * such edge exists in the graph this function throws an Error.
- *
- * @param {String} e an edge id
- */
-Digraph.prototype.source = function(e) {
-  return this._strictGetEdge(e).u;
-};
-
-/*
- * Returns the target node incident on the edge identified by the id `e`. If no
- * such edge exists in the graph this function throws an Error.
- *
- * @param {String} e an edge id
- */
-Digraph.prototype.target = function(e) {
-  return this._strictGetEdge(e).v;
-};
-
-/*
- * Returns an array of ids for all edges in the graph that have the node
- * `target` as their target. If the node `target` is not in the graph this
- * function raises an Error.
- *
- * Optionally a `source` node can also be specified. This causes the results
- * to be filtered such that only edges from `source` to `target` are included.
- * If the node `source` is specified but is not in the graph then this function
- * raises an Error.
- *
- * @param {String} target the target node id
- * @param {String} [source] an optional source node id
- */
-Digraph.prototype.inEdges = function(target, source) {
-  this._strictGetNode(target);
-  var results = Set.union(util.values(this._inEdges[target])).keys();
-  if (arguments.length > 1) {
-    this._strictGetNode(source);
-    results = results.filter(function(e) { return this.source(e) === source; }, this);
-  }
-  return results;
-};
-
-/*
- * Returns an array of ids for all edges in the graph that have the node
- * `source` as their source. If the node `source` is not in the graph this
- * function raises an Error.
- *
- * Optionally a `target` node may also be specified. This causes the results
- * to be filtered such that only edges from `source` to `target` are included.
- * If the node `target` is specified but is not in the graph then this function
- * raises an Error.
- *
- * @param {String} source the source node id
- * @param {String} [target] an optional target node id
- */
-Digraph.prototype.outEdges = function(source, target) {
-  this._strictGetNode(source);
-  var results = Set.union(util.values(this._outEdges[source])).keys();
-  if (arguments.length > 1) {
-    this._strictGetNode(target);
-    results = results.filter(function(e) { return this.target(e) === target; }, this);
-  }
-  return results;
-};
-
-/*
- * Returns an array of ids for all edges in the graph that have the `u` as
- * their source or their target. If the node `u` is not in the graph this
- * function raises an Error.
- *
- * Optionally a `v` node may also be specified. This causes the results to be
- * filtered such that only edges between `u` and `v` - in either direction -
- * are included. IF the node `v` is specified but not in the graph then this
- * function raises an Error.
- *
- * @param {String} u the node for which to find incident edges
- * @param {String} [v] option node that must be adjacent to `u`
- */
-Digraph.prototype.incidentEdges = function(u, v) {
-  if (arguments.length > 1) {
-    return Set.union([this.outEdges(u, v), this.outEdges(v, u)]).keys();
-  } else {
-    return Set.union([this.inEdges(u), this.outEdges(u)]).keys();
-  }
-};
-
-/*
- * Returns a string representation of this graph.
- */
-Digraph.prototype.toString = function() {
-  return "Digraph " + JSON.stringify(this, null, 2);
-};
-
-/*
- * Adds a new node with the id `u` to the graph and assigns it the value
- * `value`. If a node with the id is already a part of the graph this function
- * throws an Error.
- *
- * @param {String} u a node id
- * @param {Object} [value] an optional value to attach to the node
- */
-Digraph.prototype.addNode = function(u, value) {
-  u = BaseGraph.prototype.addNode.call(this, u, value);
-  this._inEdges[u] = {};
-  this._outEdges[u] = {};
-  return u;
-};
-
-/*
- * Removes a node from the graph that has the id `u`. Any edges incident on the
- * node are also removed. If the graph does not contain a node with the id this
- * function will throw an Error.
- *
- * @param {String} u a node id
- */
-Digraph.prototype.delNode = function(u) {
-  BaseGraph.prototype.delNode.call(this, u);
-  delete this._inEdges[u];
-  delete this._outEdges[u];
-};
-
-/*
- * Adds a new edge to the graph with the id `e` from a node with the id `source`
- * to a node with an id `target` and assigns it the value `value`. This graph
- * allows more than one edge from `source` to `target` as long as the id `e`
- * is unique in the set of edges. If `e` is `null` the graph will assign a
- * unique identifier to the edge.
- *
- * If `source` or `target` are not present in the graph this function will
- * throw an Error.
- *
- * @param {String} [e] an edge id
- * @param {String} source the source node id
- * @param {String} target the target node id
- * @param {Object} [value] an optional value to attach to the edge
- */
-Digraph.prototype.addEdge = function(e, source, target, value) {
-  return BaseGraph.prototype._addEdge.call(this, e, source, target, value,
-                                           this._inEdges, this._outEdges);
-};
-
-/*
- * Removes an edge in the graph with the id `e`. If no edge in the graph has
- * the id `e` this function will throw an Error.
- *
- * @param {String} e an edge id
- */
-Digraph.prototype.delEdge = function(e) {
-  BaseGraph.prototype._delEdge.call(this, e, this._inEdges, this._outEdges);
-};
-
-// Unlike BaseGraph.filterNodes, this helper just returns nodes that
-// satisfy a predicate.
-Digraph.prototype._filterNodes = function(pred) {
-  var filtered = [];
-  this.eachNode(function(u) {
-    if (pred(u)) {
-      filtered.push(u);
-    }
-  });
-  return filtered;
-};
-
-
-},{"./BaseGraph":30,"./util":50,"cp-data":5}],34:[function(require,module,exports){
-/*
- * This file is organized with in the following order:
- *
- * Exports
- * Graph constructors
- * Graph queries (e.g. nodes(), edges()
- * Graph mutators
- * Helper functions
- */
-
-var util = require("./util"),
-    BaseGraph = require("./BaseGraph"),
-/* jshint -W079 */
-    Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = Graph;
-
-/*
- * Constructor to create a new undirected multi-graph.
- */
-function Graph() {
-  BaseGraph.call(this);
-
-  /*! Map of nodeId -> { otherNodeId -> Set of edge ids } */
-  this._incidentEdges = {};
-}
-
-Graph.prototype = new BaseGraph();
-Graph.prototype.constructor = Graph;
-
-/*
- * Always returns `false`.
- */
-Graph.prototype.isDirected = function() {
-  return false;
-};
-
-/*
- * Returns all nodes that are adjacent to the node with the id `u`.
- *
- * @param {String} u a node id
- */
-Graph.prototype.neighbors = function(u) {
-  this._strictGetNode(u);
-  return Object.keys(this._incidentEdges[u])
-               .map(function(v) { return this._nodes[v].id; }, this);
-};
-
-/*
- * Returns an array of ids for all edges in the graph that are incident on `u`.
- * If the node `u` is not in the graph this function raises an Error.
- *
- * Optionally a `v` node may also be specified. This causes the results to be
- * filtered such that only edges between `u` and `v` are included. If the node
- * `v` is specified but not in the graph then this function raises an Error.
- *
- * @param {String} u the node for which to find incident edges
- * @param {String} [v] option node that must be adjacent to `u`
- */
-Graph.prototype.incidentEdges = function(u, v) {
-  this._strictGetNode(u);
-  if (arguments.length > 1) {
-    this._strictGetNode(v);
-    return v in this._incidentEdges[u] ? this._incidentEdges[u][v].keys() : [];
-  } else {
-    return Set.union(util.values(this._incidentEdges[u])).keys();
-  }
-};
-
-/*
- * Returns a string representation of this graph.
- */
-Graph.prototype.toString = function() {
-  return "Graph " + JSON.stringify(this, null, 2);
-};
-
-/*
- * Adds a new node with the id `u` to the graph and assigns it the value
- * `value`. If a node with the id is already a part of the graph this function
- * throws an Error.
- *
- * @param {String} u a node id
- * @param {Object} [value] an optional value to attach to the node
- */
-Graph.prototype.addNode = function(u, value) {
-  u = BaseGraph.prototype.addNode.call(this, u, value);
-  this._incidentEdges[u] = {};
-  return u;
-};
-
-/*
- * Removes a node from the graph that has the id `u`. Any edges incident on the
- * node are also removed. If the graph does not contain a node with the id this
- * function will throw an Error.
- *
- * @param {String} u a node id
- */
-Graph.prototype.delNode = function(u) {
-  BaseGraph.prototype.delNode.call(this, u);
-  delete this._incidentEdges[u];
-};
-
-/*
- * Adds a new edge to the graph with the id `e` between a node with the id `u`
- * and a node with an id `v` and assigns it the value `value`. This graph
- * allows more than one edge between `u` and `v` as long as the id `e`
- * is unique in the set of edges. If `e` is `null` the graph will assign a
- * unique identifier to the edge.
- *
- * If `u` or `v` are not present in the graph this function will throw an
- * Error.
- *
- * @param {String} [e] an edge id
- * @param {String} u the node id of one of the adjacent nodes
- * @param {String} v the node id of the other adjacent node
- * @param {Object} [value] an optional value to attach to the edge
- */
-Graph.prototype.addEdge = function(e, u, v, value) {
-  return BaseGraph.prototype._addEdge.call(this, e, u, v, value,
-                                           this._incidentEdges, this._incidentEdges);
-};
-
-/*
- * Removes an edge in the graph with the id `e`. If no edge in the graph has
- * the id `e` this function will throw an Error.
- *
- * @param {String} e an edge id
- */
-Graph.prototype.delEdge = function(e) {
-  BaseGraph.prototype._delEdge.call(this, e, this._incidentEdges, this._incidentEdges);
-};
-
-
-},{"./BaseGraph":30,"./util":50,"cp-data":5}],35:[function(require,module,exports){
-/* jshint -W079 */
-var Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = components;
-
-/**
- * Finds all [connected components][] in a graph and returns an array of these
- * components. Each component is itself an array that contains the ids of nodes
- * in the component.
- *
- * This function only works with undirected Graphs.
- *
- * [connected components]: http://en.wikipedia.org/wiki/Connected_component_(graph_theory)
- *
- * @param {Graph} g the graph to search for components
- */
-function components(g) {
-  var results = [];
-  var visited = new Set();
-
-  function dfs(v, component) {
-    if (!visited.has(v)) {
-      visited.add(v);
-      component.push(v);
-      g.neighbors(v).forEach(function(w) {
-        dfs(w, component);
-      });
-    }
-  }
-
-  g.nodes().forEach(function(v) {
-    var component = [];
-    dfs(v, component);
-    if (component.length > 0) {
-      results.push(component);
-    }
-  });
-
-  return results;
-}
-
-},{"cp-data":5}],36:[function(require,module,exports){
-var PriorityQueue = require("cp-data").PriorityQueue;
-
-module.exports = dijkstra;
-
-/**
- * This function is an implementation of [Dijkstra's algorithm][] which finds
- * the shortest path from **source** to all other nodes in **g**. This
- * function returns a map of `u -> { distance, predecessor }`. The distance
- * property holds the sum of the weights from **source** to `u` along the
- * shortest path or `Number.POSITIVE_INFINITY` if there is no path from
- * **source**. The predecessor property can be used to walk the individual
- * elements of the path from **source** to **u** in reverse order.
- *
- * This function takes an optional `weightFunc(e)` which returns the
- * weight of the edge `e`. If no weightFunc is supplied then each edge is
- * assumed to have a weight of 1. This function throws an Error if any of
- * the traversed edges have a negative edge weight.
- *
- * This function takes an optional `incidentFunc(u)` which returns the ids of
- * all edges incident to the node `u` for the purposes of shortest path
- * traversal. By default this function uses the `g.outEdges` for Digraphs and
- * `g.incidentEdges` for Graphs.
- *
- * This function takes `O((|E| + |V|) * log |V|)` time.
- *
- * [Dijkstra's algorithm]: http://en.wikipedia.org/wiki/Dijkstra%27s_algorithm
- *
- * @param {Graph} g the graph to search for shortest paths from **source**
- * @param {Object} source the source from which to start the search
- * @param {Function} [weightFunc] optional weight function
- * @param {Function} [incidentFunc] optional incident function
- */
-function dijkstra(g, source, weightFunc, incidentFunc) {
-  var results = {},
-      pq = new PriorityQueue();
-
-  function updateNeighbors(e) {
-    var incidentNodes = g.incidentNodes(e),
-        v = incidentNodes[0] !== u ? incidentNodes[0] : incidentNodes[1],
-        vEntry = results[v],
-        weight = weightFunc(e),
-        distance = uEntry.distance + weight;
-
-    if (weight < 0) {
-      throw new Error("dijkstra does not allow negative edge weights. Bad edge: " + e + " Weight: " + weight);
-    }
-
-    if (distance < vEntry.distance) {
-      vEntry.distance = distance;
-      vEntry.predecessor = u;
-      pq.decrease(v, distance);
-    }
-  }
-
-  weightFunc = weightFunc || function() { return 1; };
-  incidentFunc = incidentFunc || (g.isDirected()
-      ? function(u) { return g.outEdges(u); }
-      : function(u) { return g.incidentEdges(u); });
-
-  g.eachNode(function(u) {
-    var distance = u === source ? 0 : Number.POSITIVE_INFINITY;
-    results[u] = { distance: distance };
-    pq.add(u, distance);
-  });
-
-  var u, uEntry;
-  while (pq.size() > 0) {
-    u = pq.removeMin();
-    uEntry = results[u];
-    if (uEntry.distance === Number.POSITIVE_INFINITY) {
-      break;
-    }
-
-    incidentFunc(u).forEach(updateNeighbors);
-  }
-
-  return results;
-}
-
-},{"cp-data":5}],37:[function(require,module,exports){
-var dijkstra = require("./dijkstra");
-
-module.exports = dijkstraAll;
-
-/**
- * This function finds the shortest path from each node to every other
- * reachable node in the graph. It is similar to [alg.dijkstra][], but
- * instead of returning a single-source array, it returns a mapping of
- * of `source -> alg.dijksta(g, source, weightFunc, incidentFunc)`.
- *
- * This function takes an optional `weightFunc(e)` which returns the
- * weight of the edge `e`. If no weightFunc is supplied then each edge is
- * assumed to have a weight of 1. This function throws an Error if any of
- * the traversed edges have a negative edge weight.
- *
- * This function takes an optional `incidentFunc(u)` which returns the ids of
- * all edges incident to the node `u` for the purposes of shortest path
- * traversal. By default this function uses the `outEdges` function on the
- * supplied graph.
- *
- * This function takes `O(|V| * (|E| + |V|) * log |V|)` time.
- *
- * [alg.dijkstra]: dijkstra.js.html#dijkstra
- *
- * @param {Graph} g the graph to search for shortest paths from **source**
- * @param {Function} [weightFunc] optional weight function
- * @param {Function} [incidentFunc] optional incident function
- */
-function dijkstraAll(g, weightFunc, incidentFunc) {
-  var results = {};
-  g.eachNode(function(u) {
-    results[u] = dijkstra(g, u, weightFunc, incidentFunc);
-  });
-  return results;
-}
-
-},{"./dijkstra":36}],38:[function(require,module,exports){
-var tarjan = require("./tarjan");
-
-module.exports = findCycles;
-
-/*
- * Given a Digraph **g** this function returns all nodes that are part of a
- * cycle. Since there may be more than one cycle in a graph this function
- * returns an array of these cycles, where each cycle is itself represented
- * by an array of ids for each node involved in that cycle.
- *
- * [alg.isAcyclic][] is more efficient if you only need to determine whether
- * a graph has a cycle or not.
- *
- * [alg.isAcyclic]: isAcyclic.js.html#isAcyclic
- *
- * @param {Digraph} g the graph to search for cycles.
- */
-function findCycles(g) {
-  return tarjan(g).filter(function(cmpt) { return cmpt.length > 1; });
-}
-
-},{"./tarjan":44}],39:[function(require,module,exports){
-module.exports = floydWarshall;
-
-/**
- * This function is an implementation of the [Floyd-Warshall algorithm][],
- * which finds the shortest path from each node to every other reachable node
- * in the graph. It is similar to [alg.dijkstraAll][], but it handles negative
- * edge weights and is more efficient for some types of graphs. This function
- * returns a map of `source -> { target -> { distance, predecessor }`. The
- * distance property holds the sum of the weights from `source` to `target`
- * along the shortest path of `Number.POSITIVE_INFINITY` if there is no path
- * from `source`. The predecessor property can be used to walk the individual
- * elements of the path from `source` to `target` in reverse order.
- *
- * This function takes an optional `weightFunc(e)` which returns the
- * weight of the edge `e`. If no weightFunc is supplied then each edge is
- * assumed to have a weight of 1.
- *
- * This function takes an optional `incidentFunc(u)` which returns the ids of
- * all edges incident to the node `u` for the purposes of shortest path
- * traversal. By default this function uses the `outEdges` function on the
- * supplied graph.
- *
- * This algorithm takes O(|V|^3) time.
- *
- * [Floyd-Warshall algorithm]: https://en.wikipedia.org/wiki/Floyd-Warshall_algorithm
- * [alg.dijkstraAll]: dijkstraAll.js.html#dijkstraAll
- *
- * @param {Graph} g the graph to search for shortest paths from **source**
- * @param {Function} [weightFunc] optional weight function
- * @param {Function} [incidentFunc] optional incident function
- */
-function floydWarshall(g, weightFunc, incidentFunc) {
-  var results = {},
-      nodes = g.nodes();
-
-  weightFunc = weightFunc || function() { return 1; };
-  incidentFunc = incidentFunc || (g.isDirected()
-      ? function(u) { return g.outEdges(u); }
-      : function(u) { return g.incidentEdges(u); });
-
-  nodes.forEach(function(u) {
-    results[u] = {};
-    results[u][u] = { distance: 0 };
-    nodes.forEach(function(v) {
-      if (u !== v) {
-        results[u][v] = { distance: Number.POSITIVE_INFINITY };
-      }
-    });
-    incidentFunc(u).forEach(function(e) {
-      var incidentNodes = g.incidentNodes(e),
-          v = incidentNodes[0] !== u ? incidentNodes[0] : incidentNodes[1],
-          d = weightFunc(e);
-      if (d < results[u][v].distance) {
-        results[u][v] = { distance: d, predecessor: u };
-      }
-    });
-  });
-
-  nodes.forEach(function(k) {
-    var rowK = results[k];
-    nodes.forEach(function(i) {
-      var rowI = results[i];
-      nodes.forEach(function(j) {
-        var ik = rowI[k];
-        var kj = rowK[j];
-        var ij = rowI[j];
-        var altDistance = ik.distance + kj.distance;
-        if (altDistance < ij.distance) {
-          ij.distance = altDistance;
-          ij.predecessor = kj.predecessor;
-        }
-      });
-    });
-  });
-
-  return results;
-}
-
-},{}],40:[function(require,module,exports){
-var topsort = require("./topsort");
-
-module.exports = isAcyclic;
-
-/*
- * Given a Digraph **g** this function returns `true` if the graph has no
- * cycles and returns `false` if it does. This algorithm returns as soon as it
- * detects the first cycle.
- *
- * Use [alg.findCycles][] if you need the actual list of cycles in a graph.
- *
- * [alg.findCycles]: findCycles.js.html#findCycles
- *
- * @param {Digraph} g the graph to test for cycles
- */
-function isAcyclic(g) {
-  try {
-    topsort(g);
-  } catch (e) {
-    if (e instanceof topsort.CycleException) return false;
-    throw e;
-  }
-  return true;
-}
-
-},{"./topsort":45}],41:[function(require,module,exports){
-/* jshint -W079 */
-var Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = postorder;
-
-// Postorder traversal of g, calling f for each visited node. Assumes the graph
-// is a tree.
-function postorder(g, root, f) {
-  var visited = new Set();
-  if (g.isDirected()) {
-    throw new Error("This function only works for undirected graphs");
-  }
-  function dfs(u, prev) {
-    if (visited.has(u)) {
-      throw new Error("The input graph is not a tree: " + g);
-    }
-    visited.add(u);
-    g.neighbors(u).forEach(function(v) {
-      if (v !== prev) dfs(v, u);
-    });
-    f(u);
-  }
-  dfs(root);
-}
-
-},{"cp-data":5}],42:[function(require,module,exports){
-/* jshint -W079 */
-var Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = preorder;
-
-// Preorder traversal of g, calling f for each visited node. Assumes the graph
-// is a tree.
-function preorder(g, root, f) {
-  var visited = new Set();
-  if (g.isDirected()) {
-    throw new Error("This function only works for undirected graphs");
-  }
-  function dfs(u, prev) {
-    if (visited.has(u)) {
-      throw new Error("The input graph is not a tree: " + g);
-    }
-    visited.add(u);
-    f(u);
-    g.neighbors(u).forEach(function(v) {
-      if (v !== prev) dfs(v, u);
-    });
-  }
-  dfs(root);
-}
-
-},{"cp-data":5}],43:[function(require,module,exports){
-var Graph = require("../Graph"),
-    PriorityQueue = require("cp-data").PriorityQueue;
-
-module.exports = prim;
-
-/**
- * [Prim's algorithm][] takes a connected undirected graph and generates a
- * [minimum spanning tree][]. This function returns the minimum spanning
- * tree as an undirected graph. This algorithm is derived from the description
- * in "Introduction to Algorithms", Third Edition, Cormen, et al., Pg 634.
- *
- * This function takes a `weightFunc(e)` which returns the weight of the edge
- * `e`. It throws an Error if the graph is not connected.
- *
- * This function takes `O(|E| log |V|)` time.
- *
- * [Prim's algorithm]: https://en.wikipedia.org/wiki/Prim's_algorithm
- * [minimum spanning tree]: https://en.wikipedia.org/wiki/Minimum_spanning_tree
- *
- * @param {Graph} g the graph used to generate the minimum spanning tree
- * @param {Function} weightFunc the weight function to use
- */
-function prim(g, weightFunc) {
-  var result = new Graph(),
-      parents = {},
-      pq = new PriorityQueue(),
-      u;
-
-  function updateNeighbors(e) {
-    var incidentNodes = g.incidentNodes(e),
-        v = incidentNodes[0] !== u ? incidentNodes[0] : incidentNodes[1],
-        pri = pq.priority(v);
-    if (pri !== undefined) {
-      var edgeWeight = weightFunc(e);
-      if (edgeWeight < pri) {
-        parents[v] = u;
-        pq.decrease(v, edgeWeight);
-      }
-    }
-  }
-
-  if (g.order() === 0) {
-    return result;
-  }
-
-  g.eachNode(function(u) {
-    pq.add(u, Number.POSITIVE_INFINITY);
-    result.addNode(u);
-  });
-
-  // Start from an arbitrary node
-  pq.decrease(g.nodes()[0], 0);
-
-  var init = false;
-  while (pq.size() > 0) {
-    u = pq.removeMin();
-    if (u in parents) {
-      result.addEdge(null, u, parents[u]);
-    } else if (init) {
-      throw new Error("Input graph is not connected: " + g);
-    } else {
-      init = true;
-    }
-
-    g.incidentEdges(u).forEach(updateNeighbors);
-  }
-
-  return result;
-}
-
-},{"../Graph":34,"cp-data":5}],44:[function(require,module,exports){
-module.exports = tarjan;
-
-/**
- * This function is an implementation of [Tarjan's algorithm][] which finds
- * all [strongly connected components][] in the directed graph **g**. Each
- * strongly connected component is composed of nodes that can reach all other
- * nodes in the component via directed edges. A strongly connected component
- * can consist of a single node if that node cannot both reach and be reached
- * by any other specific node in the graph. Components of more than one node
- * are guaranteed to have at least one cycle.
- *
- * This function returns an array of components. Each component is itself an
- * array that contains the ids of all nodes in the component.
- *
- * [Tarjan's algorithm]: http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm
- * [strongly connected components]: http://en.wikipedia.org/wiki/Strongly_connected_component
- *
- * @param {Digraph} g the graph to search for strongly connected components
- */
-function tarjan(g) {
-  if (!g.isDirected()) {
-    throw new Error("tarjan can only be applied to a directed graph. Bad input: " + g);
-  }
-
-  var index = 0,
-      stack = [],
-      visited = {}, // node id -> { onStack, lowlink, index }
-      results = [];
-
-  function dfs(u) {
-    var entry = visited[u] = {
-      onStack: true,
-      lowlink: index,
-      index: index++
-    };
-    stack.push(u);
-
-    g.successors(u).forEach(function(v) {
-      if (!(v in visited)) {
-        dfs(v);
-        entry.lowlink = Math.min(entry.lowlink, visited[v].lowlink);
-      } else if (visited[v].onStack) {
-        entry.lowlink = Math.min(entry.lowlink, visited[v].index);
-      }
-    });
-
-    if (entry.lowlink === entry.index) {
-      var cmpt = [],
-          v;
-      do {
-        v = stack.pop();
-        visited[v].onStack = false;
-        cmpt.push(v);
-      } while (u !== v);
-      results.push(cmpt);
-    }
-  }
-
-  g.nodes().forEach(function(u) {
-    if (!(u in visited)) {
-      dfs(u);
-    }
-  });
-
-  return results;
-}
-
-},{}],45:[function(require,module,exports){
-module.exports = topsort;
-topsort.CycleException = CycleException;
-
-/*
- * Given a graph **g**, this function returns an ordered list of nodes such
- * that for each edge `u -> v`, `u` appears before `v` in the list. If the
- * graph has a cycle it is impossible to generate such a list and
- * **CycleException** is thrown.
- *
- * See [topological sorting](https://en.wikipedia.org/wiki/Topological_sorting)
- * for more details about how this algorithm works.
- *
- * @param {Digraph} g the graph to sort
- */
-function topsort(g) {
-  if (!g.isDirected()) {
-    throw new Error("topsort can only be applied to a directed graph. Bad input: " + g);
-  }
-
-  var visited = {};
-  var stack = {};
-  var results = [];
-
-  function visit(node) {
-    if (node in stack) {
-      throw new CycleException();
-    }
-
-    if (!(node in visited)) {
-      stack[node] = true;
-      visited[node] = true;
-      g.predecessors(node).forEach(function(pred) {
-        visit(pred);
-      });
-      delete stack[node];
-      results.push(node);
-    }
-  }
-
-  var sinks = g.sinks();
-  if (g.order() !== 0 && sinks.length === 0) {
-    throw new CycleException();
-  }
-
-  g.sinks().forEach(function(sink) {
-    visit(sink);
-  });
-
-  return results;
-}
-
-function CycleException() {}
-
-CycleException.prototype.toString = function() {
-  return "Graph has at least one cycle";
-};
-
-},{}],46:[function(require,module,exports){
-// This file provides a helper function that mixes-in Dot behavior to an
-// existing graph prototype.
-
-/* jshint -W079 */
-var Set = require("cp-data").Set;
-/* jshint +W079 */
-
-module.exports = compoundify;
-
-// Extends the given SuperConstructor with the ability for nodes to contain
-// other nodes. A special node id `null` is used to indicate the root graph.
-function compoundify(SuperConstructor) {
-  function Constructor() {
-    SuperConstructor.call(this);
-
-    // Map of object id -> parent id (or null for root graph)
-    this._parents = {};
-
-    // Map of id (or null) -> children set
-    this._children = {};
-    this._children[null] = new Set();
-  }
-
-  Constructor.prototype = new SuperConstructor();
-  Constructor.prototype.constructor = Constructor;
-
-  Constructor.prototype.parent = function(u, parent) {
-    this._strictGetNode(u);
-
-    if (arguments.length < 2) {
-      return this._parents[u];
-    }
-
-    if (u === parent) {
-      throw new Error("Cannot make " + u + " a parent of itself");
-    }
-    if (parent !== null) {
-      this._strictGetNode(parent);
-    }
-
-    this._children[this._parents[u]].remove(u);
-    this._parents[u] = parent;
-    this._children[parent].add(u);
-  };
-
-  Constructor.prototype.children = function(u) {
-    if (u !== null) {
-      this._strictGetNode(u);
-    }
-    return this._children[u].keys();
-  };
-
-  Constructor.prototype.addNode = function(u, value) {
-    u = SuperConstructor.prototype.addNode.call(this, u, value);
-    this._parents[u] = null;
-    this._children[u] = new Set();
-    this._children[null].add(u);
-    return u;
-  };
-
-  Constructor.prototype.delNode = function(u) {
-    // Promote all children to the parent of the subgraph
-    var parent = this.parent(u);
-    this._children[u].keys().forEach(function(child) {
-      this.parent(child, parent);
-    }, this);
-
-    this._children[parent].remove(u);
-    delete this._parents[u];
-    delete this._children[u];
-
-    return SuperConstructor.prototype.delNode.call(this, u);
-  };
-
-  Constructor.prototype.copy = function() {
-    var copy = SuperConstructor.prototype.copy.call(this);
-    this.nodes().forEach(function(u) {
-      copy.parent(u, this.parent(u));
-    }, this);
-    return copy;
-  };
-
-  Constructor.prototype.filterNodes = function(filter) {
-    var self = this,
-        copy = SuperConstructor.prototype.filterNodes.call(this, filter);
-
-    var parents = {};
-    function findParent(u) {
-      var parent = self.parent(u);
-      if (parent === null || copy.hasNode(parent)) {
-        parents[u] = parent;
-        return parent;
-      } else if (parent in parents) {
-        return parents[parent];
-      } else {
-        return findParent(parent);
-      }
-    }
-
-    copy.eachNode(function(u) { copy.parent(u, findParent(u)); });
-
-    return copy;
-  };
-
-  return Constructor;
-}
-
-},{"cp-data":5}],47:[function(require,module,exports){
-var Graph = require("../Graph"),
-    Digraph = require("../Digraph"),
-    CGraph = require("../CGraph"),
-    CDigraph = require("../CDigraph");
-
-exports.decode = function(nodes, edges, Ctor) {
-  Ctor = Ctor || Digraph;
-
-  if (typeOf(nodes) !== "Array") {
-    throw new Error("nodes is not an Array");
-  }
-
-  if (typeOf(edges) !== "Array") {
-    throw new Error("edges is not an Array");
-  }
-
-  if (typeof Ctor === "string") {
-    switch(Ctor) {
-      case "graph": Ctor = Graph; break;
-      case "digraph": Ctor = Digraph; break;
-      case "cgraph": Ctor = CGraph; break;
-      case "cdigraph": Ctor = CDigraph; break;
-      default: throw new Error("Unrecognized graph type: " + Ctor);
-    }
-  }
-
-  var graph = new Ctor();
-
-  nodes.forEach(function(u) {
-    graph.addNode(u.id, u.value);
-  });
-
-  // If the graph is compound, set up children...
-  if (graph.parent) {
-    nodes.forEach(function(u) {
-      if (u.children) {
-        u.children.forEach(function(v) {
-          graph.parent(v, u.id);
-        });
-      }
-    });
-  }
-
-  edges.forEach(function(e) {
-    graph.addEdge(e.id, e.u, e.v, e.value);
-  });
-
-  return graph;
-};
-
-exports.encode = function(graph) {
-  var nodes = [];
-  var edges = [];
-
-  graph.eachNode(function(u, value) {
-    var node = {id: u, value: value};
-    if (graph.children) {
-      var children = graph.children(u);
-      if (children.length) {
-        node.children = children;
-      }
-    }
-    nodes.push(node);
-  });
-
-  graph.eachEdge(function(e, u, v, value) {
-    edges.push({id: e, u: u, v: v, value: value});
-  });
-
-  var type;
-  if (graph instanceof CDigraph) {
-    type = "cdigraph";
-  } else if (graph instanceof CGraph) {
-    type = "cgraph";
-  } else if (graph instanceof Digraph) {
-    type = "digraph";
-  } else if (graph instanceof Graph) {
-    type = "graph";
-  } else {
-    throw new Error("Couldn't determine type of graph: " + graph);
-  }
-
-  return { nodes: nodes, edges: edges, type: type };
-};
-
-function typeOf(obj) {
-  return Object.prototype.toString.call(obj).slice(8, -1);
-}
-
-},{"../CDigraph":31,"../CGraph":32,"../Digraph":33,"../Graph":34}],48:[function(require,module,exports){
-/* jshint -W079 */
-var Set = require("cp-data").Set;
-/* jshint +W079 */
-
-exports.all = function() {
-  return function() { return true; };
-};
-
-exports.nodesFromList = function(nodes) {
-  var set = new Set(nodes);
-  return function(u) {
-    return set.has(u);
-  };
-};
-
-},{"cp-data":5}],49:[function(require,module,exports){
-var Graph = require("./Graph"),
-    Digraph = require("./Digraph");
-
-// Side-effect based changes are lousy, but node doesn't seem to resolve the
-// requires cycle.
-
-/**
- * Returns a new directed graph using the nodes and edges from this graph. The
- * new graph will have the same nodes, but will have twice the number of edges:
- * each edge is split into two edges with opposite directions. Edge ids,
- * consequently, are not preserved by this transformation.
- */
-Graph.prototype.toDigraph =
-Graph.prototype.asDirected = function() {
-  var g = new Digraph();
-  this.eachNode(function(u, value) { g.addNode(u, value); });
-  this.eachEdge(function(e, u, v, value) {
-    g.addEdge(null, u, v, value);
-    g.addEdge(null, v, u, value);
-  });
-  return g;
-};
-
-/**
- * Returns a new undirected graph using the nodes and edges from this graph.
- * The new graph will have the same nodes, but the edges will be made
- * undirected. Edge ids are preserved in this transformation.
- */
-Digraph.prototype.toGraph =
-Digraph.prototype.asUndirected = function() {
-  var g = new Graph();
-  this.eachNode(function(u, value) { g.addNode(u, value); });
-  this.eachEdge(function(e, u, v, value) {
-    g.addEdge(e, u, v, value);
-  });
-  return g;
-};
-
-},{"./Digraph":33,"./Graph":34}],50:[function(require,module,exports){
-// Returns an array of all values for properties of **o**.
-exports.values = function(o) {
-  var ks = Object.keys(o),
-      len = ks.length,
-      result = new Array(len),
-      i;
-  for (i = 0; i < len; ++i) {
-    result[i] = o[ks[i]];
-  }
-  return result;
-};
-
-},{}],51:[function(require,module,exports){
-module.exports = '0.7.4';
-
-},{}]},{},[1])
-;
diff --git a/airflow/www_rbac/static/dagre-d3.min.js b/airflow/www_rbac/static/dagre-d3.min.js
deleted file mode 100644
index 8d9ec3a5e31f6..0000000000000
--- a/airflow/www_rbac/static/dagre-d3.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-(function a(b,c,d){function e(g,h){if(!c[g]){if(!b[g]){var j=typeof require=="function"&&require;if(!h&&j)return j(g,!0);if(f)return f(g,!0);throw new Error("Cannot find module '"+g+"'")}var k=c[g]={exports:{}};b[g][0].call(k.exports,function(a){var c=b[g][1][a];return e(c?c:a)},k,k.exports,a,b,c,d)}return c[g].exports}var f=typeof require=="function"&&require;for(var g=0;g<d.length;g++)e(d[g]);return e})({1:[function(a,b,c){var d=self;d.dagreD3=a("./index")},{"./index":2}],2:[function(a,b,c){b.exports={Digraph:a("graphlib").Digraph,Graph:a("graphlib").Graph,Renderer:a("./lib/Renderer"),json:a("graphlib").converter.json,layout:a("dagre").layout,version:a("./lib/version"),debug:a("dagre").debug}},{"./lib/Renderer":3,"./lib/version":4,dagre:11,graphlib:29}],3:[function(a,b,c){function g(){this._layout=d(),this.drawNodes(l),this.drawEdgeLabels(m),this.drawEdgePaths(n),this.positionNodes(o),this.positionEdgeLabels(p),this.positionEdgePaths(u),this.zoomSetup(w),this.zoom(x),this.transition(v),this.postLayout(y),this.postRender(z),this.edgeInterpolate("bundle"),this.edgeTension(.95)}function h(a){var b=a.copy();return b.graph()===undefined&&b.graph({}),"arrowheadFix"in b.graph()||(b.graph().arrowheadFix=!0),b.nodes().forEach(function(a){var c=i(b.node(a));b.node(a,c),"label"in c||(c.label="")}),b.edges().forEach(function(a){var c=i(b.edge(a));b.edge(a,c),"label"in c||(c.label="")}),b}function i(a){var b={};for(var c in a)b[c]=a[c];return b}function j(a,b){var c=a.getBBox();b.width=c.width,b.height=c.height}function k(a,b){var c=b.run(a);return a.eachNode(function(a,b){c.node(a).label=b.label}),a.eachEdge(function(a,b,d,e){c.edge(a).label=e.label}),c}function l(a,b){var c=a.nodes().filter(function(b){return!M(a,b)}),d=b.selectAll("g.node").classed("enter",!1).data(c,function(a){return a});return d.selectAll("*").remove(),d.enter().append("g").style("opacity",0).attr("class","node enter"),d.each(function(b){var c=a.node(b),d=e.select(this);B(c,d,!0,10,10)}),this._transition(d.exit()).style("opacity",0).remove(),d}function m(a,b){var c=b.selectAll("g.edgeLabel").classed("enter",!1).data(a.edges(),function(a){return a});return c.selectAll("*").remove(),c.enter().append("g").style("opacity",0).attr("class","edgeLabel enter"),c.each(function(b){B(a.edge(b),e.select(this),!1,0,0)}),this._transition(c.exit()).style("opacity",0).remove(),c}function o(a,b){function c(b){var c=a.node(b);return"translate("+c.x+","+c.y+")"}b.filter(".enter").attr("transform",c),this._transition(b).style("opacity",1).attr("transform",c)}function p(a,b){function c(b){var c=a.edge(b),d=F(c.points);return"translate("+d.x+","+d.y+")"}b.filter(".enter").attr("transform",c),this._transition(b).style("opacity",1).attr("transform",c)}function q(a){return Object.prototype.toString.call(a)==="[object SVGEllipseElement]"}function r(a){return Object.prototype.toString.call(a)==="[object SVGCircleElement]"}function s(a){return Object.prototype.toString.call(a)==="[object SVGPolygonElement]"}function t(a,b,c){if(a.useDef){var d=c.select("defs #"+a.useDef).node();if(d){var e=d.childNodes[0];if(r(e)||q(e))return H(a,e,b);if(s(e))return L(a,e,b)}}return G(a,b)}function u(a,b,c){function g(b){var g=a.edge(b),h=a.node(a.incidentNodes(b)[0]),i=a.node(a.incidentNodes(b)[1]),j=g.points.slice(),k=j.length===0?i:j[0],l=j.length===0?h:j[j.length-1];return j.unshift(t(h,k,c)),j.push(t(i,l,c)),e.svg.line().x(function(a){return a.x}).y(function(a){return a.y}).interpolate(d).tension(f)(j)}var d=this._edgeInterpolate,f=this._edgeTension;b.filter(".enter").selectAll("path").attr("d",g),this._transition(b.selectAll("path")).attr("d",g).style("opacity",1)}function v(a){return a}function w(a,b){var c=b.property("ownerSVGElement");return c?c=e.select(c):c=b,c.select("rect.overlay").empty()&&(c.insert("rect",":first-child").attr("class","overlay").attr("width","100%").attr("height","100%").style("fill","none").style("pointer-events","all"),b=b.append("g").attr("class","zoom"),this._zoom&&c.call(this._zoom(a,b))),b}function x(a,b){return e.behavior.zoom().on("zoom",function(){b.attr("transform","translate("+e.event.translate+")scale("+e.event.scale+")")})}function y(){}function z(a,b){a.isDirected()&&A(b,"arrowhead").attr("fill","#333")}function A(a,b){var c=a.select("#"+b);if(!c.empty())return c;var d=a.select("defs");d.empty()&&(d=a.append("svg:defs"));var e=d.append("svg:marker").attr("id",b).attr("viewBox","0 0 10 10").attr("refX",8).attr("refY",5).attr("markerUnits","strokeWidth").attr("markerWidth",8).attr("markerHeight",5).attr("orient","auto");return e.append("svg:path").attr("d","M 0 0 L 10 5 L 0 10 z"),e}function B(a,b,c,d,e){if(a.useDef){b.append("use").attr("xlink:href","#"+a.useDef);return}var f=a.label,g=b.append("rect");a.width&&g.attr("width",a.width),a.height&&g.attr("height",a.height);var h=b.append("g"),i;f[0]==="<"?(C(f,h),d=e=0):(i=D(f,h,Math.floor(a.labelCols),a.labelCut),O(a.labelStyle,i));var j=h.node().getBBox();h.attr("transform","translate("+ -j.width/2+","+ -j.height/2+")");var k=b.node().getBBox();g.attr("rx",a.rx?a.rx:5).attr("ry",a.ry?a.ry:5).attr("x",-(k.width/2+d)).attr("y",-(k.height/2+e)).attr("width",k.width+2*d).attr("height",k.height+2*e).attr("fill","#fff"),c&&(O(a.style,g),a.fill&&g.style("fill",a.fill),a.stroke&&g.style("stroke",a.stroke),a["stroke-width"]&&g.style("stroke-width",a["stroke-width"]+"px"),a["stroke-dasharray"]&&g.style("stroke-dasharray",a["stroke-dasharray"]),a.href&&b.attr("class",b.attr("class")+" clickable").on("click",function(){window.open(a.href)}))}function C(a,b){var c=b.append("foreignObject").attr("width","100000"),d,e;c.append("xhtml:div").style("float","left").html(function(){return a}).each(function(){d=this.clientWidth,e=this.clientHeight}),c.attr("width",d).attr("height",e)}function D(a,b,c,d){d===undefined&&(d="false"),d=d.toString().toLowerCase()==="true";var e=b.append("text").attr("text-anchor","left");a=a.replace(/\\n/g,"\n");var f=c?E(a,c,d):a;f=f.split("\n");for(var g=0;g<f.length;g++)e.append("tspan").attr("dy","1em").attr("x","1").text(f[g]);return e}function E(a,b,c,d){d=d||"\n",b=b||75,c=c||!1;if(!a)return a;var e=".{1,"+b+"}(\\s|$)"+(c?"|.{"+b+"}|.+$":"|\\S+?(\\s|$)");return a.match(new RegExp(e,"g")).join(d)}function F(a){var b=a.length/2;if(a.length%2)return a[Math.floor(b)];var c=a[b-1],d=a[b];return{x:(c.x+d.x)/2,y:(c.y+d.y)/2}}function G(a,b){var c=a.x,d=a.y,e=b.x-c,f=b.y-d,g=a.width/2,h=a.height/2,i,j;return Math.abs(f)*g>Math.abs(e)*h?(f<0&&(h=-h),i=f===0?0:h*e/f,j=h):(e<0&&(g=-g),i=g,j=e===0?0:g*f/e),{x:c+i,y:d+j}}function H(a,b,c){var d=a.x,e=a.y,f,g;r(b)?f=g=b.r.baseVal.value:(f=b.rx.baseVal.value,g=b.ry.baseVal.value);var h=d-c.x,i=e-c.y,j=Math.sqrt(f*f*i*i+g*g*h*h),k=Math.abs(f*g*h/j);c.x<d&&(k=-k);var l=Math.abs(f*g*i/j);return c.y<e&&(l=-l),{x:d+k,y:e+l}}function I(a,b){return a*b>0}function J(a,b,c){c.some(function(c){return c[0]===a&&c[1]===b})||c.push([a,b])}function K(a,b,c,d,e,f,g,h,i){var j,k,l,m,n,o,p,q,r,s,t,u,v,w,x;j=d-b,l=a-c,n=c*b-a*d,r=j*e+l*f+n,s=j*g+l*h+n;if(r!==0&&s!==0&&I(r,s))return;k=h-f,m=e-g,o=g*f-e*h,p=k*a+m*b+o,q=k*c+m*d+o;if(p!==0&&q!==0&&I(p,q))return;t=j*m-k*l;if(t===0)return;u=Math.abs(t/2),v=l*o-m*n,w=v<0?(v-u)/t:(v+u)/t,v=k*n-j*o,x=v<0?(v-u)/t:(v+u)/t,J(w,x,i)}function L(a,b,c){var d=a.x,e=a.y,f=c.x,g=c.y,h=[],i=b.points,j=1e5,k=1e5;for(var l=0;l<i.numberOfItems;l++){var m=i.getItem(l);j=Math.min(j,m.x),k=Math.min(k,m.y)}var n=d-a.width/2-j,o=e-a.height/2-k;for(var p=0;p<i.numberOfItems;p++){var q=i.getItem(p),r=i.getItem(p<i.numberOfItems-1?p+1:0);K(d,e,f,g,n+q.x,o+q.y,n+r.x,o+r.y,h)}return h.length===1?{x:h[0][0],y:h[0][1]}:h.length>1?(h.sort(function(a,b){var d=a[0]-c.x,e=a[1]-c.y,f=Math.sqrt(d*d+e*e),g=b[0]-c.x,h=b[1]-c.y,i=Math.sqrt(g*g+h*h);return f<i?-1:f===i?0:1}),{x:h[0][0],y:h[0][1]}):(console.log("NO INTERSECTION FOUND, RETURN NODE CENTER",a),a)}function M(a,b){return"children"in a&&a.children(b).length}function N(a,b){return a.bind?a.bind(b):function(){return a.apply(b,arguments)}}function O(a,b){if(a){var c=b.attr("style")||"";b.attr("style",c+"; "+a)}}var d=a("dagre").layout,e;try{e=a("d3")}catch(f){e=window.d3}b.exports=g,g.prototype.layout=function(a){return arguments.length?(this._layout=a,this):this._layout},g.prototype.drawNodes=function(a){return arguments.length?(this._drawNodes=N(a,this),this):this._drawNodes},g.prototype.drawEdgeLabels=function(a){return arguments.length?(this._drawEdgeLabels=N(a,this),this):this._drawEdgeLabels},g.prototype.drawEdgePaths=function(a){return arguments.length?(this._drawEdgePaths=N(a,this),this):this._drawEdgePaths},g.prototype.positionNodes=function(a){return arguments.length?(this._positionNodes=N(a,this),this):this._positionNodes},g.prototype.positionEdgeLabels=function(a){return arguments.length?(this._positionEdgeLabels=N(a,this),this):this._positionEdgeLabels},g.prototype.positionEdgePaths=function(a){return arguments.length?(this._positionEdgePaths=N(a,this),this):this._positionEdgePaths},g.prototype.transition=function(a){return arguments.length?(this._transition=N(a,this),this):this._transition},g.prototype.zoomSetup=function(a){return arguments.length?(this._zoomSetup=N(a,this),this):this._zoomSetup},g.prototype.zoom=function(a){return arguments.length?(a?this._zoom=N(a,this):delete this._zoom,this):this._zoom},g.prototype.postLayout=function(a){return arguments.length?(this._postLayout=N(a,this),this):this._postLayout},g.prototype.postRender=function(a){return arguments.length?(this._postRender=N(a,this),this):this._postRender},g.prototype.edgeInterpolate=function(a){return arguments.length?(this._edgeInterpolate=a,this):this._edgeInterpolate},g.prototype.edgeTension=function(a){return arguments.length?(this._edgeTension=a,this):this._edgeTension},g.prototype.run=function(a,b){a=h(a);var c=this._zoomSetup(a,b);c.selectAll("g.edgePaths, g.edgeLabels, g.nodes").data(["edgePaths","edgeLabels","nodes"]).enter().append("g").attr("class",function(a){return a});var d=this._drawNodes(a,c.select("g.nodes")),e=this._drawEdgeLabels(a,c.select("g.edgeLabels"));d.each(function(b){j(this,a.node(b))}),e.each(function(b){j(this,a.edge(b))});var f=k(a,this._layout);a.eachNode(function(a,b){b.useDef&&(f.node(a).useDef=b.useDef)}),this._postLayout(f,c);var g=this._drawEdgePaths(a,c.select("g.edgePaths"));return this._positionNodes(f,d),this._positionEdgeLabels(f,e),this._positionEdgePaths(f,g,b),this._postRender(f,c),f};var n=function(a,b){var c=b.selectAll("g.edgePath").classed("enter",!1).data(a.edges(),function(a){return a}),d="url(#arrowhead)",f=d;return a.isDirected()?a.graph().arrowheadFix!=="false"&&a.graph().arrowheadFix!==!1&&(f=function(){var a=e.select(this).style("stroke");if(a){var c="arrowhead-"+a.replace(/[^a-zA-Z0-9]/g,"_");return A(b,c).style("fill",a),"url(#"+c+")"}return d}):f=null,c.enter().append("g").attr("class","edgePath enter").append("path").style("opacity",0),c.selectAll("path").each(function(b){O(a.edge(b).style,e.select(this))}).attr("marker-end",f),this._transition(c.exit()).style("opacity",0).remove(),c}},{d3:10,dagre:11}],4:[function(a,b,c){b.exports="0.2.9"},{}],5:[function(a,b,c){c.Set=a("./lib/Set"),c.PriorityQueue=a("./lib/PriorityQueue"),c.version=a("./lib/version")},{"./lib/PriorityQueue":6,"./lib/Set":7,"./lib/version":9}],6:[function(a,b,c){function d(){this._arr=[],this._keyIndices={}}b.exports=d,d.prototype.size=function(){return this._arr.length},d.prototype.keys=function(){return this._arr.map(function(a){return a.key})},d.prototype.has=function(a){return a in this._keyIndices},d.prototype.priority=function(a){var b=this._keyIndices[a];if(b!==undefined)return this._arr[b].priority},d.prototype.min=function(){if(this.size()===0)throw new Error("Queue underflow");return this._arr[0].key},d.prototype.add=function(a,b){var c=this._keyIndices;if(a in c)return!1;var d=this._arr,e=d.length;return c[a]=e,d.push({key:a,priority:b}),this._decrease(e),!0},d.prototype.removeMin=function(){this._swap(0,this._arr.length-1);var a=this._arr.pop();return delete this._keyIndices[a.key],this._heapify(0),a.key},d.prototype.decrease=function(a,b){var c=this._keyIndices[a];if(b>this._arr[c].priority)throw new Error("New priority is greater than current priority. Key: "+a+" Old: "+this._arr[c].priority+" New: "+b);this._arr[c].priority=b,this._decrease(c)},d.prototype._heapify=function(a){var b=this._arr,c=2*a,d=c+1,e=a;c<b.length&&(e=b[c].priority<b[e].priority?c:e,d<b.length&&(e=b[d].priority<b[e].priority?d:e),e!==a&&(this._swap(a,e),this._heapify(e)))},d.prototype._decrease=function(a){var b=this._arr,c=b[a].priority,d;while(a!==0){d=a>>1;if(b[d].priority<c)break;this._swap(a,d),a=d}},d.prototype._swap=function(a,b){var c=this._arr,d=this._keyIndices,e=c[a],f=c[b];c[a]=f,c[b]=e,d[f.key]=a,d[e.key]=b}},{}],7:[function(a,b,c){function e(a){this._size=0,this._keys={};if(a)for(var b=0,c=a.length;b<c;++b)this.add(a[b])}function f(a){var b=Object.keys(a),c=b.length,d=new Array(c),e;for(e=0;e<c;++e)d[e]=a[b[e]];return d}var d=a("./util");b.exports=e,e.intersect=function(a){if(a.length===0)return new e;var b=new e(d.isArray(a[0])?a[0]:a[0].keys());for(var c=1,f=a.length;c<f;++c){var g=b.keys(),h=d.isArray(a[c])?new e(a[c]):a[c];for(var i=0,j=g.length;i<j;++i){var k=g[i];h.has(k)||b.remove(k)}}return b},e.union=function(a){var b=d.reduce(a,function(a,b){return a+(b.size?b.size():b.length)},0),c=new Array(b),f=0;for(var g=0,h=a.length;g<h;++g){var i=a[g],j=d.isArray(i)?i:i.keys();for(var k=0,l=j.length;k<l;++k)c[f++]=j[k]}return new e(c)},e.prototype.size=function(){return this._size},e.prototype.keys=function(){return f(this._keys)},e.prototype.has=function(a){return a in this._keys},e.prototype.add=function(a){return a in this._keys?!1:(this._keys[a]=a,++this._size,!0)},e.prototype.remove=function(a){return a in this._keys?(delete this._keys[a],--this._size,!0):!1}},{"./util":8}],8:[function(a,b,c){Array.isArray?c.isArray=Array.isArray:c.isArray=function(a){return Object.prototype.toString.call(a)==="[object Array]"},"function"!=typeof Array.prototype.reduce?c.reduce=function(a,b,c){"use strict";if(null===a||"undefined"==typeof a)throw new TypeError("Array.prototype.reduce called on null or undefined");if("function"!=typeof b)throw new TypeError(b+" is not a function");var d,e,f=a.length>>>0,g=!1;1<arguments.length&&(e=c,g=!0);for(d=0;f>d;++d)a.hasOwnProperty(d)&&(g?e=b(e,a[d],d,a):(e=a[d],g=!0));if(!g)throw new TypeError("Reduce of empty array with no initial value");return e}:c.reduce=function(a,b,c){return a.reduce(b,c)}},{}],9:[function(a,b,c){b.exports="1.1.3"},{}],10:[function(a,b,c){a("./d3"),b.exports=d3,function(){delete this.d3}()},{}],11:[function(a,b,c){c.Digraph=a("graphlib").Digraph,c.Graph=a("graphlib").Graph,c.layout=a("./lib/layout"),c.version=a("./lib/version"),c.debug=a("./lib/debug")},{"./lib/debug":12,"./lib/layout":13,"./lib/version":28,graphlib:29}],12:[function(a,b,c){"use strict";var d=a("./util");c.dotOrdering=function(a){function e(b){var d=a.children(b);d.length?(c+="subgraph cluster_"+b+" {",c+='label="'+b+'";',d.forEach(function(a){e(a)}),c+="}"):(c+=b,a.node(b).dummy&&(c+=" [shape=diamond]"),c+=";")}var b=d.ordering(a.filterNodes(d.filterNonSubgraphs(a))),c="digraph {";return a.children(null).forEach(e),b.forEach(function(a){c+='subgraph { rank=same; edge [style="invis"];',c+=a.join("->"),c+="}"}),a.eachEdge(function(a,b,d){c+=b+"->"+d+";"}),c+="}",c}},{"./util":27}],13:[function(a,b,c){"use strict";var d=a("./util"),e=a("./rank"),f=a("./order"),g=a("graphlib").CGraph,h=a("graphlib").CDigraph;b.exports=function(){function j(a){var c=new h;a.eachNode(function(a,b){b===undefined&&(b={}),c.addNode(a,{width:b.width,height:b.height}),b.hasOwnProperty("rank")&&(c.node(a).prefRank=b.rank)}),a.parent&&a.nodes().forEach(function(b){c.parent(b,a.parent(b))}),a.eachEdge(function(a,b,d,e){e===undefined&&(e={});var f={e:a,minLen:e.minLen||1,width:e.width||0,height:e.height||0,points:[]};c.addEdge(null,b,d,f)});var d=a.graph()||{};return c.graph({rankDir:d.rankDir||b.rankDir,orderRestarts:d.orderRestarts}),c}function k(a){var g=i.rankSep(),h;try{return h=d.time("initLayoutGraph",j)(a),h.order()===0?h:(h.eachEdge(function(a,b,c,d){d.minLen*=2}),i.rankSep(g/2),d.time("rank.run",e.run)(h,b.rankSimplex),d.time("normalize",l)(h),d.time("order",f)(h,b.orderMaxSweeps),d.time("position",c.run)(h),d.time("undoNormalize",m)(h),d.time("fixupEdgePoints",n)(h),d.time("rank.restoreEdges",e.restoreEdges)(h),d.time("createFinalGraph",o)(h,a.isDirected()))}finally{i.rankSep(g)}}function l(a){var b=0;a.eachEdge(function(c,d,e,f){var g=a.node(d).rank,h=a.node(e).rank;if(g+1<h){for(var i=d,j=g+1,k=0;j<h;++j,++k){var l="_D"+ ++b,m={width:f.width,height:f.height,edge:{id:c,source:d,target:e,attrs:f},rank:j,dummy:!0};k===0?m.index=0:j+1===h&&(m.index=1),a.addNode(l,m),a.addEdge(null,i,l,{}),i=l}a.addEdge(null,i,e,{}),a.delEdge(c)}})}function m(a){a.eachNode(function(b,c){if(c.dummy){if("index"in c){var d=c.edge;a.hasEdge(d.id)||a.addEdge(d.id,d.source,d.target,d.attrs);var e=a.edge(d.id).points;e[c.index]={x:c.x,y:c.y,ul:c.ul,ur:c.ur,dl:c.dl,dr:c.dr}}a.delNode(b)}})}function n(a){a.eachEdge(function(a,b,c,d){d.reversed&&d.points.reverse()})}function o(a,b){var c=b?new h:new g;c.graph(a.graph()),a.eachNode(function(a,b){c.addNode(a,b)}),a.eachNode(function(b){c.parent(b,a.parent(b))}),a.eachEdge(function(a,b,d,e){c.addEdge(e.e,b,d,e)});var d=0,e=0;return a.eachNode(function(b,c){a.children(b).length||(d=Math.max(d,c.x+c.width/2),e=Math.max(e,c.y+c.height/2))}),a.eachEdge(function(a,b,c,f){var g=Math.max.apply(Math,f.points.map(function(a){return a.x})),h=Math.max.apply(Math,f.points.map(function(a){return a.y}));d=Math.max(d,g+f.width/2),e=Math.max(e,h+f.height/2)}),c.graph().width=d,c.graph().height=e,c}function p(a){return function(){return arguments.length?(a.apply(null,arguments),i):a()}}var b={debugLevel:0,orderMaxSweeps:f.DEFAULT_MAX_SWEEPS,rankSimplex:!1,rankDir:"TB"},c=a("./position")(),i={};return i.orderIters=d.propertyAccessor(i,b,"orderMaxSweeps"),i.rankSimplex=d.propertyAccessor(i,b,"rankSimplex"),i.nodeSep=p(c.nodeSep),i.edgeSep=p(c.edgeSep),i.universalSep=p(c.universalSep),i.rankSep=p(c.rankSep),i.rankDir=d.propertyAccessor(i,b,"rankDir"),i.debugAlignment=p(c.debugAlignment),i.debugLevel=d.propertyAccessor(i,b,"debugLevel",function(a){d.log.level=a,c.debugLevel(a)}),i.run=d.time("Total layout",k),i._normalize=l,i}},{"./order":14,"./position":19,"./rank":20,"./util":27,graphlib:29}],14:[function(a,b,c){function j(a,b){function o(){a.eachNode(function(a,b){n[a]=b.order})}arguments.length<2&&(b=i);var c=a.graph().orderRestarts||0,h=f(a);h.forEach(function(b){b=b.filterNodes(function(b){return!a.children(b).length})});var j=0,k,l=Number.MAX_VALUE,n={};for(var p=0;p<Number(c)+1&&l!==0;++p){k=Number.MAX_VALUE,g(a,c>0),d.log(2,"Order phase start cross count: "+a.graph().orderInitCC);var q,r,s;for(q=0,r=0;r<4&&q<b&&k>0;++q,++r,++j)m(a,h,q),s=e(a),s<k&&(r=0,k=s,s<l&&(o(),l=s)),d.log(3,"Order phase start "+p+" iter "+q+" cross count: "+s)}Object.keys(n).forEach(function(b){if(!a.children||!a.children(b).length)a.node(b).order=n[b]}),a.graph().orderCC=l,d.log(2,"Order iterations: "+j),d.log(2,"Order phase best cross count: "+a.graph().orderCC)}function k(a,b){var c={};return b.forEach(function(b){c[b]=a.inEdges(b).map(function(b){return a.node(a.source(b)).order})}),c}function l(a,b){var c={};return b.forEach(function(b){c[b]=a.outEdges(b).map(function(b){return a.node(a.target(b)).order})}),c}function m(a,b,c){c%2===0?n(a,b,c):o(a,b,c)}function n(a,b){var c;for(var d=1;d<b.length;++d)c=h(b[d],c,k(a,b[d].nodes()))}function o(a,b){var c;for(var d=b.length-2;d>=0;--d)h(b[d],c,l(a,b[d].nodes()))}"use strict";var d=a("./util"),e=a("./order/crossCount"),f=a("./order/initLayerGraphs"),g=a("./order/initOrder"),h=a("./order/sortLayer");b.exports=j;var i=24;j.DEFAULT_MAX_SWEEPS=i},{"./order/crossCount":15,"./order/initLayerGraphs":16,"./order/initOrder":17,"./order/sortLayer":18,"./util":27}],15:[function(a,b,c){function e(a){var b=0,c=d.ordering(a);for(var e=1;e<c.length;++e)b+=f(a,c[e-1],c[e]);return b}function f(a,b,c){var d=[];b.forEach(function(b){var c=[];a.outEdges(b).forEach(function(b){c.push(a.node(a.target(b)).order)}),c.sort(function(a,b){return a-b}),d=d.concat(c)});var e=1;while(e<c.length)e<<=1;var f=2*e-1;e-=1;var g=[];for(var h=0;h<f;++h)g[h]=0;var i=0;return d.forEach(function(a){var b=a+e;++g[b];while(b>0)b%2&&(i+=g[b+1]),b=b-1>>1,++g[b]}),i}"use strict";var d=a("../util");b.exports=e},{"../util":27}],16:[function(a,b,c){function f(a){function c(d){if(d===null){a.children(d).forEach(function(a){c(a)});return}var f=a.node(d);f.minRank="rank"in f?f.rank:Number.MAX_VALUE,f.maxRank="rank"in f?f.rank:Number.MIN_VALUE;var h=new e;return a.children(d).forEach(function(b){var d=c(b);h=e.union([h,d]),f.minRank=Math.min(f.minRank,a.node(b).minRank),f.maxRank=Math.max(f.maxRank,a.node(b).maxRank)}),"rank"in f&&h.add(f.rank),h.keys().forEach(function(a){a in b||(b[a]=[]),b[a].push(d)}),h}var b=[];c(null);var f=[];return b.forEach(function(b,c){f[c]=a.filterNodes(d(b))}),f}"use strict";var d=a("graphlib").filter.nodesFromList,e=a("cp-data").Set;b.exports=f},{"cp-data":5,graphlib:29}],17:[function(a,b,c){function f(a,b){var c=[];a.eachNode(function(b,d){var e=c[d.rank];if(a.children&&a.children(b).length>0)return;e||(e=c[d.rank]=[]),e.push(b)}),c.forEach(function(c){b&&e.shuffle(c),c.forEach(function(b,c){a.node(b).order=c})});var f=d(a);a.graph().orderInitCC=f,a.graph().orderCC=Number.MAX_VALUE}"use strict";var d=a("./crossCount"),e=a("../util");b.exports=f},{"../util":27,"./crossCount":15}],18:[function(a,b,c){function h(a,b,c){c=n(a,c);var d=i(a,null,b,c);return d.list.forEach(function(b,c){a.node(b).order=c}),d.constraintGraph}function i(a,b,c,f){c=c?c.filterNodes(g(a.children(b))):new e;var h={};a.children(b).forEach(function(b){if(a.children(b).length)h[b]=i(a,b,c,f),h[b].firstSG=b,h[b].lastSG=b;else{var e=f[b];h[b]={degree:e.length,barycenter:d.sum(e)/e.length,order:a.node(b).order,orderCount:1,list:[b]}}}),l(a,c,h);var k=Object.keys(h);k.sort(function(a,b){return h[a].barycenter-h[b].barycenter||h[a].order-h[b].order});var m=k.map(function(a){return h[a]}).reduce(function(b,c){return j(a,b,c)});return m}function j(a,b,c){var d=k(b.constraintGraph,c.constraintGraph);return b.lastSG!==undefined&&c.firstSG!==undefined&&(d===undefined&&(d=new e),d.hasNode(b.lastSG)||d.addNode(b.lastSG),d.addNode(c.firstSG),d.addEdge(null,b.lastSG,c.firstSG)),{degree:b.degree+c.degree,barycenter:(b.barycenter*b.degree+c.barycenter*c.degree)/(b.degree+c.degree),order:(b.order*b.orderCount+c.order*c.orderCount)/(b.orderCount+c.orderCount),orderCount:b.orderCount+c.orderCount,list:b.list.concat(c.list),firstSG:b.firstSG!==undefined?b.firstSG:c.firstSG,lastSG:c.lastSG!==undefined?c.lastSG:b.lastSG,constraintGraph:d}}function k(a,b){return a===undefined?b:b===undefined?a:(a=a.copy(),b.nodes().forEach(function(b){a.addNode(b)}),b.edges().forEach(function(b,c,d){a.addEdge(null,c,d)}),a)}function l(a,b,c){function d(a,c,d){b.inEdges(a).forEach(function(a){b.delEdge(a),b.addEdge(null,b.source(a),d)}),b.outEdges(c).forEach(function(a){b.delEdge(a),b.addEdge(null,d,b.target(a))}),b.delNode(a),b.delNode(c)}var e;while((e=m(b,c))!==undefined){var f=b.source(e),g=b.target(e),h;while((h=b.addNode(null))&&a.hasNode(h))b.delNode(h);c[h]=j(a,c[f],c[g]),delete c[f],delete c[g],d(f,g,h),b.incidentEdges(h).length===0&&b.delNode(h)}}function m(a,b){var c=f(a);for(var d=0;d<c.length;++d){var e=c[d],g=a.inEdges(e);for(var h=0;h<g.length;++h){var i=g[h];if(b[a.source(i)].barycenter>=b[e].barycenter)return i}}}function n(a,b){var c=Number.MAX_VALUE,e=0,f={};a.eachNode(function(f){if(a.children(f).length)return;var g=b[f];g.length&&(c=Math.min(c,d.min(g)),e=Math.max(e,d.max(g)))});var g=e-c;return a.eachNode(function(d){if(a.children(d).length)return;var e=b[d];e.length?f[d]=e.map(function(b){return g?(b-c)*(a.order()-1)/g:a.order()-.5}):f[d]=[a.node(d).order]}),f}"use strict";var d=a("../util"),e=a("graphlib").Digraph,f=a("graphlib").alg.topsort,g=a("graphlib").filter.nodesFromList;b.exports=h},{"../util":27,graphlib:29}],19:[function(a,b,c){"use strict";var d=a("./util");b.exports=function(){function c(b){b=b.filterNodes(d.filterNonSubgraphs(b));var c=d.ordering(b),e=f(b,c),i={};["u","d"].forEach(function(d){d==="d"&&c.reverse(),["l","r"].forEach(function(f){f==="r"&&m(c);var j=d+f,k=g(b,c,e,d==="u"?"predecessors":"successors");i[j]=h(b,c,k.pos,k.root,k.align),a.debugLevel>=3&&t(d+f,b,c,i[j]),f==="r"&&l(i[j]),f==="r"&&m(c)}),d==="d"&&c.reverse()}),k(b,c,i),b.eachNode(function(a){var c=[];for(var d in i){var e=i[d][a];r(d,b,a,e),c.push(e)}c.sort(function(a,b){return a-b}),q(b,a,(c[1]+c[2])/2)});var j=0,p=b.graph().rankDir==="BT"||b.graph().rankDir==="RL";c.forEach(function(c){var e=d.max(c.map(function(a){return o(b,a)}));j+=e/2,c.forEach(function(a){s(b,a,p?-j:j)}),j+=e/2+a.rankSep});var u=d.min(b.nodes().map(function(a){return q(b,a)-n(b,a)/2})),v=d.min(b.nodes().map(function(a){return s(b,a)-o(b,a)/2}));b.eachNode(function(a){q(b,a,q(b,a)-u),s(b,a,s(b,a)-v)})}function e(a,b){return a<b?a.toString().length+":"+a+"-"+b:b.toString().length+":"+b+"-"+a}function f(a,b){function k(a){var b=d[a];if(b<h||b>j)c[e(g[i],a)]=!0}var c={},d={},f,g,h,i,j;if(b.length<=2)return c;b[1].forEach(function(a,b){d[a]=b});for(var l=1;l<b.length-1;++l){f=b[l],g=b[l+1],h=0,i=0;for(var m=0;m<g.length;++m){var n=g[m];d[n]=m,j=undefined;if(a.node(n).dummy){var o=a.predecessors(n)[0];o!==undefined&&a.node(o).dummy&&(j=d[o])}j===undefined&&m===g.length-1&&(j=f.length-1);if(j!==undefined){for(;i<=m;++i)a.predecessors(g[i]).forEach(k);h=j}}}return c}function g(a,b,c,d){var f={},g={},h={};return b.forEach(function(a){a.forEach(function(a,b){g[a]=a,h[a]=a,f[a]=b})}),b.forEach(function(b){var i=-1;b.forEach(function(b){var j=a[d](b),k;j.length>0&&(j.sort(function(a,b){return f[a]-f[b]}),k=(j.length-1)/2,j.slice(Math.floor(k),Math.ceil(k)+1).forEach(function(a){h[b]===b&&!c[e(a,b)]&&i<f[a]&&(h[a]=b,h[b]=g[b]=g[a],i=f[a])}))})}),{pos:f,root:g,align:h}}function h(a,b,c,e,f){function l(a,b,c){b in h[a]?h[a][b]=Math.min(h[a][b],c):h[a][b]=c}function m(b){if(!(b in k)){k[b]=0;var d=b;do{if(c[d]>0){var h=e[j[d]];m(h),g[b]===b&&(g[b]=g[h]);var i=p(a,j[d])+p(a,d);g[b]!==g[h]?l(g[h],g[b],k[b]-k[h]-i):k[b]=Math.max(k[b],k[h]+i)}d=f[d]}while(d!==b)}}var g={},h={},i={},j={},k={};return b.forEach(function(a){a.forEach(function(b,c){g[b]=b,h[b]={},c>0&&(j[b]=a[c-1])})}),d.values(e).forEach(function(a){m(a)}),b.forEach(function(a){a.forEach(function(a){k[a]=k[e[a]];if(a===e[a]&&a===g[a]){var b=0;a in h&&Object.keys(h[a]).length>0&&(b=d.min(Object.keys(h[a]).map(function(b){return h[a][b]+(b in i?i[b]:0)}))),i[a]=b}})}),b.forEach(function(a){a.forEach(function(a){k[a]+=i[g[e[a]]]||0})}),k}function i(a,b,c){return d.min(b.map(function(a){var b=a[0];return c[b]}))}function j(a,b,c){return d.max(b.map(function(a){var b=a[a.length-1];return c[b]}))}function k(a,b,c){function h(a){c[l][a]+=g[l]}var d={},e={},f,g={},k=Number.POSITIVE_INFINITY;for(var l in c){var m=c[l];d[l]=i(a,b,m),e[l]=j(a,b,m);var n=e[l]-d[l];n<k&&(k=n,f=l)}["u","d"].forEach(function(a){["l","r"].forEach(function(b){var c=a+b;g[c]=b==="l"?d[f]-d[c]:e[f]-e[c]})});for(l in c)a.eachNode(h)}function l(a){for(var b in a)a[b]=-a[b]}function m(a){a.forEach(function(a){a.reverse()})}function n(a,b){switch(a.graph().rankDir){case"LR":return a.node(b).height;case"RL":return a.node(b).height;default:return a.node(b).width}}function o(a,b){switch(a.graph().rankDir){case"LR":return a.node(b).width;case"RL":return a.node(b).width;default:return a.node(b).height}}function p(b,c){if(a.universalSep!==null)return a.universalSep;var d=n(b,c),e=b.node(c).dummy?a.edgeSep:a.nodeSep;return(d+e)/2}function q(a,b,c){if(a.graph().rankDir==="LR"||a.graph().rankDir==="RL"){if(arguments.length<3)return a.node(b).y;a.node(b).y=c}else{if(arguments.length<3)return a.node(b).x;a.node(b).x=c}}function r(a,b,c,d){if(b.graph().rankDir==="LR"||b.graph().rankDir==="RL"){if(arguments.length<3)return b.node(c)[a];b.node(c)[a]=d}else{if(arguments.length<3)return b.node(c)[a];b.node(c)[a]=d}}function s(a,b,c){if(a.graph().rankDir==="LR"||a.graph().rankDir==="RL"){if(arguments.length<3)return a.node(b).x;a.node(b).x=c}else{if(arguments.length<3)return a.node(b).y;a.node(b).y=c}}function t(a,b,c,d){c.forEach(function(c,e){var f,g;c.forEach(function(c){var h=d[c];if(f){var i=p(b,f)+p(b,c);h-g<i&&console.log("Position phase: sep violation. Align: "+a+". Layer: "+e+". "+"U: "+f+" V: "+c+". Actual sep: "+(h-g)+" Expected sep: "+i)}f=c,g=h})})}var a={nodeSep:50,edgeSep:10,universalSep:null,rankSep:30},b={};return b.nodeSep=d.propertyAccessor(b,a,"nodeSep"),b.edgeSep=d.propertyAccessor(b,a,"edgeSep"),b.universalSep=d.propertyAccessor(b,a,"universalSep"),b.rankSep=d.propertyAccessor(b,a,"rankSep"),b.debugLevel=d.propertyAccessor(b,a,"debugLevel"),b.run=c,b}},{"./util":27}],20:[function(a,b,c){function l(a,b){n(a),d.time("constraints.apply",h.apply)(a),o(a),d.time("acyclic",e)(a);var c=a.filterNodes(d.filterNonSubgraphs(a));f(c),j(c).forEach(function(a){var d=c.filterNodes(k.nodesFromList(a));r(d,b)}),d.time("constraints.relax",h.relax(a)),d.time("reorientEdges",q)(a)}function m(a){e.undo(a)}function n(a){a.eachEdge(function(b,c,d,e){if(c===d){var f=p(a,b,c,d,e,0,!1),g=p(a,b,c,d,e,1,!0),h=p(a,b,c,d,e,2,!1);a.addEdge(null,f,c,{minLen:1,selfLoop:!0}),a.addEdge(null,f,g,{minLen:1,selfLoop:!0}),a.addEdge(null,c,h,{minLen:1,selfLoop:!0}),a.addEdge(null,g,h,{minLen:1,selfLoop:!0}),a.delEdge(b)}})}function o(a){a.eachEdge(function(b,c,d,e){if(c===d){var f=e.originalEdge,g=p(a,f.e,f.u,f.v,f.value,0,!0);a.addEdge(null,c,g,{minLen:1}),a.addEdge(null,g,d,{minLen:1}),a.delEdge(b)}})}function p(a,b,c,d,e,f,g){return a.addNode(null,{width:g?e.width:0,height:g?e.height:0,edge:{id:b,source:c,target:d,attrs:e},dummy:!0,index:f})}function q(a){a.eachEdge(function(b,c,d,e){a.node(c).rank>a.node(d).rank&&(a.delEdge(b),e.reversed=!0,a.addEdge(b,d,c,e))})}function r(a,b){var c=g(a);b&&(d.log(1,"Using network simplex for ranking"),i(a,c)),s(a)}function s(a){var b=d.min(a.nodes().map(function(b){return a.node(b).rank}));a.eachNode(function(a,c){c.rank-=b})}"use strict";var d=a("./util"),e=a("./rank/acyclic"),f=a("./rank/initRank"),g=a("./rank/feasibleTree"),h=a("./rank/constraints"),i=a("./rank/simplex"),j=a("graphlib").alg.components,k=a("graphlib").filter;c.run=l,c.restoreEdges=m},{"./rank/acyclic":21,"./rank/constraints":22,"./rank/feasibleTree":23,"./rank/initRank":24,"./rank/simplex":26,"./util":27,graphlib:29}],21:[function(a,b,c){function e(a){function f(d){if(d in c)return;c[d]=b[d]=!0,a.outEdges(d).forEach(function(c){var h=a.target(c),i;d===h?console.error('Warning: found self loop "'+c+'" for node "'+d+'"'):h in b?(i=a.edge(c),a.delEdge(c),i.reversed=!0,++e,a.addEdge(c,h,d,i)):f(h)}),delete b[d]}var b={},c={},e=0;return a.eachNode(function(a){f(a)}),d.log(2,"Acyclic Phase: reversed "+e+" edge(s)"),e}function f(a){a.eachEdge(function(b,c,d,e){e.reversed&&(delete e.reversed,a.delEdge(b),a.addEdge(b,d,c,e))})}"use strict";var d=a("../util");b.exports=e,b.exports.undo=f},{"../util":27}],22:[function(a,b,c){function d(a){return a!=="min"&&a!=="max"&&a.indexOf("same_")!==0?(console.error("Unsupported rank type: "+a),!1):!0}function e(a,b,c,d){a.inEdges(b).forEach(function(b){var e=a.edge(b),f;e.originalEdge?f=e:f={originalEdge:{e:b,u:a.source(b),v:a.target(b),value:e},minLen:a.edge(b).minLen},e.selfLoop&&(d=!1),d?(a.addEdge(null,c,a.source(b),f),f.reversed=!0):a.addEdge(null,a.source(b),c,f)})}function f(a,b,c,d){a.outEdges(b).forEach(function(b){var e=a.edge(b),f;e.originalEdge?f=e:f={originalEdge:{e:b,u:a.source(b),v:a.target(b),value:e},minLen:a.edge(b).minLen},e.selfLoop&&(d=!1),d?(a.addEdge(null,a.target(b),c,f),f.reversed=!0):a.addEdge(null,c,a.target(b),f)})}function g(a,b,c){c!==undefined&&a.children(b).forEach(function(b){b!==c&&!a.outEdges(c,b).length&&!a.node(b).dummy&&a.addEdge(null,c,b,{minLen:0})})}function h(a,b,c){c!==undefined&&a.children(b).forEach(function(b){b!==c&&!a.outEdges(b,c).length&&!a.node(b).dummy&&a.addEdge(null,b,c,{minLen:0})})}"use strict",c.apply=function(a){function b(c){var i={};a.children(c).forEach(function(g){if(a.children(g).length){b(g);return}var h=a.node(g),j=h.prefRank;if(j!==undefined){if(!d(j))return;j in i?i.prefRank.push(g):i.prefRank=[g];var k=i[j];k===undefined&&(k=i[j]=a.addNode(null,{originalNodes:[]}),a.parent(k,c)),e(a,g,k,j==="min"),f(a,g,k,j==="max"),a.node(k).originalNodes.push({u:g,value:h,parent:c}),a.delNode(g)}}),g(a,c,i.min),h(a,c,i.max)}b(null)},c.relax=function(a){var b=[];a.eachEdge(function(a,c,d,e){var f=e.originalEdge;f&&b.push(f)}),a.eachNode(function(
-b,c){var d=c.originalNodes;d&&(d.forEach(function(b){b.value.rank=c.rank,a.addNode(b.u,b.value),a.parent(b.u,b.parent)}),a.delNode(b))}),b.forEach(function(b){a.addEdge(b.e,b.u,b.v,b.value)})}},{}],23:[function(a,b,c){function g(a){function g(d){var e=!0;return a.predecessors(d).forEach(function(f){b.has(f)&&!h(a,f,d)&&(b.has(d)&&(c.addNode(d,{}),b.remove(d),c.graph({root:d})),c.addNode(f,{}),c.addEdge(null,f,d,{reversed:!0}),b.remove(f),g(f),e=!1)}),a.successors(d).forEach(function(f){b.has(f)&&!h(a,d,f)&&(b.has(d)&&(c.addNode(d,{}),b.remove(d),c.graph({root:d})),c.addNode(f,{}),c.addEdge(null,d,f,{}),b.remove(f),g(f),e=!1)}),e}function i(){var d=Number.MAX_VALUE;b.keys().forEach(function(c){a.predecessors(c).forEach(function(e){if(!b.has(e)){var f=h(a,e,c);Math.abs(f)<Math.abs(d)&&(d=-f)}}),a.successors(c).forEach(function(e){if(!b.has(e)){var f=h(a,c,e);Math.abs(f)<Math.abs(d)&&(d=f)}})}),c.eachNode(function(b){a.node(b).rank-=d})}var b=new d(a.nodes()),c=new e;if(b.size()===1){var f=a.nodes()[0];return c.addNode(f,{}),c.graph({root:f}),c}while(b.size()){var j=c.order()?c.nodes():b.keys();for(var k=0,l=j.length;k<l&&g(j[k]);++k);b.size()&&i()}return c}function h(a,b,c){var d=a.node(c).rank-a.node(b).rank,e=f.max(a.outEdges(b,c).map(function(b){return a.edge(b).minLen}));return d-e}"use strict";var d=a("cp-data").Set,e=a("graphlib").Digraph,f=a("../util");b.exports=g},{"../util":27,"cp-data":5,graphlib:29}],24:[function(a,b,c){function f(a){var b=e(a);b.forEach(function(b){var c=a.inEdges(b);if(c.length===0){a.node(b).rank=0;return}var e=c.map(function(b){return a.node(a.source(b)).rank+a.edge(b).minLen});a.node(b).rank=d.max(e)})}"use strict";var d=a("../util"),e=a("graphlib").alg.topsort;b.exports=f},{"../util":27,graphlib:29}],25:[function(a,b,c){function d(a,b,c,d){return Math.abs(a.node(b).rank-a.node(c).rank)-d}"use strict",b.exports={slack:d}},{}],26:[function(a,b,c){function f(a,b){g(a,b);for(;;){var c=k(b);if(c===null)break;var d=l(a,b,c);m(a,b,c,d)}}function g(a,b){function c(d){var e=b.successors(d);for(var f in e){var g=e[f];c(g)}d!==b.graph().root&&i(a,b,d)}h(b),b.eachEdge(function(a,b,c,d){d.cutValue=0}),c(b.graph().root)}function h(a){function c(d){var e=a.successors(d),f=b;for(var g in e){var h=e[g];c(h),f=Math.min(f,a.node(h).low)}a.node(d).low=f,a.node(d).lim=b++}var b=0;c(a.graph().root)}function i(a,b,c){var d=b.inEdges(c)[0],e=[],f=b.outEdges(c);for(var g in f)e.push(b.target(f[g]));var h=0,i=0,k=0,l=0,m=0,n=a.outEdges(c),o;for(var p in n){var q=a.target(n[p]);for(o in e)j(b,q,e[o])&&i++;j(b,q,c)||l++}var r=a.inEdges(c);for(var s in r){var t=a.source(r[s]);for(o in e)j(b,t,e[o])&&k++;j(b,t,c)||m++}var u=0;for(o in e){var v=b.edge(f[o]).cutValue;b.edge(f[o]).reversed?u-=v:u+=v}b.edge(d).reversed?h-=u-i+k-l+m:h+=u-i+k-l+m,b.edge(d).cutValue=h}function j(a,b,c){return a.node(c).low<=a.node(b).lim&&a.node(b).lim<=a.node(c).lim}function k(a){var b=a.edges();for(var c in b){var d=b[c],e=a.edge(d);if(e.cutValue<0)return d}return null}function l(a,b,c){var d=b.source(c),f=b.target(c),g=b.node(f).lim<b.node(d).lim?f:d,h=!b.edge(c).reversed,i=Number.POSITIVE_INFINITY,k;h?a.eachEdge(function(d,f,h,l){if(d!==c&&j(b,f,g)&&!j(b,h,g)){var m=e.slack(a,f,h,l.minLen);m<i&&(i=m,k=d)}}):a.eachEdge(function(d,f,h,l){if(d!==c&&!j(b,f,g)&&j(b,h,g)){var m=e.slack(a,f,h,l.minLen);m<i&&(i=m,k=d)}});if(k===undefined){var l=[],m=[];throw a.eachNode(function(a){j(b,a,g)?m.push(a):l.push(a)}),new Error("No edge found from outside of tree to inside")}return k}function m(a,b,c,d){function h(a){var c=b.inEdges(a);for(var d in c){var e=c[d],f=b.source(e),g=b.edge(e);h(f),b.delEdge(e),g.reversed=!g.reversed,b.addEdge(e,a,f,g)}}b.delEdge(c);var e=a.source(d),f=a.target(d);h(f);var i=e,j=b.inEdges(i);while(j.length>0)i=b.source(j[0]),j=b.inEdges(i);b.graph().root=i,b.addEdge(null,e,f,{cutValue:0}),g(a,b),n(a,b)}function n(a,b){function c(d){var e=b.successors(d);e.forEach(function(b){var e=o(a,d,b);a.node(b).rank=a.node(d).rank+e,c(b)})}c(b.graph().root)}function o(a,b,c){var e=a.outEdges(b,c);if(e.length>0)return d.max(e.map(function(b){return a.edge(b).minLen}));var f=a.inEdges(b,c);if(f.length>0)return-d.max(f.map(function(b){return a.edge(b).minLen}))}"use strict";var d=a("../util"),e=a("./rankUtil");b.exports=f},{"../util":27,"./rankUtil":25}],27:[function(a,b,c){function d(a,b){return function(){var c=(new Date).getTime();try{return b.apply(null,arguments)}finally{e(1,a+" time: "+((new Date).getTime()-c)+"ms")}}}function e(a){e.level>=a&&console.log.apply(console,Array.prototype.slice.call(arguments,1))}"use strict",c.min=function(a){return Math.min.apply(Math,a)},c.max=function(a){return Math.max.apply(Math,a)},c.all=function(a,b){for(var c=0;c<a.length;++c)if(!b(a[c]))return!1;return!0},c.sum=function(a){return a.reduce(function(a,b){return a+b},0)},c.values=function(a){return Object.keys(a).map(function(b){return a[b]})},c.shuffle=function(a){for(var b=a.length-1;b>0;--b){var c=Math.floor(Math.random()*(b+1)),d=a[c];a[c]=a[b],a[b]=d}},c.propertyAccessor=function(a,b,c,d){return function(e){return arguments.length?(b[c]=e,d&&d(e),a):b[c]}},c.ordering=function(a){var b=[];return a.eachNode(function(a,c){var d=b[c.rank]||(b[c.rank]=[]);d[c.order]=a}),b},c.filterNonSubgraphs=function(a){return function(b){return a.children(b).length===0}},d.enabled=!1,c.time=d,e.level=0,c.log=e},{}],28:[function(a,b,c){b.exports="0.4.6"},{}],29:[function(a,b,c){c.Graph=a("./lib/Graph"),c.Digraph=a("./lib/Digraph"),c.CGraph=a("./lib/CGraph"),c.CDigraph=a("./lib/CDigraph"),a("./lib/graph-converters"),c.alg={isAcyclic:a("./lib/alg/isAcyclic"),components:a("./lib/alg/components"),dijkstra:a("./lib/alg/dijkstra"),dijkstraAll:a("./lib/alg/dijkstraAll"),findCycles:a("./lib/alg/findCycles"),floydWarshall:a("./lib/alg/floydWarshall"),postorder:a("./lib/alg/postorder"),preorder:a("./lib/alg/preorder"),prim:a("./lib/alg/prim"),tarjan:a("./lib/alg/tarjan"),topsort:a("./lib/alg/topsort")},c.converter={json:a("./lib/converter/json.js")};var d=a("./lib/filter");c.filter={all:d.all,nodesFromList:d.nodesFromList},c.version=a("./lib/version")},{"./lib/CDigraph":31,"./lib/CGraph":32,"./lib/Digraph":33,"./lib/Graph":34,"./lib/alg/components":35,"./lib/alg/dijkstra":36,"./lib/alg/dijkstraAll":37,"./lib/alg/findCycles":38,"./lib/alg/floydWarshall":39,"./lib/alg/isAcyclic":40,"./lib/alg/postorder":41,"./lib/alg/preorder":42,"./lib/alg/prim":43,"./lib/alg/tarjan":44,"./lib/alg/topsort":45,"./lib/converter/json.js":47,"./lib/filter":48,"./lib/graph-converters":49,"./lib/version":51}],30:[function(a,b,c){function e(){this._value=undefined,this._nodes={},this._edges={},this._nextId=0}function f(a,b,c){(a[b]||(a[b]=new d)).add(c)}function g(a,b,c){var d=a[b];d.remove(c),d.size()===0&&delete a[b]}var d=a("cp-data").Set;b.exports=e,e.prototype.order=function(){return Object.keys(this._nodes).length},e.prototype.size=function(){return Object.keys(this._edges).length},e.prototype.graph=function(a){if(arguments.length===0)return this._value;this._value=a},e.prototype.hasNode=function(a){return a in this._nodes},e.prototype.node=function(a,b){var c=this._strictGetNode(a);if(arguments.length===1)return c.value;c.value=b},e.prototype.nodes=function(){var a=[];return this.eachNode(function(b){a.push(b)}),a},e.prototype.eachNode=function(a){for(var b in this._nodes){var c=this._nodes[b];a(c.id,c.value)}},e.prototype.hasEdge=function(a){return a in this._edges},e.prototype.edge=function(a,b){var c=this._strictGetEdge(a);if(arguments.length===1)return c.value;c.value=b},e.prototype.edges=function(){var a=[];return this.eachEdge(function(b){a.push(b)}),a},e.prototype.eachEdge=function(a){for(var b in this._edges){var c=this._edges[b];a(c.id,c.u,c.v,c.value)}},e.prototype.incidentNodes=function(a){var b=this._strictGetEdge(a);return[b.u,b.v]},e.prototype.addNode=function(a,b){if(a===undefined||a===null){do a="_"+ ++this._nextId;while(this.hasNode(a))}else if(this.hasNode(a))throw new Error("Graph already has node '"+a+"'");return this._nodes[a]={id:a,value:b},a},e.prototype.delNode=function(a){this._strictGetNode(a),this.incidentEdges(a).forEach(function(a){this.delEdge(a)},this),delete this._nodes[a]},e.prototype._addEdge=function(a,b,c,d,e,g){this._strictGetNode(b),this._strictGetNode(c);if(a===undefined||a===null){do a="_"+ ++this._nextId;while(this.hasEdge(a))}else if(this.hasEdge(a))throw new Error("Graph already has edge '"+a+"'");return this._edges[a]={id:a,u:b,v:c,value:d},f(e[c],b,a),f(g[b],c,a),a},e.prototype._delEdge=function(a,b,c){var d=this._strictGetEdge(a);g(b[d.v],d.u,a),g(c[d.u],d.v,a),delete this._edges[a]},e.prototype.copy=function(){var a=new this.constructor;return a.graph(this.graph()),this.eachNode(function(b,c){a.addNode(b,c)}),this.eachEdge(function(b,c,d,e){a.addEdge(b,c,d,e)}),a._nextId=this._nextId,a},e.prototype.filterNodes=function(a){var b=new this.constructor;return b.graph(this.graph()),this.eachNode(function(c,d){a(c)&&b.addNode(c,d)}),this.eachEdge(function(a,c,d,e){b.hasNode(c)&&b.hasNode(d)&&b.addEdge(a,c,d,e)}),b},e.prototype._strictGetNode=function(a){var b=this._nodes[a];if(b===undefined)throw new Error("Node '"+a+"' is not in graph");return b},e.prototype._strictGetEdge=function(a){var b=this._edges[a];if(b===undefined)throw new Error("Edge '"+a+"' is not in graph");return b}},{"cp-data":5}],31:[function(a,b,c){var d=a("./Digraph"),e=a("./compoundify"),f=e(d);b.exports=f,f.fromDigraph=function(a){var b=new f,c=a.graph();return c!==undefined&&b.graph(c),a.eachNode(function(a,c){c===undefined?b.addNode(a):b.addNode(a,c)}),a.eachEdge(function(a,c,d,e){e===undefined?b.addEdge(null,c,d):b.addEdge(null,c,d,e)}),b},f.prototype.toString=function(){return"CDigraph "+JSON.stringify(this,null,2)}},{"./Digraph":33,"./compoundify":46}],32:[function(a,b,c){var d=a("./Graph"),e=a("./compoundify"),f=e(d);b.exports=f,f.fromGraph=function(a){var b=new f,c=a.graph();return c!==undefined&&b.graph(c),a.eachNode(function(a,c){c===undefined?b.addNode(a):b.addNode(a,c)}),a.eachEdge(function(a,c,d,e){e===undefined?b.addEdge(null,c,d):b.addEdge(null,c,d,e)}),b},f.prototype.toString=function(){return"CGraph "+JSON.stringify(this,null,2)}},{"./Graph":34,"./compoundify":46}],33:[function(a,b,c){function g(){e.call(this),this._inEdges={},this._outEdges={}}var d=a("./util"),e=a("./BaseGraph"),f=a("cp-data").Set;b.exports=g,g.prototype=new e,g.prototype.constructor=g,g.prototype.isDirected=function(){return!0},g.prototype.successors=function(a){return this._strictGetNode(a),Object.keys(this._outEdges[a]).map(function(a){return this._nodes[a].id},this)},g.prototype.predecessors=function(a){return this._strictGetNode(a),Object.keys(this._inEdges[a]).map(function(a){return this._nodes[a].id},this)},g.prototype.neighbors=function(a){return f.union([this.successors(a),this.predecessors(a)]).keys()},g.prototype.sources=function(){var a=this;return this._filterNodes(function(b){return a.inEdges(b).length===0})},g.prototype.sinks=function(){var a=this;return this._filterNodes(function(b){return a.outEdges(b).length===0})},g.prototype.source=function(a){return this._strictGetEdge(a).u},g.prototype.target=function(a){return this._strictGetEdge(a).v},g.prototype.inEdges=function(a,b){this._strictGetNode(a);var c=f.union(d.values(this._inEdges[a])).keys();return arguments.length>1&&(this._strictGetNode(b),c=c.filter(function(a){return this.source(a)===b},this)),c},g.prototype.outEdges=function(a,b){this._strictGetNode(a);var c=f.union(d.values(this._outEdges[a])).keys();return arguments.length>1&&(this._strictGetNode(b),c=c.filter(function(a){return this.target(a)===b},this)),c},g.prototype.incidentEdges=function(a,b){return arguments.length>1?f.union([this.outEdges(a,b),this.outEdges(b,a)]).keys():f.union([this.inEdges(a),this.outEdges(a)]).keys()},g.prototype.toString=function(){return"Digraph "+JSON.stringify(this,null,2)},g.prototype.addNode=function(a,b){return a=e.prototype.addNode.call(this,a,b),this._inEdges[a]={},this._outEdges[a]={},a},g.prototype.delNode=function(a){e.prototype.delNode.call(this,a),delete this._inEdges[a],delete this._outEdges[a]},g.prototype.addEdge=function(a,b,c,d){return e.prototype._addEdge.call(this,a,b,c,d,this._inEdges,this._outEdges)},g.prototype.delEdge=function(a){e.prototype._delEdge.call(this,a,this._inEdges,this._outEdges)},g.prototype._filterNodes=function(a){var b=[];return this.eachNode(function(c){a(c)&&b.push(c)}),b}},{"./BaseGraph":30,"./util":50,"cp-data":5}],34:[function(a,b,c){function g(){e.call(this),this._incidentEdges={}}var d=a("./util"),e=a("./BaseGraph"),f=a("cp-data").Set;b.exports=g,g.prototype=new e,g.prototype.constructor=g,g.prototype.isDirected=function(){return!1},g.prototype.neighbors=function(a){return this._strictGetNode(a),Object.keys(this._incidentEdges[a]).map(function(a){return this._nodes[a].id},this)},g.prototype.incidentEdges=function(a,b){return this._strictGetNode(a),arguments.length>1?(this._strictGetNode(b),b in this._incidentEdges[a]?this._incidentEdges[a][b].keys():[]):f.union(d.values(this._incidentEdges[a])).keys()},g.prototype.toString=function(){return"Graph "+JSON.stringify(this,null,2)},g.prototype.addNode=function(a,b){return a=e.prototype.addNode.call(this,a,b),this._incidentEdges[a]={},a},g.prototype.delNode=function(a){e.prototype.delNode.call(this,a),delete this._incidentEdges[a]},g.prototype.addEdge=function(a,b,c,d){return e.prototype._addEdge.call(this,a,b,c,d,this._incidentEdges,this._incidentEdges)},g.prototype.delEdge=function(a){e.prototype._delEdge.call(this,a,this._incidentEdges,this._incidentEdges)}},{"./BaseGraph":30,"./util":50,"cp-data":5}],35:[function(a,b,c){function e(a){function e(b,d){c.has(b)||(c.add(b),d.push(b),a.neighbors(b).forEach(function(a){e(a,d)}))}var b=[],c=new d;return a.nodes().forEach(function(a){var c=[];e(a,c),c.length>0&&b.push(c)}),b}var d=a("cp-data").Set;b.exports=e},{"cp-data":5}],36:[function(a,b,c){function e(a,b,c,e){function h(b){var d=a.incidentNodes(b),e=d[0]!==i?d[0]:d[1],h=f[e],k=c(b),l=j.distance+k;if(k<0)throw new Error("dijkstra does not allow negative edge weights. Bad edge: "+b+" Weight: "+k);l<h.distance&&(h.distance=l,h.predecessor=i,g.decrease(e,l))}var f={},g=new d;c=c||function(){return 1},e=e||(a.isDirected()?function(b){return a.outEdges(b)}:function(b){return a.incidentEdges(b)}),a.eachNode(function(a){var c=a===b?0:Number.POSITIVE_INFINITY;f[a]={distance:c},g.add(a,c)});var i,j;while(g.size()>0){i=g.removeMin(),j=f[i];if(j.distance===Number.POSITIVE_INFINITY)break;e(i).forEach(h)}return f}var d=a("cp-data").PriorityQueue;b.exports=e},{"cp-data":5}],37:[function(a,b,c){function e(a,b,c){var e={};return a.eachNode(function(f){e[f]=d(a,f,b,c)}),e}var d=a("./dijkstra");b.exports=e},{"./dijkstra":36}],38:[function(a,b,c){function e(a){return d(a).filter(function(a){return a.length>1})}var d=a("./tarjan");b.exports=e},{"./tarjan":44}],39:[function(a,b,c){function d(a,b,c){var d={},e=a.nodes();return b=b||function(){return 1},c=c||(a.isDirected()?function(b){return a.outEdges(b)}:function(b){return a.incidentEdges(b)}),e.forEach(function(f){d[f]={},d[f][f]={distance:0},e.forEach(function(a){f!==a&&(d[f][a]={distance:Number.POSITIVE_INFINITY})}),c(f).forEach(function(c){var e=a.incidentNodes(c),h=e[0]!==f?e[0]:e[1],i=b(c);i<d[f][h].distance&&(d[f][h]={distance:i,predecessor:f})})}),e.forEach(function(a){var b=d[a];e.forEach(function(c){var f=d[c];e.forEach(function(c){var d=f[a],e=b[c],g=f[c],h=d.distance+e.distance;h<g.distance&&(g.distance=h,g.predecessor=e.predecessor)})})}),d}b.exports=d},{}],40:[function(a,b,c){function e(a){try{d(a)}catch(b){if(b instanceof d.CycleException)return!1;throw b}return!0}var d=a("./topsort");b.exports=e},{"./topsort":45}],41:[function(a,b,c){function e(a,b,c){function f(b,d){if(e.has(b))throw new Error("The input graph is not a tree: "+a);e.add(b),a.neighbors(b).forEach(function(a){a!==d&&f(a,b)}),c(b)}var e=new d;if(a.isDirected())throw new Error("This function only works for undirected graphs");f(b)}var d=a("cp-data").Set;b.exports=e},{"cp-data":5}],42:[function(a,b,c){function e(a,b,c){function f(b,d){if(e.has(b))throw new Error("The input graph is not a tree: "+a);e.add(b),c(b),a.neighbors(b).forEach(function(a){a!==d&&f(a,b)})}var e=new d;if(a.isDirected())throw new Error("This function only works for undirected graphs");f(b)}var d=a("cp-data").Set;b.exports=e},{"cp-data":5}],43:[function(a,b,c){function f(a,b){function i(c){var d=a.incidentNodes(c),e=d[0]!==h?d[0]:d[1],i=g.priority(e);if(i!==undefined){var j=b(c);j<i&&(f[e]=h,g.decrease(e,j))}}var c=new d,f={},g=new e,h;if(a.order()===0)return c;a.eachNode(function(a){g.add(a,Number.POSITIVE_INFINITY),c.addNode(a)}),g.decrease(a.nodes()[0],0);var j=!1;while(g.size()>0){h=g.removeMin();if(h in f)c.addEdge(null,h,f[h]);else{if(j)throw new Error("Input graph is not connected: "+a);j=!0}a.incidentEdges(h).forEach(i)}return c}var d=a("../Graph"),e=a("cp-data").PriorityQueue;b.exports=f},{"../Graph":34,"cp-data":5}],44:[function(a,b,c){function d(a){function f(h){var i=d[h]={onStack:!0,lowlink:b,index:b++};c.push(h),a.successors(h).forEach(function(a){a in d?d[a].onStack&&(i.lowlink=Math.min(i.lowlink,d[a].index)):(f(a),i.lowlink=Math.min(i.lowlink,d[a].lowlink))});if(i.lowlink===i.index){var j=[],k;do k=c.pop(),d[k].onStack=!1,j.push(k);while(h!==k);e.push(j)}}if(!a.isDirected())throw new Error("tarjan can only be applied to a directed graph. Bad input: "+a);var b=0,c=[],d={},e=[];return a.nodes().forEach(function(a){a in d||f(a)}),e}b.exports=d},{}],45:[function(a,b,c){function d(a){function f(g){if(g in c)throw new e;g in b||(c[g]=!0,b[g]=!0,a.predecessors(g).forEach(function(a){f(a)}),delete c[g],d.push(g))}if(!a.isDirected())throw new Error("topsort can only be applied to a directed graph. Bad input: "+a);var b={},c={},d=[],g=a.sinks();if(a.order()!==0&&g.length===0)throw new e;return a.sinks().forEach(function(a){f(a)}),d}function e(){}b.exports=d,d.CycleException=e,e.prototype.toString=function(){return"Graph has at least one cycle"}},{}],46:[function(a,b,c){function e(a){function b(){a.call(this),this._parents={},this._children={},this._children[null]=new d}return b.prototype=new a,b.prototype.constructor=b,b.prototype.parent=function(a,b){this._strictGetNode(a);if(arguments.length<2)return this._parents[a];if(a===b)throw new Error("Cannot make "+a+" a parent of itself");b!==null&&this._strictGetNode(b),this._children[this._parents[a]].remove(a),this._parents[a]=b,this._children[b].add(a)},b.prototype.children=function(a){return a!==null&&this._strictGetNode(a),this._children[a].keys()},b.prototype.addNode=function(b,c){return b=a.prototype.addNode.call(this,b,c),this._parents[b]=null,this._children[b]=new d,this._children[null].add(b),b},b.prototype.delNode=function(b){var c=this.parent(b);return this._children[b].keys().forEach(function(a){this.parent(a,c)},this),this._children[c].remove(b),delete this._parents[b],delete this._children[b],a.prototype.delNode.call(this,b)},b.prototype.copy=function(){var b=a.prototype.copy.call(this);return this.nodes().forEach(function(a){b.parent(a,this.parent(a))},this),b},b.prototype.filterNodes=function(b){function f(a){var b=c.parent(a);return b===null||d.hasNode(b)?(e[a]=b,b):b in e?e[b]:f(b)}var c=this,d=a.prototype.filterNodes.call(this,b),e={};return d.eachNode(function(a){d.parent(a,f(a))}),d},b}var d=a("cp-data").Set;b.exports=e},{"cp-data":5}],47:[function(a,b,c){function h(a){return Object.prototype.toString.call(a).slice(8,-1)}var d=a("../Graph"),e=a("../Digraph"),f=a("../CGraph"),g=a("../CDigraph");c.decode=function(a,b,c){c=c||e;if(h(a)!=="Array")throw new Error("nodes is not an Array");if(h(b)!=="Array")throw new Error("edges is not an Array");if(typeof c=="string")switch(c){case"graph":c=d;break;case"digraph":c=e;break;case"cgraph":c=f;break;case"cdigraph":c=g;break;default:throw new Error("Unrecognized graph type: "+c)}var i=new c;return a.forEach(function(a){i.addNode(a.id,a.value)}),i.parent&&a.forEach(function(a){a.children&&a.children.forEach(function(b){i.parent(b,a.id)})}),b.forEach(function(a){i.addEdge(a.id,a.u,a.v,a.value)}),i},c.encode=function(a){var b=[],c=[];a.eachNode(function(c,d){var e={id:c,value:d};if(a.children){var f=a.children(c);f.length&&(e.children=f)}b.push(e)}),a.eachEdge(function(a,b,d,e){c.push({id:a,u:b,v:d,value:e})});var h;if(a instanceof g)h="cdigraph";else if(a instanceof f)h="cgraph";else if(a instanceof e)h="digraph";else if(a instanceof d)h="graph";else throw new Error("Couldn't determine type of graph: "+a);return{nodes:b,edges:c,type:h}}},{"../CDigraph":31,"../CGraph":32,"../Digraph":33,"../Graph":34}],48:[function(a,b,c){var d=a("cp-data").Set;c.all=function(){return function(){return!0}},c.nodesFromList=function(a){var b=new d(a);return function(a){return b.has(a)}}},{"cp-data":5}],49:[function(a,b,c){var d=a("./Graph"),e=a("./Digraph");d.prototype.toDigraph=d.prototype.asDirected=function(){var a=new e;return this.eachNode(function(b,c){a.addNode(b,c)}),this.eachEdge(function(b,c,d,e){a.addEdge(null,c,d,e),a.addEdge(null,d,c,e)}),a},e.prototype.toGraph=e.prototype.asUndirected=function(){var a=new d;return this.eachNode(function(b,c){a.addNode(b,c)}),this.eachEdge(function(b,c,d,e){a.addEdge(b,c,d,e)}),a}},{"./Digraph":33,"./Graph":34}],50:[function(a,b,c){c.values=function(a){var b=Object.keys(a),c=b.length,d=new Array(c),e;for(e=0;e<c;++e)d[e]=a[b[e]];return d}},{}],51:[function(a,b,c){b.exports="0.7.4"},{}]},{},[1]);
\ No newline at end of file
diff --git a/airflow/www_rbac/static/dagre.css b/airflow/www_rbac/static/dagre.css
deleted file mode 100644
index acf4ff5facbc8..0000000000000
--- a/airflow/www_rbac/static/dagre.css
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-svg {
-    overflow: hidden;
-}
-
-.node rect {
-    stroke: #333;
-    stroke-width: 1px;
-    fill: #fff;
-}
-
-.edgeLabel rect {
-    fill: #fff;
-}
-
-.edgePath {
-    stroke: #333;
-    stroke-width: 1px;
-    fill: none;
-}
diff --git a/airflow/www_rbac/static/dataTables.bootstrap.css b/airflow/www_rbac/static/dataTables.bootstrap.css
deleted file mode 100644
index 8ca39c3e009e2..0000000000000
--- a/airflow/www_rbac/static/dataTables.bootstrap.css
+++ /dev/null
@@ -1,333 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-div.dataTables_length label {
-	font-weight: normal;
-	text-align: left;
-	white-space: nowrap;
-}
-
-div.dataTables_length select {
-	width: 75px;
-	display: inline-block;
-}
-
-div.dataTables_filter {
-	text-align: right;
-}
-
-div.dataTables_filter label {
-	font-weight: normal;
-	white-space: nowrap;
-	text-align: left;
-}
-
-div.dataTables_filter input {
-	margin-left: 0.5em;
-	display: inline-block;
-}
-
-div.dataTables_info {
-	padding-top: 8px;
-	white-space: nowrap;
-}
-
-div.dataTables_paginate {
-	margin: 0;
-	white-space: nowrap;
-	text-align: right;
-}
-
-div.dataTables_paginate ul.pagination {
-	margin: 2px 0;
-	white-space: nowrap;
-}
-
-@media screen and (max-width: 767px) {
-	div.dataTables_length,
-	div.dataTables_filter,
-	div.dataTables_info,
-	div.dataTables_paginate {
-		text-align: center;
-	}
-}
-
-
-table.dataTable td,
-table.dataTable th {
-	-webkit-box-sizing: content-box;
-	-moz-box-sizing: content-box;
-	box-sizing: content-box;
-}
-
-
-table.dataTable {
-	clear: both;
-	margin-top: 6px !important;
-	margin-bottom: 6px !important;
-	max-width: none !important;
-}
-
-table.dataTable thead .sorting,
-table.dataTable thead .sorting_asc,
-table.dataTable thead .sorting_desc,
-table.dataTable thead .sorting_asc_disabled,
-table.dataTable thead .sorting_desc_disabled {
-	cursor: pointer;
-}
-
-table.dataTable thead .sorting { background: url('../images/sort_both.png') no-repeat center right; }
-table.dataTable thead .sorting_asc { background: url('../images/sort_asc.png') no-repeat center right; }
-table.dataTable thead .sorting_desc { background: url('../images/sort_desc.png') no-repeat center right; }
-
-table.dataTable thead .sorting_asc_disabled { background: url('../images/sort_asc_disabled.png') no-repeat center right; }
-table.dataTable thead .sorting_desc_disabled { background: url('../images/sort_desc_disabled.png') no-repeat center right; }
-
-table.dataTable thead > tr > th {
-	padding-left: 18px;
-	padding-right: 18px;
-}
-
-table.dataTable th:active {
-	outline: none;
-}
-
-/* Scrolling */
-div.dataTables_scrollHead table {
-	margin-bottom: 0 !important;
-	border-bottom-left-radius: 0;
-	border-bottom-right-radius: 0;
-}
-
-div.dataTables_scrollHead table thead tr:last-child th:first-child,
-div.dataTables_scrollHead table thead tr:last-child td:first-child {
-	border-bottom-left-radius: 0 !important;
-	border-bottom-right-radius: 0 !important;
-}
-
-div.dataTables_scrollBody table {
-	border-top: none;
-	margin-top: 0 !important;
-	margin-bottom: 0 !important;
-}
-
-div.dataTables_scrollBody tbody tr:first-child th,
-div.dataTables_scrollBody tbody tr:first-child td {
-	border-top: none;
-}
-
-div.dataTables_scrollFoot table {
-	margin-top: 0 !important;
-	border-top: none;
-}
-
-/* Frustratingly the border-collapse:collapse used by Bootstrap makes the column
-   width calculations when using scrolling impossible to align columns. We have
-   to use separate
- */
-table.table-bordered.dataTable {
-	border-collapse: separate !important;
-}
-table.table-bordered thead th,
-table.table-bordered thead td {
-	border-left-width: 0;
-	border-top-width: 0;
-}
-table.table-bordered tbody th,
-table.table-bordered tbody td {
-	border-left-width: 0;
-	border-bottom-width: 0;
-}
-table.table-bordered th:last-child,
-table.table-bordered td:last-child {
-	border-right-width: 0;
-}
-div.dataTables_scrollHead table.table-bordered {
-	border-bottom-width: 0;
-}
-
-
-
-
-/*
- * TableTools styles
- */
-.table.dataTable tbody tr.active td,
-.table.dataTable tbody tr.active th {
-	background-color: #08C;
-	color: white;
-}
-
-.table.dataTable tbody tr.active:hover td,
-.table.dataTable tbody tr.active:hover th {
-	background-color: #0075b0 !important;
-}
-
-.table.dataTable tbody tr.active th > a,
-.table.dataTable tbody tr.active td > a {
-	color: white;
-}
-
-.table-striped.dataTable tbody tr.active:nth-child(odd) td,
-.table-striped.dataTable tbody tr.active:nth-child(odd) th {
-	background-color: #017ebc;
-}
-
-table.DTTT_selectable tbody tr {
-	cursor: pointer;
-}
-
-div.DTTT .btn:hover {
-	text-decoration: none !important;
-}
-
-ul.DTTT_dropdown.dropdown-menu {
-  z-index: 2003;
-}
-
-ul.DTTT_dropdown.dropdown-menu a {
-	color: #333 !important; /* needed only when demo_page.css is included */
-}
-
-ul.DTTT_dropdown.dropdown-menu li {
-	position: relative;
-}
-
-ul.DTTT_dropdown.dropdown-menu li:hover a {
-	background-color: #0088cc;
-	color: white !important;
-}
-
-div.DTTT_collection_background {
-	z-index: 2002;
-}
-
-/* TableTools information display */
-div.DTTT_print_info {
-	position: fixed;
-	top: 50%;
-	left: 50%;
-	width: 400px;
-	height: 150px;
-	margin-left: -200px;
-	margin-top: -75px;
-	text-align: center;
-	color: #333;
-	padding: 10px 30px;
-	opacity: 0.95;
-
-	background-color: white;
-	border: 1px solid rgba(0, 0, 0, 0.2);
-	border-radius: 6px;
-
-	-webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.5);
-	        box-shadow: 0 3px 7px rgba(0, 0, 0, 0.5);
-}
-
-div.DTTT_print_info h6 {
-	font-weight: normal;
-	font-size: 28px;
-	line-height: 28px;
-	margin: 1em;
-}
-
-div.DTTT_print_info p {
-	font-size: 14px;
-	line-height: 20px;
-}
-
-div.dataTables_processing {
-    position: absolute;
-    top: 50%;
-    left: 50%;
-    width: 100%;
-    height: 60px;
-    margin-left: -50%;
-    margin-top: -25px;
-    padding-top: 20px;
-    padding-bottom: 20px;
-    text-align: center;
-    font-size: 1.2em;
-    background-color: white;
-    background: -webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255,255,255,0)), color-stop(25%, rgba(255,255,255,0.9)), color-stop(75%, rgba(255,255,255,0.9)), color-stop(100%, rgba(255,255,255,0)));
-    background: -webkit-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: -moz-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: -ms-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: -o-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-    background: linear-gradient(to right, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);
-}
-
-
-
-/*
- * FixedColumns styles
- */
-div.DTFC_LeftHeadWrapper table,
-div.DTFC_LeftFootWrapper table,
-div.DTFC_RightHeadWrapper table,
-div.DTFC_RightFootWrapper table,
-table.DTFC_Cloned tr.even {
-    background-color: white;
-    margin-bottom: 0;
-}
-
-div.DTFC_RightHeadWrapper table ,
-div.DTFC_LeftHeadWrapper table {
-	border-bottom: none !important;
-    margin-bottom: 0 !important;
-    border-top-right-radius: 0 !important;
-    border-bottom-left-radius: 0 !important;
-    border-bottom-right-radius: 0 !important;
-}
-
-div.DTFC_RightHeadWrapper table thead tr:last-child th:first-child,
-div.DTFC_RightHeadWrapper table thead tr:last-child td:first-child,
-div.DTFC_LeftHeadWrapper table thead tr:last-child th:first-child,
-div.DTFC_LeftHeadWrapper table thead tr:last-child td:first-child {
-    border-bottom-left-radius: 0 !important;
-    border-bottom-right-radius: 0 !important;
-}
-
-div.DTFC_RightBodyWrapper table,
-div.DTFC_LeftBodyWrapper table {
-    border-top: none;
-    margin: 0 !important;
-}
-
-div.DTFC_RightBodyWrapper tbody tr:first-child th,
-div.DTFC_RightBodyWrapper tbody tr:first-child td,
-div.DTFC_LeftBodyWrapper tbody tr:first-child th,
-div.DTFC_LeftBodyWrapper tbody tr:first-child td {
-    border-top: none;
-}
-
-div.DTFC_RightFootWrapper table,
-div.DTFC_LeftFootWrapper table {
-    border-top: none;
-    margin-top: 0 !important;
-}
-
-
-/*
- * FixedHeader styles
- */
-div.FixedHeader_Cloned table {
-	margin: 0 !important
-}
-
diff --git a/airflow/www_rbac/static/jqClock.min.js b/airflow/www_rbac/static/jqClock.min.js
deleted file mode 100644
index 5528efd186774..0000000000000
--- a/airflow/www_rbac/static/jqClock.min.js
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * jQuery Clock plugin
- * Copyright (c) 2010 John R D'Orazio (priest@johnromanodorazio.com)
- * Licensed under the Apache 2.0 license:
- * https://www.apache.org/licenses/LICENSE-2.0
- */
-Date.prototype.hasOwnProperty("stdTimezoneOffset")||(Date.prototype.stdTimezoneOffset=function(){var b=this.getFullYear();if(!Date.prototype.stdTimezoneOffset.cache.hasOwnProperty(b)){for(var d=(new Date(b,0,1)).getTimezoneOffset(),h=[6,7,5,8,4,9,3,10,2,11,1],a=0;12>a;a++){var m=(new Date(b,h[a],1)).getTimezoneOffset();if(m!=d){d=Math.max(d,m);break}}Date.prototype.stdTimezoneOffset.cache[b]=d}return Date.prototype.stdTimezoneOffset.cache[b]},Date.prototype.stdTimezoneOffset.cache={});
-Date.prototype.hasOwnProperty("isDST")||(Date.prototype.isDST=function(){return this.getTimezoneOffset()<this.stdTimezoneOffset()});Date.prototype.hasOwnProperty("isLeapYear")||(Date.prototype.isLeapYear=function(){var b=this.getFullYear();return 0!=(b&3)?!1:0!=b%100||0==b%400});Date.prototype.hasOwnProperty("getDOY")||(Date.prototype.getDOY=function(){var b=this.getMonth(),d=this.getDate(),d=[0,31,59,90,120,151,181,212,243,273,304,334][b]+d;1<b&&this.isLeapYear()&&d++;return d});
-Date.prototype.hasOwnProperty("daysInMonth")||(Date.prototype.daysInMonth=function(){return[31,this.isLeapYear()?29:28,31,30,31,30,31,31,30,31,30,31][this.getMonth()]});Date.prototype.hasOwnProperty("getWOY")||(Date.prototype.getWOY=function(b){var d=new Date(+this);d.setHours(0,0,0,0);d.setDate(d.getDate()+4-(d.getDay()||7));return b?d.getFullYear():Math.ceil(((d-new Date(d.getFullYear(),0,1))/864E5+1)/7)});
-Date.prototype.hasOwnProperty("swatchTime")||(Date.prototype.swatchTime=function(){return("00"+Math.floor((60*((this.getUTCHours()+1)%24*60+this.getUTCMinutes())+this.getUTCSeconds()+.001*this.getUTCMilliseconds())/86.4)).slice(-3)});String.prototype.padStart||(String.prototype.padStart=function(b,d){b>>=0;d=String(d||" ");if(this.length>b)return String(this);b-=this.length;b>d.length&&(d+=d.repeat(b/d.length));return d.slice(0,b)+String(this)});
-(function(b,d){b.clock={version:"2.3.0",options:[{type:"string",value:"destroy",description:"Passing in 'destroy' to an already initialized clock will remove the setTimeout for that clock to stop it from ticking, and remove all html markup and data associated with the plugin instance on the dom elements"},{type:"string",value:"stop",description:"Passing in 'stop' to an already initialized clock will clear the setTimeout for that clock to stop it from ticking"},{type:"string",value:"start",description:"Passing in 'start' to an already initialized clock will restart the setTimeout for that clock to get it ticking again, as though it had never lost time"},
-{type:"object",description:"option set {}",values:[{name:"timestamp",description:"Either a javascript timestamp as produces by [JAVASCRIPT new Date().getTime()] or a php timestamp as produced by [PHP time()] ",type:"unix timestamp",values:["javascript timestamp","php timestamp"]},{name:"langSet",description:"two letter locale to be used for the translation of Day names and Month names",type:"String",values:"am ar bn bg ca zh hr cs da nl en et fi fr de el gu hi hu id it ja kn ko lv lt ms ml mr mo ps fa pl pt ro ru sr sk sl es sw sv ta te th tr uk vi".split(" ")},
-{name:"calendar",description:"Whether the date should be displayed together with the time",type:"Boolean",values:[!0,!1]},{name:"dateFormat",description:"PHP Style Format string for formatting a local date, see http://php.net/manual/en/function.date.php",type:"String",values:"dDjlNSwzWFmMntLoYy".split("")},{name:"timeFormat",description:"PHP Style Format string for formatting a local date, see http://php.net/manual/en/function.date.php",type:"String",values:"aABgGhHisveIOPZcrU".split("")},{name:"isDST",
-description:"When a client side timestamp is used, whether DST is active will be automatically determined. However this cannot be determined for a server-side timestamp which must be passed in as UTC, in that can case it can be set with this option",type:"Boolean",values:[!0,!1]},{name:"rate",description:"Defines the rate at which the clock will update, in milliseconds",type:"Integer",values:"1 - 9007199254740991 (recommended 10-60000)"}]}],methods:{destroy:"Chaining clock().destroy() has the same effect as passing the 'destroy' option as in clock('destroy')",
-stop:"Chaining clock().stop() has the same effect as passing the 'stop' option as in clock('stop')",start:"Chaining clock().start() has the same effect as passing the 'start' option as in clock('start')"}};Object.freeze(b.clock);var h=h||{};b.fn.clock=function(a){var m=this;this.initialize=function(){return this};this.destroy=function(){return m.each(function(a){a=b(this).attr("id");h.hasOwnProperty(a)&&(clearTimeout(h[a]),delete h[a]);b(this).html("");b(this).hasClass("jqclock")&&b(this).removeClass("jqclock");
-b(this).removeData("clockoptions")})};this.stop=function(){return m.each(function(a){a=b(this).attr("id");h.hasOwnProperty(a)&&(clearTimeout(h[a]),delete h[a])})};this.start=function(){return m.each(function(a){a=b(this).attr("id");var c=b(this).data("clockoptions");if(c!==d&&!1===h.hasOwnProperty(a)){var g=this;h[a]=setTimeout(function(){x(b(g))},c.rate)}})};var B=function(){return"xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g,function(a){var b=16*Math.random()|0;return("x"==a?b:b&3|8).toString(16)}).toUpperCase()},
-x=function(a){var c=b(a).data("clockoptions"),d=(new Date).getTime()+c.sysdiff,k=new Date(d),g=k.getHours(),m=k.getMinutes(),C=k.getSeconds(),B=k.getMilliseconds(),e=k.getDay(),l=k.getDate(),D=k.getMonth(),y=k.getFullYear(),p=k.isLeapYear(),w=k.getDOY(),q=k.getWOY(),v=k.getWOY(!0),G=k.daysInMonth(),H=k.swatchTime(),n=parseInt(c.tzOffset/60),z=parseInt(60*c.tzOffset),E="AM",f="",r="";11<g&&(E="PM");r=g;12<r?r-=12:0===r&&(r=12);if(!0===c.calendar){for(var f="",t=0;t<=c.dateFormat.length;t++){var F=
-c.dateFormat.charAt(t);switch(F){case "d":f+=(""+l).padStart(2,"0");break;case "D":f+=(new Intl.DateTimeFormat(c.langSet,{weekday:"short"})).format(k);break;case "j":f+=l;break;case "l":f+=(new Intl.DateTimeFormat(c.langSet,{weekday:"long"})).format(k);break;case "N":f+=0===e?7:e;break;case "S":f+=1===l||1===l%10&&11!=l?"st":2===l||2===l%10&&12!=l?"nd":3===l||3===l%10&&13!=l?"rd":"th";break;case "w":f+=e;break;case "z":f+=w-1;break;case "W":f+=q;break;case "F":f+=(new Intl.DateTimeFormat(c.langSet,
-{month:"long"})).format(k);break;case "m":f+=(D+1+"").padStart(2,"0");break;case "M":f+=(new Intl.DateTimeFormat(c.langSet,{month:"short"})).format(k);break;case "n":f+=D+1;break;case "t":f+=G;break;case "L":f+=p?1:0;break;case "o":f+=v;break;case "Y":f+=y;break;case "y":f+=y.toString().substr(2,2);break;case String.fromCharCode(92):f+=c.dateFormat.charAt(++t);break;case "%":for(var u=t+1,A=c.dateFormat;u<A.length&&"%"!=A.charAt(u);)u++;u>t+1&&u!=A.length?(f+=A.substring(t+1,u),t+=u-t):f+=F;break;
-default:f+=F}}f='<span class="clockdate">'+f+"</span>"}e="";for(p=0;p<=c.timeFormat.length;p++)switch(w=c.timeFormat.charAt(p),w){case "a":e+=E.toLowerCase();break;case "A":e+=E;break;case "B":e+=H;break;case "g":e+=r;break;case "G":e+=g;break;case "h":e+=(""+r).padStart(2,"0");break;case "H":e+=(""+g).padStart(2,"0");break;case "i":e+=(""+m).padStart(2,"0");break;case "s":e+=(""+C).padStart(2,"0");break;case "v":e+=(""+B).padStart(3,"0");break;case "e":e+=c.timezone;break;case "I":e+=c.isDST?"DST":
-"";break;case "O":e+=(0>n?"+"+(""+Math.abs(n)).padStart(2,"0"):0<n?(""+-1*n).padStart(2,"0"):"+00")+"00";break;case "P":e+=(0>n?"+"+(""+Math.abs(n)).padStart(2,"0"):0<n?(""+-1*n).padStart(2,"0"):"+00")+":00";break;case "Z":e+=0>z?""+Math.abs(z):0<z?""+-1*z:"0";break;case "c":e+=y+"-"+(D+1+"").padStart(2,"0")+"-"+(""+l).padStart(2,"0")+"T"+(""+g).padStart(2,"0")+":"+(""+m).padStart(2,"0")+":"+(""+C).padStart(2,"0")+(0>n?"+"+(""+Math.abs(n)).padStart(2,"0"):0<tzh?(""+-1*tzh).padStart(2,"0"):"+00")+
-":00";break;case "r":e+=(new Intl.DateTimeFormat(c.langSet,{weekday:"short"})).format(k)+", "+l+" "+(new Intl.DateTimeFormat(c.langSet,{month:"short"})).format(k)+" "+y+" "+(""+g).padStart(2,"0")+":"+(""+m).padStart(2,"0")+":"+(""+C).padStart(2,"0")+" "+(0>n?"+"+(""+Math.abs(n)).padStart(2,"0"):0<tzh?(""+-1*tzh).padStart(2,"0"):"+00")+"00";break;case "U":e+=Math.floor(d/1E3);break;case String.fromCharCode(92):e+=c.timeFormat.charAt(++p);break;case "%":q=p+1;for(v=c.timeFormat;q<v.length&&"%"!=v.charAt(q);)q++;
-q>p+1&&q!=v.length?(e+=v.substring(p+1,q),p+=q-p):e+=w;break;default:e+=w}r='<span class="clocktime">'+e+"</span>";b(a).html(f+r);d=b(a).attr("id");h[d]=setTimeout(function(){x(b(a))},c.rate)};this.each(function(g){if("undefined"===typeof a||"object"===typeof a){g=new Date;a=a||{};a.timestamp=a.timestamp||"localsystime";a.langSet=a.langSet||"en";a.calendar=a.hasOwnProperty("calendar")?a.calendar:!0;a.dateFormat=a.dateFormat||("en"==a.langSet?"l, F j, Y":"l, j F Y");a.timeFormat=a.timeFormat||("en"==
-a.langSet?"h:i:s A":"H:i:s");a.timezone=a.timezone||"localsystimezone";a.isDST=a.hasOwnProperty("isDST")?a.isDST:g.isDST();a.rate=a.rate||500;"string"!==typeof a.langSet&&(a.langSet=""+a.langSet);"string"===typeof a.calendar?a.calendar="false"==a.calendar?!1:!0:"boolean"!==typeof a.calendar&&(a.calendar=!!a.calendar);"string"!==typeof a.dateFormat&&(a.dateFormat=""+a.dateFormat);"string"!==typeof a.timeFormat&&(a.timeFormat=""+a.dateFormat);"string"!==typeof a.timezone&&(a.timezone=""+a.timezone);
-"string"===typeof a.isDST?a.isDST="true"==a.isDST?!0:!1:"boolean"!==typeof a.isDST&&(a.isDST=!!a.isDST);"number"!==typeof a.rate&&(a.rate=parseInt(a.rate));a.tzOffset=g.getTimezoneOffset();var c=a.tzOffset/60;a.sysdiff=0;"localsystime"!=a.timestamp?2<(g.getTime()+"").length-(a.timestamp+"").length?(a.timestamp*=1E3,a.sysdiff=a.timestamp-g.getTime()+6E4*a.tzOffset):(a.sysdiff=a.timestamp-g.getTime(),"localsystimezone"==a.timezone&&(a.timezone="UTC",0>c?a.timezone+="+"+Math.abs(c):0<c&&(a.timezone+=
--1*c))):"localsystimezone"==a.timezone&&(a.timezone="UTC",0>c?a.timezone+="+"+Math.abs(c):0<c&&(a.timezone+=-1*c));b(this).hasClass("jqclock")||b(this).addClass("jqclock");b(this).is("[id]")||b(this).attr("id",B());b(this).data("clockoptions",a);!1===h.hasOwnProperty(b(this).attr("id"))&&x(b(this))}else if("string"===typeof a)switch(g=b(this).attr("id"),a){case "destroy":h.hasOwnProperty(g)&&(clearTimeout(h[g]),delete h[g]);b(this).html("");b(this).hasClass("jqclock")&&b(this).removeClass("jqclock");
-b(this).removeData("clockoptions");break;case "stop":h.hasOwnProperty(g)&&(clearTimeout(h[g]),delete h[g]);break;case "start":var m=this,c=b(this).data("clockoptions");c!==d&&!1===h.hasOwnProperty(g)&&(h[g]=setTimeout(function(){x(b(m))},c.rate))}});return this.initialize()};return this})(jQuery);
diff --git a/airflow/www_rbac/static/jquery.dataTables.css b/airflow/www_rbac/static/jquery.dataTables.css
deleted file mode 100644
index 6a2e88d942260..0000000000000
--- a/airflow/www_rbac/static/jquery.dataTables.css
+++ /dev/null
@@ -1,495 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-/*
- * Table styles
- */
-table.dataTable {
-  width: 100%;
-  margin: 0 auto;
-  clear: both;
-  border-collapse: separate;
-  border-spacing: 0;
-  /*
-   * Header and footer styles
-   */
-  /*
-   * Body styles
-   */
-}
-table.dataTable thead th,
-table.dataTable tfoot th {
-  font-weight: bold;
-}
-table.dataTable thead th,
-table.dataTable thead td {
-  padding: 10px 18px;
-  border-bottom: 1px solid #111111;
-}
-table.dataTable thead th:active,
-table.dataTable thead td:active {
-  outline: none;
-}
-table.dataTable tfoot th,
-table.dataTable tfoot td {
-  padding: 10px 18px 6px 18px;
-  border-top: 1px solid #111111;
-}
-table.dataTable thead .sorting_asc,
-table.dataTable thead .sorting_desc,
-table.dataTable thead .sorting {
-  cursor: pointer;
-  *cursor: hand;
-}
-table.dataTable thead .sorting {
-  background: url("../images/sort_both.png") no-repeat center right;
-}
-table.dataTable thead .sorting_asc {
-  background: url("../images/sort_asc.png") no-repeat center right;
-}
-table.dataTable thead .sorting_desc {
-  background: url("../images/sort_desc.png") no-repeat center right;
-}
-table.dataTable thead .sorting_asc_disabled {
-  background: url("../images/sort_asc_disabled.png") no-repeat center right;
-}
-table.dataTable thead .sorting_desc_disabled {
-  background: url("../images/sort_desc_disabled.png") no-repeat center right;
-}
-table.dataTable tbody tr {
-  background-color: white;
-}
-table.dataTable tbody tr.selected {
-  background-color: #b0bed9;
-}
-table.dataTable tbody th,
-table.dataTable tbody td {
-  padding: 8px 10px;
-}
-table.dataTable.row-border tbody th, table.dataTable.row-border tbody td, table.dataTable.display tbody th, table.dataTable.display tbody td {
-  border-top: 1px solid #dddddd;
-}
-table.dataTable.row-border tbody tr:first-child th,
-table.dataTable.row-border tbody tr:first-child td, table.dataTable.display tbody tr:first-child th,
-table.dataTable.display tbody tr:first-child td {
-  border-top: none;
-}
-table.dataTable.cell-border tbody th, table.dataTable.cell-border tbody td {
-  border-top: 1px solid #dddddd;
-  border-right: 1px solid #dddddd;
-}
-table.dataTable.cell-border tbody tr th:first-child,
-table.dataTable.cell-border tbody tr td:first-child {
-  border-left: 1px solid #dddddd;
-}
-table.dataTable.cell-border tbody tr:first-child th,
-table.dataTable.cell-border tbody tr:first-child td {
-  border-top: none;
-}
-table.dataTable.stripe tbody tr.odd, table.dataTable.display tbody tr.odd {
-  background-color: #f9f9f9;
-}
-table.dataTable.stripe tbody tr.odd.selected, table.dataTable.display tbody tr.odd.selected {
-  background-color: #abb9d3;
-}
-table.dataTable.hover tbody tr:hover,
-table.dataTable.hover tbody tr.odd:hover,
-table.dataTable.hover tbody tr.even:hover, table.dataTable.display tbody tr:hover,
-table.dataTable.display tbody tr.odd:hover,
-table.dataTable.display tbody tr.even:hover {
-  background-color: whitesmoke;
-}
-table.dataTable.hover tbody tr:hover.selected,
-table.dataTable.hover tbody tr.odd:hover.selected,
-table.dataTable.hover tbody tr.even:hover.selected, table.dataTable.display tbody tr:hover.selected,
-table.dataTable.display tbody tr.odd:hover.selected,
-table.dataTable.display tbody tr.even:hover.selected {
-  background-color: #a9b7d1;
-}
-table.dataTable.order-column tbody tr > .sorting_1,
-table.dataTable.order-column tbody tr > .sorting_2,
-table.dataTable.order-column tbody tr > .sorting_3, table.dataTable.display tbody tr > .sorting_1,
-table.dataTable.display tbody tr > .sorting_2,
-table.dataTable.display tbody tr > .sorting_3 {
-  background-color: #f9f9f9;
-}
-table.dataTable.order-column tbody tr.selected > .sorting_1,
-table.dataTable.order-column tbody tr.selected > .sorting_2,
-table.dataTable.order-column tbody tr.selected > .sorting_3, table.dataTable.display tbody tr.selected > .sorting_1,
-table.dataTable.display tbody tr.selected > .sorting_2,
-table.dataTable.display tbody tr.selected > .sorting_3 {
-  background-color: #acbad4;
-}
-table.dataTable.display tbody tr.odd > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd > .sorting_1 {
-  background-color: #f1f1f1;
-}
-table.dataTable.display tbody tr.odd > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd > .sorting_2 {
-  background-color: #f3f3f3;
-}
-table.dataTable.display tbody tr.odd > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd > .sorting_3 {
-  background-color: whitesmoke;
-}
-table.dataTable.display tbody tr.odd.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_1 {
-  background-color: #a6b3cd;
-}
-table.dataTable.display tbody tr.odd.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_2 {
-  background-color: #a7b5ce;
-}
-table.dataTable.display tbody tr.odd.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_3 {
-  background-color: #a9b6d0;
-}
-table.dataTable.display tbody tr.even > .sorting_1, table.dataTable.order-column.stripe tbody tr.even > .sorting_1 {
-  background-color: #f9f9f9;
-}
-table.dataTable.display tbody tr.even > .sorting_2, table.dataTable.order-column.stripe tbody tr.even > .sorting_2 {
-  background-color: #fbfbfb;
-}
-table.dataTable.display tbody tr.even > .sorting_3, table.dataTable.order-column.stripe tbody tr.even > .sorting_3 {
-  background-color: #fdfdfd;
-}
-table.dataTable.display tbody tr.even.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_1 {
-  background-color: #acbad4;
-}
-table.dataTable.display tbody tr.even.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_2 {
-  background-color: #adbbd6;
-}
-table.dataTable.display tbody tr.even.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_3 {
-  background-color: #afbdd8;
-}
-table.dataTable.display tbody tr:hover > .sorting_1,
-table.dataTable.display tbody tr.odd:hover > .sorting_1,
-table.dataTable.display tbody tr.even:hover > .sorting_1, table.dataTable.order-column.hover tbody tr:hover > .sorting_1,
-table.dataTable.order-column.hover tbody tr.odd:hover > .sorting_1,
-table.dataTable.order-column.hover tbody tr.even:hover > .sorting_1 {
-  background-color: #eaeaea;
-}
-table.dataTable.display tbody tr:hover > .sorting_2,
-table.dataTable.display tbody tr.odd:hover > .sorting_2,
-table.dataTable.display tbody tr.even:hover > .sorting_2, table.dataTable.order-column.hover tbody tr:hover > .sorting_2,
-table.dataTable.order-column.hover tbody tr.odd:hover > .sorting_2,
-table.dataTable.order-column.hover tbody tr.even:hover > .sorting_2 {
-  background-color: #ebebeb;
-}
-table.dataTable.display tbody tr:hover > .sorting_3,
-table.dataTable.display tbody tr.odd:hover > .sorting_3,
-table.dataTable.display tbody tr.even:hover > .sorting_3, table.dataTable.order-column.hover tbody tr:hover > .sorting_3,
-table.dataTable.order-column.hover tbody tr.odd:hover > .sorting_3,
-table.dataTable.order-column.hover tbody tr.even:hover > .sorting_3 {
-  background-color: #eeeeee;
-}
-table.dataTable.display tbody tr:hover.selected > .sorting_1,
-table.dataTable.display tbody tr.odd:hover.selected > .sorting_1,
-table.dataTable.display tbody tr.even:hover.selected > .sorting_1, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_1,
-table.dataTable.order-column.hover tbody tr.odd:hover.selected > .sorting_1,
-table.dataTable.order-column.hover tbody tr.even:hover.selected > .sorting_1 {
-  background-color: #a1aec7;
-}
-table.dataTable.display tbody tr:hover.selected > .sorting_2,
-table.dataTable.display tbody tr.odd:hover.selected > .sorting_2,
-table.dataTable.display tbody tr.even:hover.selected > .sorting_2, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_2,
-table.dataTable.order-column.hover tbody tr.odd:hover.selected > .sorting_2,
-table.dataTable.order-column.hover tbody tr.even:hover.selected > .sorting_2 {
-  background-color: #a2afc8;
-}
-table.dataTable.display tbody tr:hover.selected > .sorting_3,
-table.dataTable.display tbody tr.odd:hover.selected > .sorting_3,
-table.dataTable.display tbody tr.even:hover.selected > .sorting_3, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_3,
-table.dataTable.order-column.hover tbody tr.odd:hover.selected > .sorting_3,
-table.dataTable.order-column.hover tbody tr.even:hover.selected > .sorting_3 {
-  background-color: #a4b2cb;
-}
-table.dataTable.no-footer {
-  border-bottom: 1px solid #111111;
-}
-table.dataTable.nowrap th, table.dataTable.nowrap td {
-  white-space: nowrap;
-}
-table.dataTable.compact thead th,
-table.dataTable.compact thead td {
-  padding: 5px 9px;
-}
-table.dataTable.compact tfoot th,
-table.dataTable.compact tfoot td {
-  padding: 5px 9px 3px 9px;
-}
-table.dataTable.compact tbody th,
-table.dataTable.compact tbody td {
-  padding: 4px 5px;
-}
-table.dataTable th.dt-left,
-table.dataTable td.dt-left {
-  text-align: left;
-}
-table.dataTable th.dt-center,
-table.dataTable td.dt-center,
-table.dataTable td.dataTables_empty {
-  text-align: center;
-}
-table.dataTable th.dt-right,
-table.dataTable td.dt-right {
-  text-align: right;
-}
-table.dataTable th.dt-justify,
-table.dataTable td.dt-justify {
-  text-align: justify;
-}
-table.dataTable th.dt-nowrap,
-table.dataTable td.dt-nowrap {
-  white-space: nowrap;
-}
-table.dataTable thead th.dt-head-left,
-table.dataTable thead td.dt-head-left,
-table.dataTable tfoot th.dt-head-left,
-table.dataTable tfoot td.dt-head-left {
-  text-align: left;
-}
-table.dataTable thead th.dt-head-center,
-table.dataTable thead td.dt-head-center,
-table.dataTable tfoot th.dt-head-center,
-table.dataTable tfoot td.dt-head-center {
-  text-align: center;
-}
-table.dataTable thead th.dt-head-right,
-table.dataTable thead td.dt-head-right,
-table.dataTable tfoot th.dt-head-right,
-table.dataTable tfoot td.dt-head-right {
-  text-align: right;
-}
-table.dataTable thead th.dt-head-justify,
-table.dataTable thead td.dt-head-justify,
-table.dataTable tfoot th.dt-head-justify,
-table.dataTable tfoot td.dt-head-justify {
-  text-align: justify;
-}
-table.dataTable thead th.dt-head-nowrap,
-table.dataTable thead td.dt-head-nowrap,
-table.dataTable tfoot th.dt-head-nowrap,
-table.dataTable tfoot td.dt-head-nowrap {
-  white-space: nowrap;
-}
-table.dataTable tbody th.dt-body-left,
-table.dataTable tbody td.dt-body-left {
-  text-align: left;
-}
-table.dataTable tbody th.dt-body-center,
-table.dataTable tbody td.dt-body-center {
-  text-align: center;
-}
-table.dataTable tbody th.dt-body-right,
-table.dataTable tbody td.dt-body-right {
-  text-align: right;
-}
-table.dataTable tbody th.dt-body-justify,
-table.dataTable tbody td.dt-body-justify {
-  text-align: justify;
-}
-table.dataTable tbody th.dt-body-nowrap,
-table.dataTable tbody td.dt-body-nowrap {
-  white-space: nowrap;
-}
-
-table.dataTable,
-table.dataTable th,
-table.dataTable td {
-  -webkit-box-sizing: content-box;
-  -moz-box-sizing: content-box;
-  box-sizing: content-box;
-}
-
-/*
- * Control feature layout
- */
-.dataTables_wrapper {
-  position: relative;
-  clear: both;
-  *zoom: 1;
-  zoom: 1;
-}
-.dataTables_wrapper .dataTables_length {
-  float: left;
-}
-.dataTables_wrapper .dataTables_filter {
-  float: right;
-  text-align: right;
-}
-.dataTables_wrapper .dataTables_filter input {
-  margin-left: 0.5em;
-}
-.dataTables_wrapper .dataTables_info {
-  clear: both;
-  float: left;
-  padding-top: 0.755em;
-}
-.dataTables_wrapper .dataTables_paginate {
-  float: right;
-  text-align: right;
-  padding-top: 0.25em;
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button {
-  box-sizing: border-box;
-  display: inline-block;
-  min-width: 1.5em;
-  padding: 0.5em 1em;
-  margin-left: 2px;
-  text-align: center;
-  text-decoration: none !important;
-  cursor: pointer;
-  *cursor: hand;
-  color: #333333 !important;
-  border: 1px solid transparent;
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button.current, .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover {
-  color: #333333 !important;
-  border: 1px solid #cacaca;
-  background-color: white;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, white), color-stop(100%, gainsboro));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, white 0%, gainsboro 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, white 0%, gainsboro 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(top, white 0%, gainsboro 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(top, white 0%, gainsboro 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to bottom, white 0%, gainsboro 100%);
-  /* W3C */
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button.disabled, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active {
-  cursor: default;
-  color: #666 !important;
-  border: 1px solid transparent;
-  background: transparent;
-  box-shadow: none;
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button:hover {
-  color: white !important;
-  border: 1px solid #111111;
-  background-color: #585858;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111111));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, #585858 0%, #111111 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, #585858 0%, #111111 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(top, #585858 0%, #111111 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(top, #585858 0%, #111111 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to bottom, #585858 0%, #111111 100%);
-  /* W3C */
-}
-.dataTables_wrapper .dataTables_paginate .paginate_button:active {
-  outline: none;
-  background-color: #2b2b2b;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);
-  /* W3C */
-  box-shadow: inset 0 0 3px #111;
-}
-.dataTables_wrapper .dataTables_processing {
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  width: 100%;
-  height: 40px;
-  margin-left: -50%;
-  margin-top: -25px;
-  padding-top: 20px;
-  text-align: center;
-  font-size: 1.2em;
-  background-color: white;
-  background: -webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255, 255, 255, 0)), color-stop(25%, rgba(255, 255, 255, 0.9)), color-stop(75%, rgba(255, 255, 255, 0.9)), color-stop(100%, rgba(255, 255, 255, 0)));
-  /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* FF3.6+ */
-  background: -ms-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* IE10+ */
-  background: -o-linear-gradient(left, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* Opera 11.10+ */
-  background: linear-gradient(to right, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 0.9) 25%, rgba(255, 255, 255, 0.9) 75%, rgba(255, 255, 255, 0) 100%);
-  /* W3C */
-}
-.dataTables_wrapper .dataTables_length,
-.dataTables_wrapper .dataTables_filter,
-.dataTables_wrapper .dataTables_info,
-.dataTables_wrapper .dataTables_processing,
-.dataTables_wrapper .dataTables_paginate {
-  color: #333333;
-}
-.dataTables_wrapper .dataTables_scroll {
-  clear: both;
-}
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody {
-  *margin-top: -1px;
-  -webkit-overflow-scrolling: touch;
-}
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th > div.dataTables_sizing,
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td > div.dataTables_sizing {
-  height: 0;
-  overflow: hidden;
-  margin: 0 !important;
-  padding: 0 !important;
-}
-.dataTables_wrapper.no-footer .dataTables_scrollBody {
-  border-bottom: 1px solid #111111;
-}
-.dataTables_wrapper.no-footer div.dataTables_scrollHead table,
-.dataTables_wrapper.no-footer div.dataTables_scrollBody table {
-  border-bottom: none;
-}
-.dataTables_wrapper:after {
-  visibility: hidden;
-  display: block;
-  content: "";
-  clear: both;
-  height: 0;
-}
-
-@media screen and (max-width: 767px) {
-  .dataTables_wrapper .dataTables_info,
-  .dataTables_wrapper .dataTables_paginate {
-    float: none;
-    text-align: center;
-  }
-  .dataTables_wrapper .dataTables_paginate {
-    margin-top: 0.5em;
-  }
-}
-@media screen and (max-width: 640px) {
-  .dataTables_wrapper .dataTables_length,
-  .dataTables_wrapper .dataTables_filter {
-    float: none;
-    text-align: center;
-  }
-  .dataTables_wrapper .dataTables_filter {
-    margin-top: 0.5em;
-  }
-}
diff --git a/airflow/www_rbac/static/jquery.dataTables.min.js b/airflow/www_rbac/static/jquery.dataTables.min.js
deleted file mode 100644
index 59d4dac21f531..0000000000000
--- a/airflow/www_rbac/static/jquery.dataTables.min.js
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * This combined file was created by the DataTables downloader builder:
- *   https://datatables.net/download
- *
- * To rebuild or modify this file with the latest versions of the included
- * software please visit:
- *   https://datatables.net/download/#bs/dt-1.10.10
- *
- * Included libraries:
- *   DataTables 1.10.10
- */
-
-/*!
- DataTables 1.10.10
- ©2008-2015 SpryMedia Ltd - datatables.net/license
-*/
-(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Y(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()),
-d[c]=e,"o"===b[1]&&Y(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Y(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Fa(a){var b=m.defaults.oLanguage,c=a.sZeroRecords;!a.sEmptyTable&&(c&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(c&&"Loading..."===b.sLoadingRecords)&&F(a,a,"sZeroRecords","sLoadingRecords");
-a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&db(a)}function eb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%":"");"boolean"===typeof a.scrollX&&(a.scrollX=
-a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&J(m.models.oSearch,a[b])}function fb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;b&&!h.isArray(b)&&(a.aDataSort=[b])}function gb(a){if(!m.__browser){var b={};m.__browser=b;var c=h("<div/>").css({position:"fixed",top:0,left:0,height:1,width:1,overflow:"hidden"}).append(h("<div/>").css({position:"absolute",top:1,left:1,
-width:100,overflow:"scroll"}).append(h("<div/>").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,m.__browser);a.oScroll.iBarWidth=m.__browser.barWidth}function hb(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==e;)a.hasOwnProperty(d)&&
-(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ga(a,b){var c=m.defaults.column,d=a.aoColumns.length,c=h.extend({},m.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},m.models.oSearch,c[d]);la(a,d,h(b).data())}function la(a,b,c){var b=a.aoColumns[b],d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=e.attr("width")||null;var f=
-(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(fb(c),J(m.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=Q(g),i=b.mRender?Q(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&
-(c(g.sort)||c(g.type)||c(g.filter));b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return R(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone,b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=
-d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function U(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Ha(a);for(var c=0,d=b.length;c<d;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&Z(a);v(a,null,"column-sizing",[a])}function $(a,b){var c=aa(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function ba(a,b){var c=aa(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function ca(a){return aa(a,
-"bVisible").length}function aa(a,b){var c=[];h.map(a.aoColumns,function(a,e){a[b]&&c.push(e)});return c}function Ia(a){var b=a.aoColumns,c=a.aoData,d=m.ext.type.detect,e,f,g,j,i,h,l,q,u;e=0;for(f=b.length;e<f;e++)if(l=b[e],u=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=d.length;g<j;g++){i=0;for(h=c.length;i<h;i++){u[i]===k&&(u[i]=B(a,i,e,"type"));q=d[g](u[i],a);if(!q&&g!==d.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function ib(a,
-b,c,d){var e,f,g,j,i,o,l=a.aoColumns;if(b)for(e=b.length-1;0<=e;e--){o=b[e];var q=o.targets!==k?o.targets:o.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Ga(a);d(q[f],o)}else if("number"===typeof q[f]&&0>q[f])d(l.length+q[f],o);else if("string"===typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&d(j,o)}}if(c){e=0;for(a=c.length;e<a;e++)d(e,c[e])}}function N(a,b,c,d){var e=a.aoData.length,f=h.extend(!0,
-{},m.models.oRow,{src:c?"dom":"data",idx:e});f._aData=b;a.aoData.push(f);for(var g=a.aoColumns,j=0,i=g.length;j<i;j++)g[j].sType=null;a.aiDisplayMaster.push(e);b=a.rowIdFn(b);b!==k&&(a.aIds[b]=f);(c||!a.oFeatures.bDeferRender)&&Ja(a,e,c,d);return e}function ma(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,e){c=Ka(a,e);return N(a,c.data,e,c.cells)})}function B(a,b,c,d){var e=a.iDraw,f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,i=f.fnGetData(g,d,{settings:a,row:b,col:c});
-if(i===k)return a.iDrawError!=e&&null===j&&(K(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b+", column "+c,4),a.iDrawError=e),j;if((i===g||null===i)&&null!==j)i=j;else if("function"===typeof i)return i.call(g);return null===i&&"display"==d?"":i}function jb(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d,{settings:a,row:b,col:c})}function La(a){return h.map(a.match(/(\\.|[^\.])+/g)||[""],function(a){return a.replace(/\\./g,".")})}function Q(a){if(h.isPlainObject(a)){var b=
-{};h.each(a,function(a,c){c&&(b[a]=Q(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=La(f);for(var i=0,o=j.length;i<o;i++){f=j[i].match(da);g=j[i].match(V);if(f){j[i]=j[i].replace(da,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=
-j.join(".");if(h.isArray(a)){i=0;for(o=a.length;i<o;i++)g.push(c(a[i],b,j))}a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(V,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===k)return k;a=a[j[i]]}}return a};return function(b,e){return c(b,e,a)}}return function(b){return b[a]}}function R(a){if(h.isPlainObject(a))return R(a._);if(null===a)return function(){};if("function"===typeof a)return function(b,d,e){a(b,"set",d,e)};if("string"===typeof a&&(-1!==a.indexOf(".")||
--1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,d,e){var e=La(e),f;f=e[e.length-1];for(var g,j,i=0,o=e.length-1;i<o;i++){g=e[i].match(da);j=e[i].match(V);if(g){e[i]=e[i].replace(da,"");a[e[i]]=[];f=e.slice();f.splice(0,i+1);g=f.join(".");if(h.isArray(d)){j=0;for(o=d.length;j<o;j++)f={},b(f,d[j],g),a[e[i]].push(f)}else a[e[i]]=d;return}j&&(e[i]=e[i].replace(V,""),a=a[e[i]](d));if(null===a[e[i]]||a[e[i]]===k)a[e[i]]={};a=a[e[i]]}if(f.match(V))a[f.replace(V,"")](d);else a[f.replace(da,"")]=
-d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Ma(a){return D(a.aoData,"_aData")}function na(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0;a.aIds={}}function oa(a,b,c){for(var d=-1,e=0,f=a.length;e<f;e++)a[e]==b?d=e:a[e]>b&&a[e]--; -1!=d&&c===k&&a.splice(d,1)}function ea(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=
-Ka(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}e._aSortData=null;e._aFilterData=null;g=a.aoColumns;if(d!==k)g[d].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;Na(a,e)}}function Ka(a,b,c,d){var e=[],f=b.firstChild,g,j,i=0,o,l=a.aoColumns,q=a._rowReadObject,d=d!==k?d:q?{}:[],u=function(a,b){if("string"===typeof a){var c=a.indexOf("@");-1!==c&&(c=a.substring(c+1),R(a)(d,b.getAttribute(c)))}},S=function(a){if(c===k||
-c===i)j=l[i],o=h.trim(a.innerHTML),j&&j._bAttrSrc?(R(j.mData._)(d,o),u(j.mData.sort,a),u(j.mData.type,a),u(j.mData.filter,a)):q?(j._setter||(j._setter=R(j.mData)),j._setter(d,o)):d[i]=o;i++};if(f)for(;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)S(f),e.push(f);f=f.nextSibling}else{e=b.anCells;f=0;for(g=e.length;f<g;f++)S(e[f])}if(b=b.firstChild?b:b.nTr)(b=b.getAttribute("id"))&&R(a.rowId)(d,b);return{data:d,cells:e}}function Ja(a,b,c,d){var e=a.aoData[b],f=e._aData,g=[],j,i,h,l,q;if(null===
-e.nTr){j=c||H.createElement("tr");e.nTr=j;e.anCells=g;j._DT_RowIndex=b;Na(a,e);l=0;for(q=a.aoColumns.length;l<q;l++){h=a.aoColumns[l];i=c?d[l]:H.createElement(h.sCellType);i._DT_CellIndex={row:b,column:l};g.push(i);if(!c||h.mRender||h.mData!==l)i.innerHTML=B(a,b,l,"display");h.sClass&&(i.className+=" "+h.sClass);h.bVisible&&!c?j.appendChild(i):!h.bVisible&&c&&i.parentNode.removeChild(i);h.fnCreatedCell&&h.fnCreatedCell.call(a.oInstance,i,B(a,b,l),f,b,l)}v(a,"aoRowCreatedCallback",null,[j,f,b])}e.nTr.setAttribute("role",
-"row")}function Na(a,b){var c=b.nTr,d=b._aData;if(c){var e=a.rowIdFn(d);e&&(c.id=e);d.DT_RowClass&&(e=d.DT_RowClass.split(" "),b.__rowc=b.__rowc?pa(b.__rowc.concat(e)):e,h(c).removeClass(b.__rowc.join(" ")).addClass(d.DT_RowClass));d.DT_RowAttr&&h(c).attr(d.DT_RowAttr);d.DT_RowData&&h(c).data(d.DT_RowData)}}function kb(a){var b,c,d,e,f,g=a.nTHead,j=a.nTFoot,i=0===h("th, td",g).length,o=a.oClasses,l=a.aoColumns;i&&(e=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],d=h(f.nTh).addClass(f.sClass),
-i&&d.appendTo(e),a.oFeatures.bSort&&(d.addClass(f.sSortingClass),!1!==f.bSortable&&(d.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Oa(a,f.nTh,b))),f.sTitle!=d[0].innerHTML&&d.html(f.sTitle),Pa(a,"header")(a,d,f,o);i&&fa(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(o.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(o.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}
-function ga(a,b,c){var d,e,f,g=[],j=[],i=a.aoColumns.length,o;if(b){c===k&&(c=!1);d=0;for(e=b.length;d<e;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);j.push([])}d=0;for(e=g.length;d<e;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(o=i=1,j[d][f]===k){a.appendChild(g[d][f].cell);for(j[d][f]=1;g[d+i]!==k&&g[d][f].cell==g[d+i][f].cell;)j[d+i][f]=1,i++;for(;g[d][f+o]!==k&&g[d][f].cell==g[d][f+o].cell;){for(c=
-0;c<i;c++)j[d+c][f+o]=1;o++}h(g[d][f].cell).attr("rowspan",i).attr("colspan",o)}}}}function O(a){var b=v(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,d=a.asStripeClasses,e=d.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==y(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,o=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&
-!lb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:o;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ja(a,l);l=q.nTr;if(0!==e){var u=d[c%e];q._sRowStripe!=u&&(h(l).removeClass(q._sRowStripe).addClass(u),q._sRowStripe=u)}v(a,"aoRowCallback",null,[l,q._aData,c,j]);b.push(l);c++}}else c=f.sZeroRecords,1==a.iDraw&&"ajax"==y(a)?c=f.sLoadingRecords:f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":e?d[0]:""}).append(h("<td />",{valign:"top",colSpan:ca(a),
-"class":a.oClasses.sRowEmpty}).html(c))[0];v(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ma(a),g,o,i]);v(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ma(a),g,o,i]);d=h(a.nTBody);d.children().detach();d.append(h(b));v(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;c.bSort&&mb(a);d?ha(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;O(a);a._drawHold=
-!1}function nb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),d=a.oFeatures,e=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,o,l,q,u=0;u<f.length;u++){g=null;j=f[u];if("<"==j){i=h("<div/>")[0];o=f[u+1];if("'"==o||'"'==o){l="";for(q=2;f[u+q]!=o;)l+=f[u+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(o=l.split("."),
-i.id=o[0].substr(1,o[0].length-1),i.className=o[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;u+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=ob(a);else if("f"==j&&d.bFilter)g=pb(a);else if("r"==j&&d.bProcessing)g=qb(a);else if("t"==j)g=rb(a);else if("i"==j&&d.bInfo)g=sb(a);else if("p"==j&&d.bPaginate)g=tb(a);else if(0!==m.ext.feature.length){i=m.ext.feature;q=0;for(o=i.length;q<o;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&
-(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),e.append(g))}c.replaceWith(e);a.nHolding=null}function fa(a,b){var c=h(b).children("tr"),d,e,f,g,j,i,o,l,q,u;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<i;f++){d=c[f];for(e=d.firstChild;e;){if("TD"==e.nodeName.toUpperCase()||"TH"==e.nodeName.toUpperCase()){l=1*e.getAttribute("colspan");q=1*e.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;o=g;u=1===l?!0:!1;for(j=0;j<l;j++)for(g=
-0;g<q;g++)a[f+g][o+j]={cell:e,unique:u},a[f+g].nTr=d}e=e.nextSibling}}}function qa(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],fa(c,b)));for(var b=0,e=c.length;b<e;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function ra(a,b,c){v(a,"aoServerParams","serverParams",[b]);if(b&&h.isArray(b)){var d={},e=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(e);c?(c=c[0],d[c]||(d[c]=[]),d[c].push(b.value)):d[b.name]=b.value});b=d}var f,g=a.ajax,
-j=a.oInstance,i=function(b){v(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var o=h.isFunction(f)?f(b,a):f,b=h.isFunction(f)&&o?o:h.extend(!0,b,o);delete g.data}o={data:b,success:function(b){var c=b.error||b.sError;c&&K(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,c){var d=v(a,null,"xhr",[a,null,a.jqXHR]);-1===h.inArray(!0,d)&&("parsererror"==c?K(a,0,"Invalid JSON response",1):4===b.readyState&&K(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=
-b;v(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(o,{url:g||a.sAjaxSource})):h.isFunction(g)?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(o,g)),g.data=f)}function lb(a){return a.bAjaxDataGet?(a.iDraw++,C(a,!0),ra(a,ub(a),function(b){vb(a,b)}),!1):!0}function ub(a){var b=a.aoColumns,c=b.length,d=a.oFeatures,e=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,o,
-l,q=W(a);g=a._iDisplayStart;i=!1!==d.bPaginate?a._iDisplayLength:-1;var k=function(a,b){j.push({name:a,value:b})};k("sEcho",a.iDraw);k("iColumns",c);k("sColumns",D(b,"sName").join(","));k("iDisplayStart",g);k("iDisplayLength",i);var S={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:e.sSearch,regex:e.bRegex}};for(g=0;g<c;g++)o=b[g],l=f[g],i="function"==typeof o.mData?"function":o.mData,S.columns.push({data:i,name:o.sName,searchable:o.bSearchable,orderable:o.bSortable,search:{value:l.sSearch,
-regex:l.bRegex}}),k("mDataProp_"+g,i),d.bFilter&&(k("sSearch_"+g,l.sSearch),k("bRegex_"+g,l.bRegex),k("bSearchable_"+g,o.bSearchable)),d.bSort&&k("bSortable_"+g,o.bSortable);d.bFilter&&(k("sSearch",e.sSearch),k("bRegex",e.bRegex));d.bSort&&(h.each(q,function(a,b){S.order.push({column:b.col,dir:b.dir});k("iSortCol_"+a,b.col);k("sSortDir_"+a,b.dir)}),k("iSortingCols",q.length));b=m.ext.legacy.ajax;return null===b?a.sAjaxSource?j:S:b?j:S}function vb(a,b){var c=sa(a,b),d=b.sEcho!==k?b.sEcho:b.draw,e=
-b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(d){if(1*d<a.iDraw)return;a.iDraw=1*d}na(a);a._iRecordsTotal=parseInt(e,10);a._iRecordsDisplay=parseInt(f,10);d=0;for(e=c.length;d<e;d++)N(a,c[d]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;O(a);a._bInitComplete||ta(a,b);a.bAjaxDataGet=!0;C(a,!1)}function sa(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||
-b[c]:""!==c?Q(c)(b):b}function pb(a){var b=a.oClasses,c=a.sTableId,d=a.oLanguage,e=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),f=function(){var b=!this.value?"":this.value;b!=e.sSearch&&(ha(a,{sSearch:b,bRegex:e.bRegex,bSmart:e.bSmart,bCaseInsensitive:e.bCaseInsensitive}),a._iDisplayStart=0,O(a))},g=null!==
-a.searchDelay?a.searchDelay:"ssp"===y(a)?400:0,i=h("input",b).val(e.sSearch).attr("placeholder",d.sSearchPlaceholder).bind("keyup.DT search.DT input.DT paste.DT cut.DT",g?ua(f,g):f).bind("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==H.activeElement&&i.val(e.sSearch)}catch(d){}});return b[0]}function ha(a,b,c){var d=a.oPreviousSearch,e=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;
-d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};Ia(a);if("ssp"!=y(a)){wb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<e.length;b++)xb(a,e[b].sSearch,b,e[b].bEscapeRegex!==k?!e[b].bEscapeRegex:e[b].bRegex,e[b].bSmart,e[b].bCaseInsensitive);yb(a)}else f(b);a.bFiltered=!0;v(a,null,"search",[a])}function yb(a){for(var b=m.ext.search,c=a.aiDisplay,d,e,f=0,g=b.length;f<g;f++){for(var j=[],i=0,o=c.length;i<o;i++)e=c[i],d=a.aoData[e],b[f](a,
-d._aFilterData,e,d._aData,i)&&j.push(e);c.length=0;h.merge(c,j)}}function xb(a,b,c,d,e,f){if(""!==b)for(var g=a.aiDisplay,d=Qa(b,d,e,f),e=g.length-1;0<=e;e--)b=a.aoData[g[e]]._aFilterData[c],d.test(b)||g.splice(e,1)}function wb(a,b,c,d,e,f){var d=Qa(b,d,e,f),e=a.oPreviousSearch.sSearch,f=a.aiDisplayMaster,g;0!==m.ext.search.length&&(c=!0);g=zb(a);if(0>=b.length)a.aiDisplay=f.slice();else{if(g||c||e.length>b.length||0!==b.indexOf(e)||a.bSorted)a.aiDisplay=f.slice();b=a.aiDisplay;for(c=b.length-1;0<=
-c;c--)d.test(a.aoData[b[c]]._sFilterRow)||b.splice(c,1)}}function Qa(a,b,c,d){a=b?a:va(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',"")}).join(")(?=.*?")+").*$");return RegExp(a,d?"i":"")}function va(a){return a.replace(Yb,"\\$1")}function zb(a){var b=a.aoColumns,c,d,e,f,g,j,i,h,l=m.ext.type.search;c=!1;d=0;for(f=a.aoData.length;d<f;d++)if(h=a.aoData[d],!h._aFilterData){j=[];e=0;for(g=b.length;e<
-g;e++)c=b[e],c.bSearchable?(i=B(a,d,e,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(wa.innerHTML=i,i=Zb?wa.textContent:wa.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);h._aFilterData=j;h._sFilterRow=j.join("  ");c=!0}return c}function Ab(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,caseInsensitive:a.bCaseInsensitive}}function Bb(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,
-bCaseInsensitive:a.caseInsensitive}}function sb(a){var b=a.sTableId,c=a.aanFeatures.i,d=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Cb,sName:"information"}),d.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",b+"_info"));return d[0]}function Cb(a){var b=a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,d=a._iDisplayStart+1,e=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&
-(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Db(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,d,e,f,g,j));h(b).html(j)}}function Db(a,b){var c=a.fnFormatNumber,d=a._iDisplayStart+1,e=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===e;return b.replace(/_START_/g,c.call(a,d)).replace(/_END_/g,c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(d/e))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/
-e)))}function ia(a){var b,c,d=a.iInitDisplayStart,e=a.aoColumns,f;c=a.oFeatures;var g=a.bDeferLoading;if(a.bInitialised){nb(a);kb(a);ga(a,a.aoHeader);ga(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Ha(a);b=0;for(c=e.length;b<c;b++)f=e[b],f.sWidth&&(f.nTh.style.width=w(f.sWidth));v(a,null,"preInit",[a]);T(a);e=y(a);if("ssp"!=e||g)"ajax"==e?ra(a,[],function(c){var f=sa(a,c);for(b=0;b<f.length;b++)N(a,f[b]);a.iInitDisplayStart=d;T(a);C(a,!1);ta(a,c)},a):(C(a,!1),ta(a))}else setTimeout(function(){ia(a)},200)}
-function ta(a,b){a._bInitComplete=!0;(b||a.oInit.aaData)&&U(a);v(a,null,"plugin-init",[a,b]);v(a,"aoInitComplete","init",[a,b])}function Ra(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Sa(a);v(a,null,"length",[a,c])}function ob(a){for(var b=a.oClasses,c=a.sTableId,d=a.aLengthMenu,e=h.isArray(d[0]),f=e?d[0]:d,d=e?d[1]:d,e=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)e[0][g]=new Option(d[g],f[g]);var i=h("<div><label/></div>").addClass(b.sLength);
-a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).bind("change.DT",function(){Ra(a,h(this).val());O(a)});h(a.nTable).bind("length.dt.DT",function(b,c,d){a===c&&h("select",i).val(d)});return i[0]}function tb(a){var b=a.sPaginationType,c=m.ext.pager[b],d="function"===typeof c,e=function(a){O(a)},b=h("<div/>").addClass(a.oClasses.sPaging+b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+
-"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;l<k;l++)Pa(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,e)},sName:"pagination"}));return b}function Ta(a,b,c){var d=a._iDisplayStart,e=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===e?d=0:"number"===typeof b?(d=b*e,d>f&&(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==
-b?d+e<f&&(d+=e):"last"==b?d=Math.floor((f-1)/e)*e:K(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==d;a._iDisplayStart=d;b&&(v(a,null,"page",[a]),c&&O(a));return b}function qb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");v(a,null,"processing",[a,b])}function rb(a){var b=h(a.nTable);b.attr("role",
-"grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),o=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,width:d?!d?null:w(d):"100%"}).append(h("<div/>",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||
-"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:w(d)}).append(b));l&&i.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:w(d):"100%"}).append(h("<div/>",{"class":f.sScrollFootInner}).append(o.removeAttr("id").css("margin-left",0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),
-k=b[0],f=b[1],u=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(u.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=u;a.aoDrawCallback.push({fn:Z,sName:"scrolling"});return i[0]}function Z(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth,f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,o=j.children("table"),j=a.nScrollBody,l=h(j),q=j.style,u=h(a.nScrollFoot).children("div"),
-m=u.children("table"),n=h(a.nTHead),p=h(a.nTable),t=p[0],v=t.style,r=a.nTFoot?h(a.nTFoot):null,Eb=a.oBrowser,Ua=Eb.bScrollOversize,s,L,P,x,y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!==L&&a.scrollBarVis!==k)a.scrollBarVis=L,U(a);else{a.scrollBarVis=L;p.children("thead, tfoot").remove();x=n.clone().prependTo(p);n=n.find("tr");L=x.find("tr");x.find("th, td").removeAttr("tabindex");
-r&&(P=r.clone().prependTo(p),s=r.find("tr"),P=P.find("tr"));c||(q.width="100%",f[0].style.width="100%");h.each(qa(a,x),function(b,c){B=$(a,b);c.style.width=a.aoColumns[B].sWidth});r&&I(function(a){a.style.width=""},P);f=p.outerWidth();if(""===c){v.width="100%";if(Ua&&(p.find("tbody").height()>j.offsetHeight||"scroll"==l.css("overflow-y")))v.width=w(p.outerWidth()-b);f=p.outerWidth()}else""!==d&&(v.width=w(d),f=p.outerWidth());I(C,L);I(function(a){A.push(a.innerHTML);y.push(w(h(a).css("width")))},
-L);I(function(a,b){a.style.width=y[b]},n);h(L).height(0);r&&(I(C,P),I(function(a){z.push(w(h(a).css("width")))},P),I(function(a,b){a.style.width=z[b]},s),h(P).height(0));I(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+A[b]+"</div>";a.style.width=y[b]},L);r&&I(function(a,b){a.innerHTML="";a.style.width=z[b]},P);if(p.outerWidth()<f){s=j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(Ua&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))v.width=
-w(s-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else s="100%";q.width=w(s);g.width=w(s);r&&(a.nScrollFoot.style.width=w(s));!e&&Ua&&(q.height=w(t.offsetHeight+b));c=p.outerWidth();o[0].style.width=w(c);i.width=w(c);d=p.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+(Eb.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";r&&(m[0].style.width=w(c),u[0].style.width=w(c),u[0].style[e]=d?b+"px":"0px");l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=
-0}}function I(a,b,c){for(var d=0,e=0,f=b.length,g,j;e<f;){g=b[e].firstChild;for(j=c?c[e].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,d):a(g,d),d++),g=g.nextSibling,j=c?j.nextSibling:null;e++}}function Ha(a){var b=a.nTable,c=a.aoColumns,d=a.oScroll,e=d.sY,f=d.sX,g=d.sXInner,j=c.length,i=aa(a,"bVisible"),o=h("th",a.nTHead),l=b.getAttribute("width"),k=b.parentNode,u=!1,m,n,p=a.oBrowser,d=p.bScrollOversize;(m=b.style.width)&&-1!==m.indexOf("%")&&(l=m);for(m=0;m<i.length;m++)n=c[i[m]],null!==n.sWidth&&
-(n.sWidth=Fb(n.sWidthOrig,k),u=!0);if(d||!u&&!f&&!e&&j==ca(a)&&j==o.length)for(m=0;m<j;m++)i=$(a,m),null!==i&&(c[i].sWidth=w(o.eq(m).width()));else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var t=h("<tr/>").appendTo(j.find("tbody"));j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");o=qa(a,j.find("thead")[0]);for(m=0;m<i.length;m++)n=c[i[m]],o[m].style.width=null!==n.sWidthOrig&&
-""!==n.sWidthOrig?w(n.sWidthOrig):"",n.sWidthOrig&&f&&h(o[m]).append(h("<div/>").css({width:n.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(m=0;m<i.length;m++)u=i[m],n=c[u],h(Gb(a,u)).clone(!1).append(n.sContentPadding).appendTo(t);n=h("<div/>").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()<k.clientWidth&&l&&j.width(k.clientWidth)):e?j.width(k.clientWidth):
-l&&j.width(l);for(m=e=0;m<i.length;m++)k=h(o[m]),g=k.outerWidth()-k.width(),k=p.bBounding?Math.ceil(o[m].getBoundingClientRect().width):k.outerWidth(),e+=k,c[i[m]].sWidth=w(k-g);b.style.width=w(e);n.remove()}l&&(b.style.width=w(l));if((l||f)&&!a._reszEvt)b=function(){h(E).bind("resize.DT-"+a.sInstance,ua(function(){U(a)}))},d?setTimeout(b,1E3):b(),a._reszEvt=!0}function ua(a,b){var c=b!==k?b:200,d,e;return function(){var b=this,g=+new Date,j=arguments;d&&g<d+c?(clearTimeout(e),e=setTimeout(function(){d=
-k;a.apply(b,j)},c)):(d=g,a.apply(b,j))}}function Fb(a,b){if(!a)return 0;var c=h("<div/>").css("width",w(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Gb(a,b){var c=Hb(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("<td/>").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Hb(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;f<g;f++)c=B(a,f,b,"display")+"",c=c.replace($b,""),c=c.replace(/&nbsp;/g," "),c.length>d&&(d=c.length,e=f);return e}function w(a){return null===
-a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function W(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var o=[];f=function(a){a.length&&!h.isArray(a[0])?o.push(a):h.merge(o,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<o.length;a++){i=o[a][0];f=e[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=e[g].sType||"string",o[a]._idx===k&&(o[a]._idx=h.inArray(o[a][1],e[g].asSorting)),d.push({src:i,col:g,dir:o[a][1],
-index:o[a]._idx,type:j,formatter:m.ext.type.order[j+"-pre"]})}return d}function mb(a){var b,c,d=[],e=m.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;Ia(a);h=W(a);b=0;for(c=h.length;b<c;b++)j=h[b],j.formatter&&g++,Ib(a,j.col);if("ssp"!=y(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)d[i[b]]=b;g===h.length?i.sort(function(a,b){var c,e,g,j,i=h.length,k=f[a]._aSortData,m=f[b]._aSortData;for(g=0;g<i;g++)if(j=h[g],c=k[j.col],e=m[j.col],c=c<e?-1:c>e?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];
-return c<e?-1:c>e?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,m=f[a]._aSortData,p=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=m[i.col],g=p[i.col],i=e[i.type+"-"+i.dir]||e["string-"+i.dir],c=i(c,g),0!==c)return c;c=d[a];g=d[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Jb(a){for(var b,c,d=a.aoColumns,e=W(a),a=a.oLanguage.oAria,f=0,g=d.length;f<g;f++){c=d[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<e.length&&e[0].col==f?(i.setAttribute("aria-sort",
-"asc"==e[0].dir?"ascending":"descending"),c=j[e[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",b)}}function Va(a,b,c,d){var e=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof e[0]&&(e=a.aaSorting=[e]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,D(e,"0")),-1!==c?(b=g(e[c],!0),null===b&&1===e.length&&(b=0),null===b?e.splice(c,1):(e[c][1]=f[b],e[c]._idx=
-b)):(e.push([b,f[0],0]),e[e.length-1]._idx=0)):e.length&&e[0][0]==b?(b=g(e[0]),e.length=1,e[0][1]=f[b],e[0]._idx=b):(e.length=0,e.push([b,f[0]]),e[0]._idx=0);T(a);"function"==typeof d&&d(a)}function Oa(a,b,c,d){var e=a.aoColumns[c];Wa(b,{},function(b){!1!==e.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Va(a,c,b.shiftKey,d);"ssp"!==y(a)&&C(a,!1)},0)):Va(a,c,b.shiftKey,d))})}function xa(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,d=W(a),e=a.oFeatures,f,g;if(e.bSort&&e.bSortClasses){e=
-0;for(f=b.length;e<f;e++)g=b[e].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>e?e+1:3));e=0;for(f=d.length;e<f;e++)g=d[e].src,h(D(a.aoData,"anCells",g)).addClass(c+(2>e?e+1:3))}a.aLastSort=d}function Ib(a,b){var c=a.aoColumns[b],d=m.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,ba(a,b)));for(var f,g=m.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],c._aSortData||(c._aSortData=[]),!c._aSortData[b]||d)f=d?e[j]:B(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function ya(a){if(a.oFeatures.bStateSave&&
-!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),search:Ab(a.oPreviousSearch),columns:h.map(a.aoColumns,function(b,d){return{visible:b.bVisible,search:Ab(a.aoPreSearchCols[d])}})};v(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,b)}}function Kb(a){var b,c,d=a.aoColumns;if(a.oFeatures.bStateSave){var e=a.fnStateLoadCallback.call(a.oInstance,a);if(e&&e.time&&(b=v(a,"aoStateLoadParams",
-"stateLoadParams",[a,e]),-1===h.inArray(!1,b)&&(b=a.iStateDuration,!(0<b&&e.time<+new Date-1E3*b)&&d.length===e.columns.length))){a.oLoadedState=h.extend(!0,{},e);e.start!==k&&(a._iDisplayStart=e.start,a.iInitDisplayStart=e.start);e.length!==k&&(a._iDisplayLength=e.length);e.order!==k&&(a.aaSorting=[],h.each(e.order,function(b,c){a.aaSorting.push(c[0]>=d.length?[0,c[1]]:c)}));e.search!==k&&h.extend(a.oPreviousSearch,Bb(e.search));b=0;for(c=e.columns.length;b<c;b++){var f=e.columns[b];f.visible!==
-k&&(d[b].bVisible=f.visible);f.search!==k&&h.extend(a.aoPreSearchCols[b],Bb(f.search))}v(a,"aoStateLoaded","stateLoaded",[a,e])}}}function za(a){var b=m.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function K(a,b,c,d){c="DataTables warning: "+(a?"table id="+a.sTableId+" - ":"")+c;d&&(c+=". For more information about this error, please see http://datatables.net/tn/"+d);if(b)E.console&&console.log&&console.log(c);else if(b=m.ext,b=b.sErrMode||b.errMode,a&&v(a,null,"error",[a,d,c]),"alert"==
-b)alert(c);else{if("throw"==b)throw Error(c);"function"==typeof b&&b(a,d,c)}}function F(a,b,c,d){h.isArray(c)?h.each(c,function(c,d){h.isArray(d)?F(a,b,d[0],d[1]):F(a,b,d)}):(d===k&&(d=c),b[c]!==k&&(a[d]=b[c]))}function Lb(a,b,c){var d,e;for(e in b)b.hasOwnProperty(e)&&(d=b[e],h.isPlainObject(d)?(h.isPlainObject(a[e])||(a[e]={}),h.extend(!0,a[e],d)):a[e]=c&&"data"!==e&&"aaData"!==e&&h.isArray(d)?d.slice():d);return a}function Wa(a,b,c){h(a).bind("click.DT",b,function(b){a.blur();c(b)}).bind("keypress.DT",
-b,function(a){13===a.which&&(a.preventDefault(),c(a))}).bind("selectstart.DT",function(){return!1})}function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function v(a,b,c,d){var e=[];b&&(e=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,d)}));null!==c&&(b=h.Event(c+".dt"),h(a.nTable).trigger(b,d),e.push(b.result));return e}function Sa(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),d=a._iDisplayLength;b>=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Pa(a,b){var c=
-a.renderer,d=m.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"===typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function Aa(a,b){var c=[],c=Mb.numbers_length,d=Math.floor(c/2);b<=c?c=X(0,b):a<=d?(c=X(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=X(b-(c-2),b):(c=X(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function db(a){h.each({num:function(b){return Ba(b,
-a)},"num-fmt":function(b){return Ba(b,a,Xa)},"html-num":function(b){return Ba(b,a,Ca)},"html-num-fmt":function(b){return Ba(b,a,Ca,Xa)}},function(b,c){s.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(s.type.search[b+a]=s.type.search.html)})}function Nb(a){return function(){var b=[za(this[m.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return m.ext.internal[a].apply(this,b)}}var m,s,t,p,r,Ya={},Ob=/[\r\n]/g,Ca=/<.*?>/g,ac=/^[\w\+\-]/,bc=/[\w\+\-]$/,Yb=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)",
-"g"),Xa=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfk]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Pb=function(a){var b=parseInt(a,10);return!isNaN(b)&&isFinite(a)?b:null},Qb=function(a,b){Ya[b]||(Ya[b]=RegExp(va(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Ya[b],"."):a},Za=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Qb(a,b));c&&d&&(a=a.replace(Xa,""));return!isNaN(parseFloat(a))&&isFinite(a)},Rb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===
-typeof a)?null:Za(a.replace(Ca,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;e<f;e++)a[e]&&a[e][b]&&d.push(a[e][b][c]);else for(;e<f;e++)a[e]&&d.push(a[e][b]);return d},ja=function(a,b,c,d){var e=[],f=0,g=b.length;if(d!==k)for(;f<g;f++)a[b[f]][c]&&e.push(a[b[f]][c][d]);else for(;f<g;f++)e.push(a[b[f]][c]);return e},X=function(a,b){var c=[],d;b===k?(b=0,d=a):(d=b,b=a);for(var e=b;e<d;e++)c.push(e);return c},Sb=function(a){for(var b=[],c=0,d=a.length;c<d;c++)a[c]&&b.push(a[c]);
-return b},pa=function(a){var b=[],c,d,e=a.length,f,g=0;d=0;a:for(;d<e;d++){c=a[d];for(f=0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b},A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},da=/\[.*?\]$/,V=/\(\)$/,wa=h("<div>")[0],Zb=wa.textContent!==k,$b=/<.*?>/g;m=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new t(za(this[s.iApiIndex])):new t(this)};this.fnAddData=function(a,b){var c=this.api(!0),
-d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&Z(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],
-h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data():c.row(a).data()||null}return c.data().toArray()};
-this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};this.fnPageChange=function(a,b){var c=this.api(!0).page(a);
-(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return za(this[s.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust();(d===k||d)&&h.draw();return 0};this.fnVersionCheck=
-s.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=s.internal;for(var e in m.ext.internal)e&&(this[e]=Nb(e));this.each(function(){var e={},e=1<d?Lb(e,a,!0):a,g=0,j,i=this.getAttribute("id"),o=!1,l=m.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())K(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{eb(l);fb(l.column);J(l,l,!0);J(l.column,l.column,!0);J(l,h.extend(e,q.data()));var u=m.settings,g=0;for(j=u.length;g<j;g++){var p=u[g];if(p.nTable==
-this||p.nTHead.parentNode==this||p.nTFoot&&p.nTFoot.parentNode==this){g=e.bRetrieve!==k?e.bRetrieve:l.bRetrieve;if(c||g)return p.oInstance;if(e.bDestroy!==k?e.bDestroy:l.bDestroy){p.oInstance.fnDestroy();break}else{K(p,0,"Cannot reinitialise DataTable",3);return}}if(p.sTableId==this.id){u.splice(g,1);break}}if(null===i||""===i)this.id=i="DataTables_Table_"+m.ext._unique++;var n=h.extend(!0,{},m.models.oSettings,{sDestroyWidth:q[0].style.width,sInstance:i,sTableId:i});n.nTable=this;n.oApi=b.internal;
-n.oInit=e;u.push(n);n.oInstance=1===b.length?b:q.dataTable();eb(e);e.oLanguage&&Fa(e.oLanguage);e.aLengthMenu&&!e.iDisplayLength&&(e.iDisplayLength=h.isArray(e.aLengthMenu[0])?e.aLengthMenu[0][0]:e.aLengthMenu[0]);e=Lb(h.extend(!0,{},l),e);F(n.oFeatures,e,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));F(n,e,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu",
-"sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"],["bJQueryUI","bJUI"]]);F(n.oScroll,e,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);F(n.oLanguage,e,"fnInfoCallback");z(n,"aoDrawCallback",
-e.fnDrawCallback,"user");z(n,"aoServerParams",e.fnServerParams,"user");z(n,"aoStateSaveParams",e.fnStateSaveParams,"user");z(n,"aoStateLoadParams",e.fnStateLoadParams,"user");z(n,"aoStateLoaded",e.fnStateLoaded,"user");z(n,"aoRowCallback",e.fnRowCallback,"user");z(n,"aoRowCreatedCallback",e.fnCreatedRow,"user");z(n,"aoHeaderCallback",e.fnHeaderCallback,"user");z(n,"aoFooterCallback",e.fnFooterCallback,"user");z(n,"aoInitComplete",e.fnInitComplete,"user");z(n,"aoPreDrawCallback",e.fnPreDrawCallback,
-"user");n.rowIdFn=Q(e.rowId);gb(n);i=n.oClasses;e.bJQueryUI?(h.extend(i,m.ext.oJUIClasses,e.oClasses),e.sDom===l.sDom&&"lfrtip"===l.sDom&&(n.sDom='<"H"lfr>t<"F"ip>'),n.renderer)?h.isPlainObject(n.renderer)&&!n.renderer.header&&(n.renderer.header="jqueryui"):n.renderer="jqueryui":h.extend(i,m.ext.classes,e.oClasses);q.addClass(i.sTable);n.iInitDisplayStart===k&&(n.iInitDisplayStart=e.iDisplayStart,n._iDisplayStart=e.iDisplayStart);null!==e.iDeferLoading&&(n.bDeferLoading=!0,g=h.isArray(e.iDeferLoading),
-n._iRecordsDisplay=g?e.iDeferLoading[0]:e.iDeferLoading,n._iRecordsTotal=g?e.iDeferLoading[1]:e.iDeferLoading);var t=n.oLanguage;h.extend(!0,t,e.oLanguage);""!==t.sUrl&&(h.ajax({dataType:"json",url:t.sUrl,success:function(a){Fa(a);J(l.oLanguage,a);h.extend(true,t,a);ia(n)},error:function(){ia(n)}}),o=!0);null===e.asStripeClasses&&(n.asStripeClasses=[i.sStripeOdd,i.sStripeEven]);var g=n.asStripeClasses,r=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(g,function(a){return r.hasClass(a)}))&&
-(h("tbody tr",this).removeClass(g.join(" ")),n.asDestroyStripes=g.slice());u=[];g=this.getElementsByTagName("thead");0!==g.length&&(fa(n.aoHeader,g[0]),u=qa(n));if(null===e.aoColumns){p=[];g=0;for(j=u.length;g<j;g++)p.push(null)}else p=e.aoColumns;g=0;for(j=p.length;g<j;g++)Ga(n,u?u[g]:null);ib(n,e.aoColumnDefs,p,function(a,b){la(n,a,b)});if(r.length){var s=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h(r[0]).children("th, td").each(function(a,b){var c=n.aoColumns[a];if(c.mData===
-a){var d=s(b,"sort")||s(b,"order"),e=s(b,"filter")||s(b,"search");if(d!==null||e!==null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};la(n,a)}}})}var w=n.oFeatures;e.bStateSave&&(w.bStateSave=!0,Kb(n,e),z(n,"aoDrawCallback",ya,"state_save"));if(e.aaSorting===k){u=n.aaSorting;g=0;for(j=u.length;g<j;g++)u[g][1]=n.aoColumns[g].asSorting[0]}xa(n);w.bSort&&z(n,"aoDrawCallback",function(){if(n.bSorted){var a=W(n),b={};h.each(a,function(a,
-c){b[c.src]=c.dir});v(n,null,"order",[n,a,b]);Jb(n)}});z(n,"aoDrawCallback",function(){(n.bSorted||y(n)==="ssp"||w.bDeferRender)&&xa(n)},"sc");g=q.children("caption").each(function(){this._captionSide=q.css("caption-side")});j=q.children("thead");0===j.length&&(j=h("<thead/>").appendTo(this));n.nTHead=j[0];j=q.children("tbody");0===j.length&&(j=h("<tbody/>").appendTo(this));n.nTBody=j[0];j=q.children("tfoot");if(0===j.length&&0<g.length&&(""!==n.oScroll.sX||""!==n.oScroll.sY))j=h("<tfoot/>").appendTo(this);
-0===j.length||0===j.children().length?q.addClass(i.sNoFooter):0<j.length&&(n.nTFoot=j[0],fa(n.aoFooter,n.nTFoot));if(e.aaData)for(g=0;g<e.aaData.length;g++)N(n,e.aaData[g]);else(n.bDeferLoading||"dom"==y(n))&&ma(n,h(n.nTBody).children("tr"));n.aiDisplay=n.aiDisplayMaster.slice();n.bInitialised=!0;!1===o&&ia(n)}});b=null;return this};var Tb=[],x=Array.prototype,cc=function(a){var b,c,d=m.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=
-h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};t=function(a,b){if(!(this instanceof t))return new t(a,b);var c=[],d=function(a){(a=cc(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;e<f;e++)d(a[e]);else d(a);this.context=pa(c);b&&h.merge(this,b);this.selector={rows:null,cols:null,opts:null};
-t.extend(this,this,Tb)};m.Api=t;h.extend(t.prototype,{any:function(){return 0!==this.count()},concat:x.concat,context:[],count:function(){return this.flatten().length},each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=this.context;return b.length>a?new t(b[a],this[a]):null},filter:function(a){var b=[];if(x.filter)b=x.filter.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new t(this.context,
-b)},flatten:function(){var a=[];return new t(this.context,a.concat.apply(a,this.toArray()))},join:x.join,indexOf:x.indexOf||function(a,b){for(var c=b||0,d=this.length;c<d;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,d){var e=[],f,g,h,i,o,l=this.context,m,p,r=this.selector;"string"===typeof a&&(d=c,c=b,b=a,a=!1);g=0;for(h=l.length;g<h;g++){var n=new t(l[g]);if("table"===b)f=c.call(n,l[g],g),f!==k&&e.push(f);else if("columns"===b||"rows"===b)f=c.call(n,l[g],this[g],g),f!==k&&e.push(f);
-else if("column"===b||"column-rows"===b||"row"===b||"cell"===b){p=this[g];"column-rows"===b&&(m=Da(l[g],r.opts));i=0;for(o=p.length;i<o;i++)f=p[i],f="cell"===b?c.call(n,l[g],f.row,f.column,g,i):c.call(n,l[g],f,g,i,m),f!==k&&e.push(f)}}return e.length||d?(a=new t(l,a?e.concat.apply([],e):e),b=a.selector,b.rows=r.rows,b.cols=r.cols,b.opts=r.opts,a):this},lastIndexOf:x.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(x.map)b=
-x.map.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)b.push(a.call(this,this[c],c));return new t(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:x.pop,push:x.push,reduce:x.reduce||function(a,b){return hb(this,a,b,0,this.length,1)},reduceRight:x.reduceRight||function(a,b){return hb(this,a,b,this.length-1,-1,-1)},reverse:x.reverse,selector:null,shift:x.shift,sort:x.sort,splice:x.splice,toArray:function(){return x.slice.call(this)},to$:function(){return h(this)},
-toJQuery:function(){return h(this)},unique:function(){return new t(this.context,pa(this))},unshift:x.unshift});t.extend=function(a,b,c){if(c.length&&b&&(b instanceof t||b.__dt_wrapper)){var d,e,f,g=function(a,b,c){return function(){var d=b.apply(a,arguments);t.extend(d,d,c.methodExt);return d}};d=0;for(e=c.length;d<e;d++)f=c[d],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,t.extend(a,b[f.name],f.propExt)}};t.register=p=function(a,b){if(h.isArray(a))for(var c=
-0,d=a.length;c<d;c++)t.register(a[c],b);else for(var e=a.split("."),f=Tb,g,j,c=0,d=e.length;c<d;c++){g=(j=-1!==e[c].indexOf("()"))?e[c].replace("()",""):e[c];var i;a:{i=0;for(var k=f.length;i<k;i++)if(f[i].name===g){i=f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===d-1?i.val=b:f=j?i.methodExt:i.propExt}};t.registerPlural=r=function(a,b,c){t.register(a,c);t.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof t?a.length?h.isArray(a[0])?
-new t(a.context,a[0]):a[0]:k:a})};p("tables()",function(a){var b;if(a){b=t;var c=this.context;if("number"===typeof a)a=[c[a]];else var d=h.map(c,function(a){return a.nTable}),a=h(d).filter(a).map(function(){var a=h.inArray(this,d);return c[a]}).toArray();b=new b(a)}else b=this;return b});p("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new t(b[0]):a});r("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});r("tables().body()",
-"table().body()",function(){return this.iterator("table",function(a){return a.nTBody},1)});r("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});r("tables().footer()","table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});r("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});p("draw()",function(a){return this.iterator("table",
-function(b){"page"===a?O(b):("string"===typeof a&&(a="full-hold"===a?!1:!0),T(b,!1===a))})});p("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Ta(b,a)})});p("page.info()",function(){if(0===this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a.oFeatures.bPaginate?a._iDisplayLength:-1,d=a.fnRecordsDisplay(),e=-1===c;return{page:e?0:Math.floor(b/c),pages:e?1:Math.ceil(d/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),
-recordsDisplay:d,serverSide:"ssp"===y(a)}});p("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:k:this.iterator("table",function(b){Ra(b,a)})});var Ub=function(a,b,c){if(c){var d=new t(a);d.one("draw",function(){c(d.ajax.json())})}if("ssp"==y(a))T(a,b);else{C(a,!0);var e=a.jqXHR;e&&4!==e.readyState&&e.abort();ra(a,[],function(c){na(a);for(var c=sa(a,c),d=0,e=c.length;d<e;d++)N(a,c[d]);T(a,b);C(a,!1)})}};p("ajax.json()",function(){var a=this.context;if(0<
-a.length)return a[0].json});p("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});p("ajax.reload()",function(a,b){return this.iterator("table",function(c){Ub(c,!1===b,a)})});p("ajax.url()",function(a){var b=this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});p("ajax.url().load()",function(a,b){return this.iterator("table",
-function(c){Ub(c,!1===b,a)})});var $a=function(a,b,c,d,e){var f=[],g,j,i,o,l,m;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(o=b.length;i<o;i++){j=b[i]&&b[i].split?b[i].split(","):[b[i]];l=0;for(m=j.length;l<m;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&(f=f.concat(g))}a=s.selector[a];if(a.length){i=0;for(o=a.length;i<o;i++)f=a[i](d,e,f)}return pa(f)},ab=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",
-order:"current",page:"all"},a)},bb=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},Da=function(a,b){var c,d,e,f=[],g=a.aiDisplay;c=a.aiDisplayMaster;var j=b.search;d=b.order;e=b.page;if("ssp"==y(a))return"removed"===j?[]:X(0,c.length);if("current"==e){c=a._iDisplayStart;for(d=a.fnDisplayEnd();c<d;c++)f.push(g[c])}else if("current"==d||"applied"==d)f="none"==j?c.slice():"applied"==j?g.slice():h.map(c,
-function(a){return-1===h.inArray(a,g)?a:null});else if("index"==d||"original"==d){c=0;for(d=a.aoData.length;c<d;c++)"none"==j?f.push(c):(e=h.inArray(c,g),(-1===e&&"removed"==j||0<=e&&"applied"==j)&&f.push(c))}return f};p("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=ab(b),c=this.iterator("table",function(c){var e=b;return $a("row",a,function(a){var b=Pb(a);if(b!==null&&!e)return[b];var j=Da(c,e);if(b!==null&&h.inArray(b,j)!==-1)return[b];if(!a)return j;if(typeof a==="function")return h.map(j,
-function(b){var e=c.aoData[b];return a(b,e._aData,e.nTr)?b:null});b=Sb(ja(c.aoData,j,"nTr"));if(a.nodeName&&h.inArray(a,b)!==-1)return[a._DT_RowIndex];if(typeof a==="string"&&a.charAt(0)==="#"){j=c.aIds[a.replace(/^#/,"")];if(j!==k)return[j.idx]}return h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},c,e)},1);c.selector.rows=a;c.selector.opts=b;return c});p("rows().nodes()",function(){return this.iterator("row",function(a,b){return a.aoData[b].nTr||k},1)});p("rows().data()",function(){return this.iterator(!0,
-"rows",function(a,b){return ja(a.aoData,b,"_aData")},1)});r("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var d=b.aoData[c];return"search"===a?d._aFilterData:d._aSortData},1)});r("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",function(b,c){ea(b,c,a)})});r("rows().indexes()","row().index()",function(){return this.iterator("row",function(a,b){return b},1)});r("rows().ids()","row().id()",function(a){for(var b=[],c=this.context,
-d=0,e=c.length;d<e;d++)for(var f=0,g=this[d].length;f<g;f++){var h=c[d].rowIdFn(c[d].aoData[this[d][f]]._aData);b.push((!0===a?"#":"")+h)}return new t(c,b)});r("rows().remove()","row().remove()",function(){var a=this;this.iterator("row",function(b,c,d){var e=b.aoData,f=e[c],g,h,i,o,l;e.splice(c,1);g=0;for(h=e.length;g<h;g++)if(i=e[g],l=i.anCells,null!==i.nTr&&(i.nTr._DT_RowIndex=g),null!==l){i=0;for(o=l.length;i<o;i++)l[i]._DT_CellIndex.row=g}oa(b.aiDisplayMaster,c);oa(b.aiDisplay,c);oa(a[d],c,!1);
-Sa(b);c=b.rowIdFn(f._aData);c!==k&&delete b.aIds[c]});this.iterator("table",function(a){for(var c=0,d=a.aoData.length;c<d;c++)a.aoData[c].idx=c});return this});p("rows.add()",function(a){var b=this.iterator("table",function(b){var c,f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(ma(b,c)[0]):h.push(N(b,c));return h},1),c=this.rows(-1);c.pop();h.merge(c,b);return c});p("row()",function(a,b){return bb(this.rows(a,b))});p("row().data()",function(a){var b=
-this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:k;b[0].aoData[this[0]]._aData=a;ea(b[0],this[0],"data");return this});p("row().node()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]].nTr||null:null});p("row.add()",function(a){a instanceof h&&a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?ma(b,a)[0]:N(b,a)});return this.row(b[0])});var cb=function(a,b){var c=a.context;if(c.length&&
-(c=c[0].aoData[b!==k?b:a[0]])&&c._details)c._details.remove(),c._detailsShow=k,c._details=k},Vb=function(a,b){var c=a.context;if(c.length&&a.length){var d=c[0].aoData[a[0]];if(d._details){(d._detailsShow=b)?d._details.insertAfter(d.nTr):d._details.detach();var e=c[0],f=new t(e),g=e.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){e===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];
-a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",function(a,b){if(e===b)for(var c,d=ca(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",d)}),f.on("destroy.dt.DT_details",function(a,b){if(e===b)for(var c=0,d=g.length;c<d;c++)g[c]._details&&cb(f,c)}))}}};p("row().child()",function(a,b){var c=this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===
-a)cb(this);else if(c.length&&this.length){var d=c[0],c=c[0].aoData[this[0]],e=[],f=function(a,b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?e.push(a):(c=h("<tr><td/></tr>").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=ca(d),e.push(c[0]))};f(a,b);c._details&&c._details.remove();c._details=h(e);c._detailsShow&&c._details.insertAfter(c.nTr)}return this});p(["row().child.show()","row().child().show()"],function(){Vb(this,
-!0);return this});p(["row().child.hide()","row().child().hide()"],function(){Vb(this,!1);return this});p(["row().child.remove()","row().child().remove()"],function(){cb(this);return this});p("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var dc=/^(.+):(name|visIdx|visible)$/,Wb=function(a,b,c,d,e){for(var c=[],d=0,f=e.length;d<f;d++)c.push(B(a,e[d],b));return c};p("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&
-(b=a,a="");var b=ab(b),c=this.iterator("table",function(c){var e=a,f=b,g=c.aoColumns,j=D(g,"sName"),i=D(g,"nTh");return $a("column",e,function(a){var b=Pb(a);if(a==="")return X(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var e=Da(c,f);return h.map(g,function(b,f){return a(f,Wb(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(dc):"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var m=h.map(g,function(a,b){return a.bVisible?b:null});
-return[m[m.length+b]]}return[$(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null})}else return h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray()},c,f)},1);c.selector.cols=a;c.selector.opts=b;return c});r("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});r("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});r("columns().data()",
-"column().data()",function(){return this.iterator("column-rows",Wb,1)});r("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});r("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});r("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,
-e,"anCells",b)},1)});r("columns().visible()","column().visible()",function(a,b){return this.iterator("column",function(c,d){if(a===k)return c.aoColumns[d].bVisible;var e=c.aoColumns,f=e[d],g=c.aoData,j,i,m;if(a!==k&&f.bVisible!==a){if(a){var l=h.inArray(!0,D(e,"bVisible"),d+1);j=0;for(i=g.length;j<i;j++)m=g[j].nTr,e=g[j].anCells,m&&m.insertBefore(e[d],e[l]||null)}else h(D(c.aoData,"anCells",d)).detach();f.bVisible=a;ga(c,c.aoHeader);ga(c,c.aoFooter);if(b===k||b)U(c),(c.oScroll.sX||c.oScroll.sY)&&
-Z(c);v(c,null,"column-visibility",[c,d,a,b]);ya(c)}})});r("columns().indexes()","column().index()",function(a){return this.iterator("column",function(b,c){return"visible"===a?ba(b,c):c},1)});p("columns.adjust()",function(){return this.iterator("table",function(a){U(a)},1)});p("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===a||"toData"===a)return $(c,b);if("fromData"===a||"toVisible"===a)return ba(c,b)}});p("column()",function(a,b){return bb(this.columns(a,
-b))});p("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=ab(c),f=b.aoData,g=Da(b,e),j=Sb(ja(f,g,"anCells")),i=h([].concat.apply([],j)),l,m=b.aoColumns.length,o,p,t,r,s,v;return $a("cell",d,function(a){var c=typeof a==="function";if(a===null||a===k||c){o=[];p=0;for(t=g.length;p<t;p++){l=g[p];for(r=0;r<m;r++){s={row:l,column:r};if(c){v=f[l];a(s,B(b,l,r),v.anCells?
-v.anCells[r]:null)&&o.push(s)}else o.push(s)}}return o}return h.isPlainObject(a)?[a]:i.filter(a).map(function(a,b){return{row:b._DT_CellIndex.row,column:b._DT_CellIndex.column}}).toArray()},b,e)});var d=this.columns(b,c),e=this.rows(a,c),f,g,j,i,m,l=this.iterator("table",function(a,b){f=[];g=0;for(j=e[b].length;g<j;g++){i=0;for(m=d[b].length;i<m;i++)f.push({row:e[b][g],column:d[b][i]})}return f},1);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});r("cells().nodes()","cell().node()",function(){return this.iterator("cell",
-function(a,b,c){return(a=a.aoData[b].anCells)?a[c]:k},1)});p("cells().data()",function(){return this.iterator("cell",function(a,b,c){return B(a,b,c)},1)});r("cells().cache()","cell().cache()",function(a){a="search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,d){return b.aoData[c][a][d]},1)});r("cells().render()","cell().render()",function(a){return this.iterator("cell",function(b,c,d){return B(b,c,d,a)},1)});r("cells().indexes()","cell().index()",function(){return this.iterator("cell",
-function(a,b,c){return{row:b,column:c,columnVisible:ba(a,c)}},1)});r("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,d){ea(b,c,a,d)})});p("cell()",function(a,b,c){return bb(this.cells(a,b,c))});p("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?B(b[0],c[0].row,c[0].column):k;jb(b[0],c[0].row,c[0].column,a);ea(b[0],c[0].row,"data",c[0].column);return this});p("order()",function(a,b){var c=this.context;if(a===
-k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:h.isArray(a[0])||(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});p("order.listener()",function(a,b,c){return this.iterator("table",function(d){Oa(d,a,b,c)})});p("order.fixed()",function(a){if(!a){var b=this.context,b=b.length?b[0].aaSortingFixed:k;return h.isArray(b)?{pre:b}:b}return this.iterator("table",function(b){b.aaSortingFixed=h.extend(!0,{},a)})});p(["columns().order()",
-"column().order()"],function(a){var b=this;return this.iterator("table",function(c,d){var e=[];h.each(b[d],function(b,c){e.push([c,a])});c.aaSorting=e})});p("search()",function(a,b,c,d){var e=this.context;return a===k?0!==e.length?e[0].oPreviousSearch.sSearch:k:this.iterator("table",function(e){e.oFeatures.bFilter&&ha(e,h.extend({},e.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),1)})});r("columns().search()","column().search()",function(a,
-b,c,d){return this.iterator("column",function(e,f){var g=e.aoPreSearchCols;if(a===k)return g[f].sSearch;e.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),ha(e,e.oPreviousSearch,1))})});p("state()",function(){return this.context.length?this.context[0].oSavedState:null});p("state.clear()",function(){return this.iterator("table",function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});p("state.loaded()",function(){return this.context.length?
-this.context[0].oLoadedState:null});p("state.save()",function(){return this.iterator("table",function(a){ya(a)})});m.versionCheck=m.fnVersionCheck=function(a){for(var b=m.version.split("."),a=a.split("."),c,d,e=0,f=a.length;e<f;e++)if(c=parseInt(b[e],10)||0,d=parseInt(a[e],10)||0,c!==d)return c>d;return!0};m.isDataTable=m.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;h.each(m.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:
-null;if(e.nTable===b||f===b||g===b)c=!0});return c};m.tables=m.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(m.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new t(c):c};m.util={throttle:ua,escapeRegex:va};m.camelToHungarian=J;p("$()",function(a,b){var c=this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){p(b+"()",function(){var a=Array.prototype.slice.call(arguments);
-a[0].match(/\.dt\b/)||(a[0]+=".dt");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});p("clear()",function(){return this.iterator("table",function(a){na(a)})});p("settings()",function(){return new t(this.context,this.context)});p("init()",function(){var a=this.context;return a.length?a[0].oInit:null});p("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});p("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=
-b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),p;b.bDestroying=!0;v(b,"aoDestroyCallback","destroy",[b]);a||(new t(b)).columns().visible(!0);k.unbind(".DT").find(":not(tbody *)").unbind(".DT");h(E).unbind(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];xa(b);
-h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);b.bJUI&&(h("th span."+d.sSortIcon+", td span."+d.sSortIcon,g).detach(),h("th, td",g).each(function(){var a=h("div."+d.sSortJUIWrapper,this);h(this).append(a.contents());a.detach()}));f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable),(p=
-b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%p])}));c=h.inArray(b,m.settings);-1!==c&&m.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){p(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});p("i18n()",function(a,b,c){var d=this.context[0],a=Q(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:a._);
-return a.replace("%d",c)});m.version="1.10.10";m.settings=[];m.models={};m.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};m.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};m.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,
-sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};m.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,
-bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+
-a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},
-oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},
-m.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};Y(m.defaults);m.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};
-Y(m.defaults.column);m.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],
-aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",
-iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?
-1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};
-m.ext=s={buttons:{},classes:{},build:"bs/dt-1.10.10",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:m.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:m.version};h.extend(s,{afnFiltering:s.search,aTypes:s.type.detect,ofnSearch:s.type.search,oSort:s.type.order,afnSortData:s.order,aoFeatures:s.feature,oApi:s.internal,oStdClasses:s.classes,
-oPagination:s.pager});h.extend(m.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",
-sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",
-sJUIHeader:"",sJUIFooter:""});var Ea="",Ea="",G=Ea+"ui-state-default",ka=Ea+"css_right ui-icon ui-icon-",Xb=Ea+"fg-toolbar ui-toolbar ui-widget-header ui-helper-clearfix";h.extend(m.ext.oJUIClasses,m.ext.classes,{sPageButton:"fg-button ui-button "+G,sPageButtonActive:"ui-state-disabled",sPageButtonDisabled:"ui-state-disabled",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:G+" sorting_asc",sSortDesc:G+" sorting_desc",sSortable:G+" sorting",
-sSortableAsc:G+" sorting_asc_disabled",sSortableDesc:G+" sorting_desc_disabled",sSortableNone:G+" sorting_disabled",sSortJUIAsc:ka+"triangle-1-n",sSortJUIDesc:ka+"triangle-1-s",sSortJUI:ka+"carat-2-n-s",sSortJUIAscAllowed:ka+"carat-1-n",sSortJUIDescAllowed:ka+"carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead "+G,sScrollFoot:"dataTables_scrollFoot "+G,sHeaderTH:G,sFooterTH:G,sJUIHeader:Xb+" ui-corner-tl ui-corner-tr",sJUIFooter:Xb+
-" ui-corner-bl ui-corner-br"});var Mb=m.ext.pager;h.extend(Mb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[Aa(a,b)]},simple_numbers:function(a,b){return["previous",Aa(a,b),"next"]},full_numbers:function(a,b){return["first","previous",Aa(a,b),"next","last"]},_numbers:Aa,numbers_length:7});h.extend(!0,m.ext.renderer,{pageButton:{_:function(a,b,c,d,e,f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||
-{},k,l,m=0,p=function(b,d){var n,r,t,s,v=function(b){Ta(a,b.data.action,true)};n=0;for(r=d.length;n<r;n++){s=d[n];if(h.isArray(s)){t=h("<"+(s.DT_el||"div")+"/>").appendTo(b);p(t,s)}else{k=null;l="";switch(s){case "ellipsis":b.append('<span class="ellipsis">&#x2026;</span>');break;case "first":k=j.sFirst;l=s+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":k=j.sPrevious;l=s+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":k=j.sNext;l=s+(e<f-1?"":" "+g.sPageButtonDisabled);break;case "last":k=
-j.sLast;l=s+(e<f-1?"":" "+g.sPageButtonDisabled);break;default:k=s+1;l=e===s?g.sPageButtonActive:""}if(k!==null){t=h("<a>",{"class":g.sPageButton+" "+l,"aria-controls":a.sTableId,"aria-label":i[s],"data-dt-idx":m,tabindex:a.iTabIndex,id:c===0&&typeof s==="string"?a.sTableId+"_"+s:null}).html(k).appendTo(b);Wa(t,{action:s},v);m++}}}},r;try{r=h(b).find(H.activeElement).data("dt-idx")}catch(n){}p(h(b).empty(),d);r&&h(b).find("[data-dt-idx="+r+"]").focus()}}});h.extend(m.ext.type.detect,[function(a,b){var c=
-b.oLanguage.sDecimal;return Za(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&(!ac.test(a)||!bc.test(a)))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Rb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Rb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":
-null}]);h.extend(m.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Ob," ").replace(Ca,""):""},string:function(a){return M(a)?a:"string"===typeof a?a.replace(Ob," "):a}});var Ba=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Qb(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(s.type.order,{"date-pre":function(a){return Date.parse(a)||0},"html-pre":function(a){return M(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():
-a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});db("");h.extend(!0,m.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass)}})},jqueryui:function(a,
-b,c,d){h("<div/>").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]=="asc"?d.sSortJUIAsc:
-h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});m.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return f;f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g,a)+f+(e||"")}}},text:function(){return{display:function(a){return"string"===typeof a?a.replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;"):
-a}}}};h.extend(m.ext.internal,{_fnExternApiFunc:Nb,_fnBuildAjax:ra,_fnAjaxUpdate:lb,_fnAjaxParameters:ub,_fnAjaxUpdateDraw:vb,_fnAjaxDataSrc:sa,_fnAddColumn:Ga,_fnColumnOptions:la,_fnAdjustColumnSizing:U,_fnVisibleToColumnIndex:$,_fnColumnIndexToVisible:ba,_fnVisbleColumns:ca,_fnGetColumns:aa,_fnColumnTypes:Ia,_fnApplyColumnDefs:ib,_fnHungarianMap:Y,_fnCamelToHungarian:J,_fnLanguageCompat:Fa,_fnBrowserDetect:gb,_fnAddData:N,_fnAddTr:ma,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==k?b._DT_RowIndex:
-null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:jb,_fnSplitObjNotation:La,_fnGetObjectDataFn:Q,_fnSetObjectDataFn:R,_fnGetDataMaster:Ma,_fnClearTable:na,_fnDeleteIndex:oa,_fnInvalidate:ea,_fnGetRowElements:Ka,_fnCreateTr:Ja,_fnBuildHead:kb,_fnDrawHead:ga,_fnDraw:O,_fnReDraw:T,_fnAddOptionsHtml:nb,_fnDetectHeader:fa,_fnGetUniqueThs:qa,_fnFeatureHtmlFilter:pb,_fnFilterComplete:ha,_fnFilterCustom:yb,_fnFilterColumn:xb,_fnFilter:wb,_fnFilterCreateSearch:Qa,
-_fnEscapeRegex:va,_fnFilterData:zb,_fnFeatureHtmlInfo:sb,_fnUpdateInfo:Cb,_fnInfoMacros:Db,_fnInitialise:ia,_fnInitComplete:ta,_fnLengthChange:Ra,_fnFeatureHtmlLength:ob,_fnFeatureHtmlPaginate:tb,_fnPageChange:Ta,_fnFeatureHtmlProcessing:qb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:rb,_fnScrollDraw:Z,_fnApplyToChildren:I,_fnCalculateColumnWidths:Ha,_fnThrottle:ua,_fnConvertToWidth:Fb,_fnGetWidestNode:Gb,_fnGetMaxLenString:Hb,_fnStringToCss:w,_fnSortFlatten:W,_fnSort:mb,_fnSortAria:Jb,_fnSortListener:Va,
-_fnSortAttachListener:Oa,_fnSortingClasses:xa,_fnSortData:Ib,_fnSaveState:ya,_fnLoadState:Kb,_fnSettingsFromNode:za,_fnLog:K,_fnMap:F,_fnBindAction:Wa,_fnCallbackReg:z,_fnCallbackFire:v,_fnLengthOverflow:Sa,_fnRenderer:Pa,_fnDataSource:y,_fnRowAttributes:Na,_fnCalculateEnd:function(){}});h.fn.dataTable=m;m.$=h;h.fn.dataTableSettings=m.settings;h.fn.dataTableExt=m.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};h.each(m,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable});
-
-
-/*!
- DataTables Bootstrap 3 integration
- ©2011-2015 SpryMedia Ltd - datatables.net/license
-*/
-(function(b){"function"===typeof define&&define.amd?define(["jquery","datatables.net"],function(a){return b(a,window,document)}):"object"===typeof exports?module.exports=function(a,e){a||(a=window);if(!e||!e.fn.dataTable)e=require("datatables.net")(a,e).$;return b(e,a,a.document)}:b(jQuery,window,document)})(function(b,a,e){var d=b.fn.dataTable;b.extend(!0,d.defaults,{dom:"<'row'<'col-sm-6'l><'col-sm-6'f>><'row'<'col-sm-12'tr>><'row'<'col-sm-5'i><'col-sm-7'p>>",renderer:"bootstrap"});b.extend(d.ext.classes,
-{sWrapper:"dataTables_wrapper form-inline dt-bootstrap",sFilterInput:"form-control input-sm",sLengthSelect:"form-control input-sm",sProcessing:"dataTables_processing panel panel-default"});d.ext.renderer.pageButton.bootstrap=function(a,h,r,m,j,n){var o=new d.Api(a),s=a.oClasses,k=a.oLanguage.oPaginate,t=a.oLanguage.oAria.paginate||{},f,g,p=0,q=function(d,e){var l,h,i,c,m=function(a){a.preventDefault();!b(a.currentTarget).hasClass("disabled")&&o.page()!=a.data.action&&o.page(a.data.action).draw("page")};
-l=0;for(h=e.length;l<h;l++)if(c=e[l],b.isArray(c))q(d,c);else{g=f="";switch(c){case "ellipsis":f="&#x2026;";g="disabled";break;case "first":f=k.sFirst;g=c+(0<j?"":" disabled");break;case "previous":f=k.sPrevious;g=c+(0<j?"":" disabled");break;case "next":f=k.sNext;g=c+(j<n-1?"":" disabled");break;case "last":f=k.sLast;g=c+(j<n-1?"":" disabled");break;default:f=c+1,g=j===c?"active":""}f&&(i=b("<li>",{"class":s.sPageButton+" "+g,id:0===r&&"string"===typeof c?a.sTableId+"_"+c:null}).append(b("<a>",{href:"#",
-"aria-controls":a.sTableId,"aria-label":t[c],"data-dt-idx":p,tabindex:a.iTabIndex}).html(f)).appendTo(d),a.oApi._fnBindAction(i,{action:c},m),p++)}},i;try{i=b(h).find(e.activeElement).data("dt-idx")}catch(u){}q(b(h).empty().html('<ul class="pagination"/>').children("ul"),m);i&&b(h).find("[data-dt-idx="+i+"]").focus()};d.TableTools&&(b.extend(!0,d.TableTools.classes,{container:"DTTT btn-group",buttons:{normal:"btn btn-default",disabled:"disabled"},collection:{container:"DTTT_dropdown dropdown-menu",
-buttons:{normal:"",disabled:"disabled"}},print:{info:"DTTT_print_info"},select:{row:"active"}}),b.extend(!0,d.TableTools.DEFAULTS.oTags,{collection:{container:"ul",button:"li",liner:"a"}}));return d});
-
-
diff --git a/airflow/www_rbac/static/js/base.js b/airflow/www_rbac/static/js/base.js
new file mode 100644
index 0000000000000..f9385481023a5
--- /dev/null
+++ b/airflow/www_rbac/static/js/base.js
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {defaultFormatWithTZ, moment} from './datetime-utils';
+
+function displayTime() {
+  let utcTime = moment().utc().format(defaultFormatWithTZ);
+  $('#clock')
+    .attr("data-original-title", function() {
+      return hostName
+    })
+    .html(utcTime);
+
+  setTimeout(displayTime, 1000);
+}
+
+var el = document.createElement("span");
+
+export function escapeHtml(text) {
+  el.textContent = text;
+  return el.innerHTML;
+}
+
+window.escapeHtml = escapeHtml;
+
+function postAsForm(url, parameters) {
+  var form = $("<form></form>");
+
+  form.attr("method", "POST");
+  form.attr("action", url);
+
+  $.each(parameters || {}, function(key, value) {
+    var field = $('<input></input>');
+
+    field.attr("type", "hidden");
+    field.attr("name", key);
+    field.attr("value", value);
+
+    form.append(field);
+  });
+
+  var field = $('<input></input>');
+
+  field.attr("type", "hidden");
+  field.attr("name", "csrf_token");
+  field.attr("value", csrfToken);
+
+  form.append(field);
+
+  // The form needs to be a part of the document in order for us to be able
+  // to submit it.
+  $(document.body).append(form);
+  form.submit();
+}
+
+window.postAsForm = postAsForm;
+
+$(document).ready(function () {
+  displayTime();
+  $('span').tooltip();
+  $.ajaxSetup({
+    beforeSend: function(xhr, settings) {
+      if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+        xhr.setRequestHeader("X-CSRFToken", csrfToken);
+      }
+    }
+  });
+});
diff --git a/airflow/www_rbac/static/js/connection_form.js b/airflow/www_rbac/static/js/connection_form.js
new file mode 100644
index 0000000000000..17ca29066a21b
--- /dev/null
+++ b/airflow/www_rbac/static/js/connection_form.js
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Created by janomar on 23/07/15.
+ */
+
+$(document).ready(function () {
+  var config = {
+    jdbc: {
+      hidden_fields: ['port', 'schema', 'extra'],
+      relabeling: {'host': 'Connection URL'},
+    },
+    google_cloud_platform: {
+      hidden_fields: ['host', 'schema', 'login', 'password', 'port', 'extra'],
+      relabeling: {},
+    },
+    cloudant: {
+      hidden_fields: ['port', 'extra'],
+      relabeling: {
+        'host': 'Account',
+        'login': 'Username (or API Key)',
+        'schema': 'Database'
+      }
+    },
+    docker: {
+      hidden_fields: ['port', 'schema'],
+      relabeling: {
+        'host': 'Registry URL',
+        'login': 'Username',
+      },
+    },
+    qubole: {
+      hidden_fields: ['login', 'schema', 'port', 'extra'],
+      relabeling: {
+        'host': 'API Endpoint',
+        'password': 'Auth Token',
+      },
+      placeholders: {
+        'host': 'https://<env>.qubole.com/api'
+      }
+    },
+    ssh: {
+      hidden_fields: ['schema'],
+      relabeling: {
+        'login': 'Username',
+      }
+    },
+  };
+
+  function connTypeChange(connectionType) {
+    $(".hide").removeClass("hide");
+    $.each($("[id^='extra__']"), function () {
+      $(this).parent().parent().addClass('hide')
+    });
+    $.each($("[id^='extra__" + connectionType + "']"), function () {
+      $(this).parent().parent().removeClass('hide')
+    });
+    $("label[orig_text]").each(function () {
+      $(this).text($(this).attr("orig_text"));
+    });
+    $(".form-control").each(function(){$(this).attr('placeholder', '')});
+
+    if (config[connectionType] != undefined) {
+      $.each(config[connectionType].hidden_fields, function (i, field) {
+        $("#" + field).parent().parent().addClass('hide')
+      });
+      $.each(config[connectionType].relabeling, function (k, v) {
+        lbl = $("label[for='" + k + "']");
+        lbl.attr("orig_text", lbl.text());
+        $("label[for='" + k + "']").text(v);
+      });
+      $.each(config[connectionType].placeholders, function(k, v){
+        $("#" + k).attr('placeholder', v);
+      });
+    }
+  }
+
+  var connectionType = $("#conn_type").val();
+  $("#conn_type").on('change', function (e) {
+    connectionType = $("#conn_type").val();
+    connTypeChange(connectionType);
+  });
+  connTypeChange(connectionType);
+});
diff --git a/airflow/www_rbac/static/js/datetime-utils.js b/airflow/www_rbac/static/js/datetime-utils.js
new file mode 100644
index 0000000000000..08e94694794c0
--- /dev/null
+++ b/airflow/www_rbac/static/js/datetime-utils.js
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const moment = require('moment-timezone');
+export const defaultFormat = 'YYYY-MM-DD, HH:mm:ss';
+export const defaultFormatWithTZ = 'YYYY-MM-DD, HH:mm:ss z';
+
+
+const makeDateTimeHTML = (start, end) => {
+  return (
+    `Started: ${start.format(defaultFormat)} <br> Ended: ${end.format(defaultFormat)} <br>`
+  )
+};
+
+export const generateTooltipDateTime = (startDate, endDate, dagTZ) => {
+  const tzFormat = 'z (Z)';
+  const localTZ = moment.tz.guess();
+  startDate = moment.utc(startDate);
+  endDate = moment.utc(endDate);
+  dagTZ = dagTZ.toUpperCase();
+
+  // Generate UTC Start and End Date
+  let tooltipHTML = '<br><strong>UTC</strong><br>';
+  tooltipHTML += makeDateTimeHTML(startDate, endDate);
+
+  // Generate User's Local Start and End Date
+  tooltipHTML += `<br><strong>Local: ${moment.tz(localTZ).format(tzFormat)}</strong><br>`;
+  tooltipHTML += makeDateTimeHTML(startDate.local(), endDate.local());
+
+  // Generate DAG's Start and End Date
+  if (dagTZ !== 'UTC' && dagTZ !== localTZ) {
+    tooltipHTML += `<br><strong>DAG's TZ: ${moment.tz(dagTZ).format(tzFormat)}</strong><br>`;
+    tooltipHTML += makeDateTimeHTML(startDate.tz(dagTZ), endDate.tz(dagTZ));
+  }
+
+  return tooltipHTML
+};
+
+
+export const converAndFormatUTC = (datetime, tz) => {
+  let dateTimeObj = moment.utc(datetime);
+  if (tz) dateTimeObj = dateTimeObj.tz(tz);
+  return dateTimeObj.format(defaultFormatWithTZ)
+}
diff --git a/airflow/www_rbac/static/gantt-chart-d3v2.js b/airflow/www_rbac/static/js/gantt-chart-d3v2.js
similarity index 99%
rename from airflow/www_rbac/static/gantt-chart-d3v2.js
rename to airflow/www_rbac/static/js/gantt-chart-d3v2.js
index d21311a1c541d..245a0147e9f72 100644
--- a/airflow/www_rbac/static/gantt-chart-d3v2.js
+++ b/airflow/www_rbac/static/js/gantt-chart-d3v2.js
@@ -129,7 +129,7 @@ d3.gantt = function() {
       call_modal(d.taskName, d.executionDate);
     })
     .attr("class", function(d){
-      if(taskStatus[d.status] == null){ return "bar";}
+      if(taskStatus[d.status] == null){ return "null";}
       return taskStatus[d.status];
     })
     .attr("y", 0)
diff --git a/airflow/www_rbac/static/js/graph.js b/airflow/www_rbac/static/js/graph.js
new file mode 100644
index 0000000000000..416d4198d3572
--- /dev/null
+++ b/airflow/www_rbac/static/js/graph.js
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { generateTooltipDateTime, converAndFormatUTC } from './datetime-utils';
+import { escapeHtml } from './base';
+
+// Assigning css classes based on state to nodes
+// Initiating the tooltips
+function update_nodes_states(task_instances) {
+  $.each(task_instances, function (task_id, ti) {
+    $('tspan').filter(function (index) {
+      return $(this).text() === task_id;
+    })
+      .parent().parent().parent().parent()
+      .attr("class", "node enter " + (ti.state ? ti.state : "no_status"))
+      .attr("data-toggle", "tooltip")
+      .attr("data-original-title", function (d) {
+        // Tooltip
+        const task = tasks[task_id];
+        let tt = "";
+        if(ti.task_id != undefined) {
+          tt +=  "Task_id: " + escapeHtml(task.task_id) + "<br>";
+        }
+        tt += "Run: " + converAndFormatUTC(task.execution_date) + "<br>";
+        if(ti.run_id != undefined) {
+          tt += "run_id: <nobr>" + escapeHtml(task.run_id) + "</nobr><br>";
+        }
+        tt += "Operator: " + escapeHtml(task.task_type) + "<br>";
+        tt += "Duration: " + escapeHtml(task.duration) + "<br>";
+        tt += "Started: " + escapeHtml(ti.start_date) + "<br>";
+        tt += generateTooltipDateTime(ti.start_date, ti.end_date, dagTZ); // dagTZ has been defined in dag.html
+        return tt;
+      });
+  });
+}
+
+function initRefreshButton() {
+  d3.select("#refresh_button").on("click",
+    function () {
+      $("#loading").css("display", "block");
+      $("div#svg_container").css("opacity", "0.2");
+      $.get(getTaskInstanceURL)
+        .done(
+          function (task_instances) {
+            update_nodes_states(JSON.parse(task_instances));
+            $("#loading").hide();
+            $("div#svg_container").css("opacity", "1");
+            $('#error').hide();
+          }
+        ).fail(function (jqxhr, textStatus, err) {
+        $('#error_msg').html(textStatus + ': ' + err);
+        $('#error').show();
+        $('#loading').hide();
+        $('#chart_section').hide(1000);
+        $('#datatable_section').hide(1000);
+      });
+    }
+  );
+}
+
+initRefreshButton();
+update_nodes_states(task_instances);
diff --git a/airflow/www_rbac/static/nv.d3.css b/airflow/www_rbac/static/nv.d3.css
deleted file mode 100644
index 7ca7555ec000b..0000000000000
--- a/airflow/www_rbac/static/nv.d3.css
+++ /dev/null
@@ -1,788 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-
-/********************
- * HTML CSS
- */
-
-
-.chartWrap {
-  margin: 0;
-  padding: 0;
-  overflow: hidden;
-}
-
-/********************
-  Box shadow and border radius styling
-*/
-.nvtooltip.with-3d-shadow, .with-3d-shadow .nvtooltip {
-  -moz-box-shadow: 0 5px 10px rgba(0,0,0,.2);
-  -webkit-box-shadow: 0 5px 10px rgba(0,0,0,.2);
-  box-shadow: 0 5px 10px rgba(0,0,0,.2);
-
-  -webkit-border-radius: 6px;
-  -moz-border-radius: 6px;
-  border-radius: 6px;
-}
-
-/********************
- * TOOLTIP CSS
- */
-
-.nvtooltip {
-  position: absolute;
-  background-color: rgba(255,255,255,1.0);
-  padding: 1px;
-  border: 1px solid rgba(0,0,0,.2);
-  z-index: 10000;
-
-  font-family: Arial;
-  font-size: 13px;
-  text-align: left;
-  pointer-events: none;
-
-  white-space: nowrap;
-
-  -webkit-touch-callout: none;
-  -webkit-user-select: none;
-  -khtml-user-select: none;
-  -moz-user-select: none;
-  -ms-user-select: none;
-  user-select: none;
-}
-
-/*Give tooltips that old fade in transition by
-    putting a "with-transitions" class on the container div.
-*/
-.nvtooltip.with-transitions, .with-transitions .nvtooltip {
-  transition: opacity 250ms linear;
-  -moz-transition: opacity 250ms linear;
-  -webkit-transition: opacity 250ms linear;
-
-  transition-delay: 250ms;
-  -moz-transition-delay: 250ms;
-  -webkit-transition-delay: 250ms;
-}
-
-.nvtooltip.x-nvtooltip,
-.nvtooltip.y-nvtooltip {
-  padding: 8px;
-}
-
-.nvtooltip h3 {
-  margin: 0;
-  padding: 4px 14px;
-  line-height: 18px;
-  font-weight: normal;
-  background-color: rgba(247,247,247,0.75);
-  text-align: center;
-
-  border-bottom: 1px solid #ebebeb;
-
-  -webkit-border-radius: 5px 5px 0 0;
-  -moz-border-radius: 5px 5px 0 0;
-  border-radius: 5px 5px 0 0;
-}
-
-.nvtooltip p {
-  margin: 0;
-  padding: 5px 14px;
-  text-align: center;
-}
-
-.nvtooltip span {
-  display: inline-block;
-  margin: 2px 0;
-}
-
-.nvtooltip table {
-  margin: 6px;
-  border-spacing:0;
-}
-
-
-.nvtooltip table td {
-  padding: 2px 9px 2px 0;
-  vertical-align: middle;
-}
-
-.nvtooltip table td.key {
-  font-weight:normal;
-}
-.nvtooltip table td.value {
-  text-align: right;
-  font-weight: bold;
-}
-
-.nvtooltip table tr.highlight td {
-  padding: 1px 9px 1px 0;
-  border-bottom-style: solid;
-  border-bottom-width: 1px;
-  border-top-style: solid;
-  border-top-width: 1px;
-}
-
-.nvtooltip table td.legend-color-guide div {
-  width: 8px;
-  height: 8px;
-  vertical-align: middle;
-}
-
-.nvtooltip .footer {
-  padding: 3px;
-  text-align: center;
-}
-
-
-.nvtooltip-pending-removal {
-  position: absolute;
-  pointer-events: none;
-}
-
-
-/********************
- * SVG CSS
- */
-
-
-svg {
-  -webkit-touch-callout: none;
-  -webkit-user-select: none;
-  -khtml-user-select: none;
-  -moz-user-select: none;
-  -ms-user-select: none;
-  user-select: none;
-  /* Trying to get SVG to act like a greedy block in all browsers */
-  display: block;
-  width:100%;
-  height:100%;
-}
-
-
-svg text {
-  font: normal 12px Arial;
-}
-
-svg .title {
- font: bold 14px Arial;
-}
-
-.nvd3 .nv-background {
-  fill: white;
-  fill-opacity: 0;
-  /*
-  pointer-events: none;
-  */
-}
-
-.nvd3.nv-noData {
-  font-size: 18px;
-  font-weight: bold;
-}
-
-
-/**********
-*  Brush
-*/
-
-.nv-brush .extent {
-  fill-opacity: .125;
-  shape-rendering: crispEdges;
-}
-
-
-
-/**********
-*  Legend
-*/
-
-.nvd3 .nv-legend .nv-series {
-  cursor: pointer;
-}
-
-.nvd3 .nv-legend .disabled circle {
-  fill-opacity: 0;
-}
-
-
-
-/**********
-*  Axes
-*/
-.nvd3 .nv-axis {
-  pointer-events:none;
-}
-
-.nvd3 .nv-axis path {
-  fill: none;
-  stroke: #000;
-  stroke-opacity: .75;
-  shape-rendering: crispEdges;
-}
-
-.nvd3 .nv-axis path.domain {
-  stroke-opacity: .75;
-}
-
-.nvd3 .nv-axis.nv-x path.domain {
-  stroke-opacity: 0;
-}
-
-.nvd3 .nv-axis line {
-  fill: none;
-  stroke: #e5e5e5;
-  shape-rendering: crispEdges;
-}
-
-.nvd3 .nv-axis .zero line,
-/*this selector may not be necessary*/ .nvd3 .nv-axis line.zero {
-  stroke-opacity: .75;
-}
-
-.nvd3 .nv-axis .nv-axisMaxMin text {
-  font-weight: bold;
-}
-
-.nvd3 .x  .nv-axis .nv-axisMaxMin text,
-.nvd3 .x2 .nv-axis .nv-axisMaxMin text,
-.nvd3 .x3 .nv-axis .nv-axisMaxMin text {
-  text-anchor: middle
-}
-
-
-
-/**********
-*  Brush
-*/
-
-.nv-brush .resize path {
-  fill: #eee;
-  stroke: #666;
-}
-
-
-
-/**********
-*  Bars
-*/
-
-.nvd3 .nv-bars .negative rect {
-    zfill: brown;
-}
-
-.nvd3 .nv-bars rect {
-  zfill: steelblue;
-  fill-opacity: .75;
-
-  transition: fill-opacity 250ms linear;
-  -moz-transition: fill-opacity 250ms linear;
-  -webkit-transition: fill-opacity 250ms linear;
-}
-
-.nvd3 .nv-bars rect.hover {
-  fill-opacity: 1;
-}
-
-.nvd3 .nv-bars .hover rect {
-  fill: lightblue;
-}
-
-.nvd3 .nv-bars text {
-  fill: rgba(0,0,0,0);
-}
-
-.nvd3 .nv-bars .hover text {
-  fill: rgba(0,0,0,1);
-}
-
-
-/**********
-*  Bars
-*/
-
-.nvd3 .nv-multibar .nv-groups rect,
-.nvd3 .nv-multibarHorizontal .nv-groups rect,
-.nvd3 .nv-discretebar .nv-groups rect {
-  stroke-opacity: 0;
-
-  transition: fill-opacity 250ms linear;
-  -moz-transition: fill-opacity 250ms linear;
-  -webkit-transition: fill-opacity 250ms linear;
-}
-
-.nvd3 .nv-multibar .nv-groups rect:hover,
-.nvd3 .nv-multibarHorizontal .nv-groups rect:hover,
-.nvd3 .nv-discretebar .nv-groups rect:hover {
-  fill-opacity: 1;
-}
-
-.nvd3 .nv-discretebar .nv-groups text,
-.nvd3 .nv-multibarHorizontal .nv-groups text {
-  font-weight: bold;
-  fill: rgba(0,0,0,1);
-  stroke: rgba(0,0,0,0);
-}
-
-/***********
-*  Pie Chart
-*/
-
-.nvd3.nv-pie path {
-  stroke-opacity: 0;
-  transition: fill-opacity 250ms linear, stroke-width 250ms linear, stroke-opacity 250ms linear;
-  -moz-transition: fill-opacity 250ms linear, stroke-width 250ms linear, stroke-opacity 250ms linear;
-  -webkit-transition: fill-opacity 250ms linear, stroke-width 250ms linear, stroke-opacity 250ms linear;
-
-}
-
-.nvd3.nv-pie .nv-slice text {
-  stroke: #000;
-  stroke-width: 0;
-}
-
-.nvd3.nv-pie path {
-  stroke: #fff;
-  stroke-width: 1px;
-  stroke-opacity: 1;
-}
-
-.nvd3.nv-pie .hover path {
-  fill-opacity: .7;
-}
-.nvd3.nv-pie .nv-label {
-  pointer-events: none;
-}
-.nvd3.nv-pie .nv-label rect {
-  fill-opacity: 0;
-  stroke-opacity: 0;
-}
-
-/**********
-* Lines
-*/
-
-.nvd3 .nv-groups path.nv-line {
-  fill: none;
-  stroke-width: 1.5px;
-  /*
-  stroke-linecap: round;
-  shape-rendering: geometricPrecision;
-
-  transition: stroke-width 250ms linear;
-  -moz-transition: stroke-width 250ms linear;
-  -webkit-transition: stroke-width 250ms linear;
-
-  transition-delay: 250ms
-  -moz-transition-delay: 250ms;
-  -webkit-transition-delay: 250ms;
-  */
-}
-
-.nvd3 .nv-groups path.nv-line.nv-thin-line {
-  stroke-width: 1px;
-}
-
-
-.nvd3 .nv-groups path.nv-area {
-  stroke: none;
-  /*
-  stroke-linecap: round;
-  shape-rendering: geometricPrecision;
-
-  stroke-width: 2.5px;
-  transition: stroke-width 250ms linear;
-  -moz-transition: stroke-width 250ms linear;
-  -webkit-transition: stroke-width 250ms linear;
-
-  transition-delay: 250ms
-  -moz-transition-delay: 250ms;
-  -webkit-transition-delay: 250ms;
-  */
-}
-
-.nvd3 .nv-line.hover path {
-  stroke-width: 6px;
-}
-
-/*
-.nvd3.scatter .groups .point {
-  fill-opacity: 0.1;
-  stroke-opacity: 0.1;
-}
-  */
-
-.nvd3.nv-line .nvd3.nv-scatter .nv-groups .nv-point {
-  fill-opacity: 0;
-  stroke-opacity: 0;
-}
-
-.nvd3.nv-scatter.nv-single-point .nv-groups .nv-point {
-  fill-opacity: .5 !important;
-  stroke-opacity: .5 !important;
-}
-
-
-.with-transitions .nvd3 .nv-groups .nv-point {
-  transition: stroke-width 250ms linear, stroke-opacity 250ms linear;
-  -moz-transition: stroke-width 250ms linear, stroke-opacity 250ms linear;
-  -webkit-transition: stroke-width 250ms linear, stroke-opacity 250ms linear;
-
-}
-
-.nvd3.nv-scatter .nv-groups .nv-point.hover,
-.nvd3 .nv-groups .nv-point.hover {
-  stroke-width: 7px;
-  fill-opacity: .95 !important;
-  stroke-opacity: .95 !important;
-}
-
-
-.nvd3 .nv-point-paths path {
-  stroke: #aaa;
-  stroke-opacity: 0;
-  fill: #eee;
-  fill-opacity: 0;
-}
-
-
-
-.nvd3 .nv-indexLine {
-  cursor: ew-resize;
-}
-
-
-/**********
-* Distribution
-*/
-
-.nvd3 .nv-distribution {
-  pointer-events: none;
-}
-
-
-
-/**********
-*  Scatter
-*/
-
-/* **Attempting to remove this for useVoronoi(false), need to see if it's required anywhere
-.nvd3 .nv-groups .nv-point {
-  pointer-events: none;
-}
-*/
-
-.nvd3 .nv-groups .nv-point.hover {
-  stroke-width: 20px;
-  stroke-opacity: .5;
-}
-
-.nvd3 .nv-scatter .nv-point.hover {
-  fill-opacity: 1;
-}
-
-/*
-.nv-group.hover .nv-point {
-  fill-opacity: 1;
-}
-*/
-
-
-/**********
-*  Stacked Area
-*/
-
-.nvd3.nv-stackedarea path.nv-area {
-  fill-opacity: .7;
-  /*
-  stroke-opacity: .65;
-  fill-opacity: 1;
-  */
-  stroke-opacity: 0;
-
-  transition: fill-opacity 250ms linear, stroke-opacity 250ms linear;
-  -moz-transition: fill-opacity 250ms linear, stroke-opacity 250ms linear;
-  -webkit-transition: fill-opacity 250ms linear, stroke-opacity 250ms linear;
-
-  /*
-  transition-delay: 500ms;
-  -moz-transition-delay: 500ms;
-  -webkit-transition-delay: 500ms;
-  */
-
-}
-
-.nvd3.nv-stackedarea path.nv-area.hover {
-  fill-opacity: .9;
-  /*
-  stroke-opacity: .85;
-  */
-}
-/*
-.d3stackedarea .groups path {
-  stroke-opacity: 0;
-}
-  */
-
-
-
-.nvd3.nv-stackedarea .nv-groups .nv-point {
-  stroke-opacity: 0;
-  fill-opacity: 0;
-}
-
-/*
-.nvd3.nv-stackedarea .nv-groups .nv-point.hover {
-  stroke-width: 20px;
-  stroke-opacity: .75;
-  fill-opacity: 1;
-}*/
-
-
-
-/**********
-*  Line Plus Bar
-*/
-
-.nvd3.nv-linePlusBar .nv-bar rect {
-  fill-opacity: .75;
-}
-
-.nvd3.nv-linePlusBar .nv-bar rect:hover {
-  fill-opacity: 1;
-}
-
-
-/**********
-*  Bullet
-*/
-
-.nvd3.nv-bullet { font: 10px sans-serif; }
-.nvd3.nv-bullet .nv-measure { fill-opacity: .8; }
-.nvd3.nv-bullet .nv-measure:hover { fill-opacity: 1; }
-.nvd3.nv-bullet .nv-marker { stroke: #000; stroke-width: 2px; }
-.nvd3.nv-bullet .nv-markerTriangle { stroke: #000; fill: #fff; stroke-width: 1.5px; }
-.nvd3.nv-bullet .nv-tick line { stroke: #666; stroke-width: .5px; }
-.nvd3.nv-bullet .nv-range.nv-s0 { fill: #eee; }
-.nvd3.nv-bullet .nv-range.nv-s1 { fill: #ddd; }
-.nvd3.nv-bullet .nv-range.nv-s2 { fill: #ccc; }
-.nvd3.nv-bullet .nv-title { font-size: 14px; font-weight: bold; }
-.nvd3.nv-bullet .nv-subtitle { fill: #999; }
-
-
-.nvd3.nv-bullet .nv-range {
-  fill: #bababa;
-  fill-opacity: .4;
-}
-.nvd3.nv-bullet .nv-range:hover {
-  fill-opacity: .7;
-}
-
-
-
-/**********
-* Sparkline
-*/
-
-.nvd3.nv-sparkline path {
-  fill: none;
-}
-
-.nvd3.nv-sparklineplus g.nv-hoverValue {
-  pointer-events: none;
-}
-
-.nvd3.nv-sparklineplus .nv-hoverValue line {
-  stroke: #333;
-  stroke-width: 1.5px;
- }
-
-.nvd3.nv-sparklineplus,
-.nvd3.nv-sparklineplus g {
-  pointer-events: all;
-}
-
-.nvd3 .nv-hoverArea {
-  fill-opacity: 0;
-  stroke-opacity: 0;
-}
-
-.nvd3.nv-sparklineplus .nv-xValue,
-.nvd3.nv-sparklineplus .nv-yValue {
-  /*
-  stroke: #666;
-  */
-  stroke-width: 0;
-  font-size: .9em;
-  font-weight: normal;
-}
-
-.nvd3.nv-sparklineplus .nv-yValue {
-  stroke: #f66;
-}
-
-.nvd3.nv-sparklineplus .nv-maxValue {
-  stroke: #2ca02c;
-  fill: #2ca02c;
-}
-
-.nvd3.nv-sparklineplus .nv-minValue {
-  stroke: #d62728;
-  fill: #d62728;
-}
-
-.nvd3.nv-sparklineplus .nv-currentValue {
-  /*
-  stroke: #444;
-  fill: #000;
-  */
-  font-weight: bold;
-  font-size: 1.1em;
-}
-
-/**********
-* historical stock
-*/
-
-.nvd3.nv-ohlcBar .nv-ticks .nv-tick {
-  stroke-width: 2px;
-}
-
-.nvd3.nv-ohlcBar .nv-ticks .nv-tick.hover {
-  stroke-width: 4px;
-}
-
-.nvd3.nv-ohlcBar .nv-ticks .nv-tick.positive {
- stroke: #2ca02c;
-}
-
-.nvd3.nv-ohlcBar .nv-ticks .nv-tick.negative {
- stroke: #d62728;
-}
-
-.nvd3.nv-historicalStockChart .nv-axis .nv-axislabel {
-  font-weight: bold;
-}
-
-.nvd3.nv-historicalStockChart .nv-dragTarget {
-  fill-opacity: 0;
-  stroke: none;
-  cursor: move;
-}
-
-.nvd3 .nv-brush .extent {
-  /*
-  cursor: ew-resize !important;
-  */
-  fill-opacity: 0 !important;
-}
-
-.nvd3 .nv-brushBackground rect {
-  stroke: #000;
-  stroke-width: .4;
-  fill: #fff;
-  fill-opacity: .7;
-}
-
-
-
-/**********
-* Indented Tree
-*/
-
-
-/**
- * TODO: the following 3 selectors are based on classes used in the example.  I should either make them standard and leave them here, or move to a CSS file not included in the library
- */
-.nvd3.nv-indentedtree .name {
-  margin-left: 5px;
-}
-
-.nvd3.nv-indentedtree .clickable {
-  color: #08C;
-  cursor: pointer;
-}
-
-.nvd3.nv-indentedtree span.clickable:hover {
-  color: #005580;
-  text-decoration: underline;
-}
-
-
-.nvd3.nv-indentedtree .nv-childrenCount {
-  display: inline-block;
-  margin-left: 5px;
-}
-
-.nvd3.nv-indentedtree .nv-treeicon {
-  cursor: pointer;
-  /*
-  cursor: n-resize;
-  */
-}
-
-.nvd3.nv-indentedtree .nv-treeicon.nv-folded {
-  cursor: pointer;
-  /*
-  cursor: s-resize;
-  */
-}
-
-/**********
-* Parallel Coordinates
-*/
-
-.nvd3 .background path {
-  fill: none;
-  stroke: #ccc;
-  stroke-opacity: .4;
-  shape-rendering: crispEdges;
-}
-
-.nvd3 .foreground path {
-  fill: none;
-  stroke: steelblue;
-  stroke-opacity: .7;
-}
-
-.nvd3 .brush .extent {
-  fill-opacity: .3;
-  stroke: #fff;
-  shape-rendering: crispEdges;
-}
-
-.nvd3 .axis line, .axis path {
-  fill: none;
-  stroke: #000;
-  shape-rendering: crispEdges;
-}
-
-.nvd3 .axis text {
-  text-shadow: 0 1px 0 #fff;
-}
-
-/****
-Interactive Layer
-*/
-.nvd3 .nv-interactiveGuideLine {
-  pointer-events:none;
-}
-.nvd3 line.nv-guideline {
-  stroke: #ccc;
-}
\ No newline at end of file
diff --git a/airflow/www_rbac/static/nv.d3.js b/airflow/www_rbac/static/nv.d3.js
deleted file mode 100644
index faa9a00ad1b0f..0000000000000
--- a/airflow/www_rbac/static/nv.d3.js
+++ /dev/null
@@ -1,14260 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-(function(){
-
-var nv = window.nv || {};
-
-
-nv.version = '1.1.11b';
-nv.dev = true //set false when in production
-
-window.nv = nv;
-
-nv.tooltip = {}; // For the tooltip system
-nv.utils = {}; // Utility subsystem
-nv.models = {}; //stores all the possible models/components
-nv.charts = {}; //stores all the ready to use charts
-nv.graphs = []; //stores all the graphs currently on the page
-nv.logs = {}; //stores some statistics and potential error messages
-
-nv.dispatch = d3.dispatch('render_start', 'render_end');
-
-// *************************************************************************
-//  Development render timers - disabled if dev = false
-
-if (nv.dev) {
-  nv.dispatch.on('render_start', function(e) {
-    nv.logs.startTime = +new Date();
-  });
-
-  nv.dispatch.on('render_end', function(e) {
-    nv.logs.endTime = +new Date();
-    nv.logs.totalTime = nv.logs.endTime - nv.logs.startTime;
-    nv.log('total', nv.logs.totalTime); // used for development, to keep track of graph generation times
-  });
-}
-
-// ********************************************
-//  Public Core NV functions
-
-// Logs all arguments, and returns the last so you can test things in place
-// Note: in IE8 console.log is an object not a function, and if modernizr is used
-// then calling Function.prototype.bind with with anything other than a function
-// causes a TypeError to be thrown.
-nv.log = function() {
-  if (nv.dev && console.log && console.log.apply)
-    console.log.apply(console, arguments)
-  else if (nv.dev && typeof console.log == "function" && Function.prototype.bind) {
-    var log = Function.prototype.bind.call(console.log, console);
-    log.apply(console, arguments);
-  }
-  return arguments[arguments.length - 1];
-};
-
-
-nv.render = function render(step) {
-  step = step || 1; // number of graphs to generate in each timeout loop
-
-  nv.render.active = true;
-  nv.dispatch.render_start();
-
-  setTimeout(function() {
-    var chart, graph;
-
-    for (var i = 0; i < step && (graph = nv.render.queue[i]); i++) {
-      chart = graph.generate();
-      if (typeof graph.callback == typeof(Function)) graph.callback(chart);
-      nv.graphs.push(chart);
-    }
-
-    nv.render.queue.splice(0, i);
-
-    if (nv.render.queue.length) setTimeout(arguments.callee, 0);
-    else { nv.render.active = false; nv.dispatch.render_end(); }
-  }, 0);
-};
-
-nv.render.active = false;
-nv.render.queue = [];
-
-nv.addGraph = function(obj) {
-  if (typeof arguments[0] === typeof(Function))
-    obj = {generate: arguments[0], callback: arguments[1]};
-
-  nv.render.queue.push(obj);
-
-  if (!nv.render.active) nv.render();
-};
-
-nv.identity = function(d) { return d; };
-
-nv.strip = function(s) { return s.replace(/(\s|&)/g,''); };
-
-function daysInMonth(month,year) {
-  return (new Date(year, month+1, 0)).getDate();
-}
-
-function d3_time_range(floor, step, number) {
-  return function(t0, t1, dt) {
-    var time = floor(t0), times = [];
-    if (time < t0) step(time);
-    if (dt > 1) {
-      while (time < t1) {
-        var date = new Date(+time);
-        if ((number(date) % dt === 0)) times.push(date);
-        step(time);
-      }
-    } else {
-      while (time < t1) { times.push(new Date(+time)); step(time); }
-    }
-    return times;
-  };
-}
-
-d3.time.monthEnd = function(date) {
-  return new Date(date.getFullYear(), date.getMonth(), 0);
-};
-
-d3.time.monthEnds = d3_time_range(d3.time.monthEnd, function(date) {
-    date.setUTCDate(date.getUTCDate() + 1);
-    date.setDate(daysInMonth(date.getMonth() + 1, date.getFullYear()));
-  }, function(date) {
-    return date.getMonth();
-  }
-);
-
-/* Utility class to handle creation of an interactive layer.
-This places a rectangle on top of the chart. When you mouse move over it, it sends a dispatch
-containing the X-coordinate. It can also render a vertical line where the mouse is located.
-
-dispatch.elementMousemove is the important event to latch onto.  It is fired whenever the mouse moves over
-the rectangle. The dispatch is given one object which contains the mouseX/Y location.
-It also has 'pointXValue', which is the conversion of mouseX to the x-axis scale.
-*/
-nv.interactiveGuideline = function() {
-	"use strict";
-	var tooltip = nv.models.tooltip();
-	//Public settings
-	var width = null
-	, height = null
-    //Please pass in the bounding chart's top and left margins
-    //This is important for calculating the correct mouseX/Y positions.
-	, margin = {left: 0, top: 0}
-	, xScale = d3.scale.linear()
-	, yScale = d3.scale.linear()
-	, dispatch = d3.dispatch('elementMousemove', 'elementMouseout','elementDblclick')
-	, showGuideLine = true
-	, svgContainer = null  
-    //Must pass in the bounding chart's <svg> container.
-    //The mousemove event is attached to this container.
-	;
-
-	//Private variables
-	var isMSIE = navigator.userAgent.indexOf("MSIE") !== -1  //Check user-agent for Microsoft Internet Explorer.
-	;
-
-
-	function layer(selection) {
-		selection.each(function(data) {
-				var container = d3.select(this);
-				
-				var availableWidth = (width || 960), availableHeight = (height || 400);
-
-				var wrap = container.selectAll("g.nv-wrap.nv-interactiveLineLayer").data([data]);
-				var wrapEnter = wrap.enter()
-								.append("g").attr("class", " nv-wrap nv-interactiveLineLayer");
-								
-				
-				wrapEnter.append("g").attr("class","nv-interactiveGuideLine");
-				
-				if (!svgContainer) {
-					return;
-				}
-
-                function mouseHandler() {
-                      var d3mouse = d3.mouse(this);
-                      var mouseX = d3mouse[0];
-                      var mouseY = d3mouse[1];
-                      var subtractMargin = true;
-                      var mouseOutAnyReason = false;
-                      if (isMSIE) {
-                         /*
-                            D3.js (or maybe SVG.getScreenCTM) has a nasty bug in Internet Explorer 10.
-                            d3.mouse() returns incorrect X,Y mouse coordinates when mouse moving
-                            over a rect in IE 10.
-                            However, d3.event.offsetX/Y also returns the mouse coordinates
-                            relative to the triggering <rect>. So we use offsetX/Y on IE.  
-                         */
-                         mouseX = d3.event.offsetX;
-                         mouseY = d3.event.offsetY;
-
-                         /*
-                            On IE, if you attach a mouse event listener to the <svg> container,
-                            it will actually trigger it for all the child elements (like <path>, <circle>, etc).
-                            When this happens on IE, the offsetX/Y is set to where ever the child element
-                            is located.
-                            As a result, we do NOT need to subtract margins to figure out the mouse X/Y
-                            position under this scenario. Removing the line below *will* cause 
-                            the interactive layer to not work right on IE.
-                         */
-                         if(d3.event.target.tagName !== "svg")
-                            subtractMargin = false;
-
-                         if (d3.event.target.className.baseVal.match("nv-legend"))
-                         	mouseOutAnyReason = true;
-                          
-                      }
-
-                      if(subtractMargin) {
-                         mouseX -= margin.left;
-                         mouseY -= margin.top;
-                      }
-
-                      /* If mouseX/Y is outside of the chart's bounds,
-                      trigger a mouseOut event.
-                      */
-                      if (mouseX < 0 || mouseY < 0 
-                        || mouseX > availableWidth || mouseY > availableHeight
-                        || (d3.event.relatedTarget && d3.event.relatedTarget.ownerSVGElement === undefined)
-                        || mouseOutAnyReason
-                        ) 
-                      {
-                      		if (isMSIE) {
-                      			if (d3.event.relatedTarget 
-                      				&& d3.event.relatedTarget.ownerSVGElement === undefined
-                      				&& d3.event.relatedTarget.className.match(tooltip.nvPointerEventsClass)) {
-                      				return;
-                      			}
-                      		}
-                            dispatch.elementMouseout({
-                               mouseX: mouseX,
-                               mouseY: mouseY
-                            });
-                            layer.renderGuideLine(null); //hide the guideline
-                            return;
-                      }
-                      
-                      var pointXValue = xScale.invert(mouseX);
-                      dispatch.elementMousemove({
-                            mouseX: mouseX,
-                            mouseY: mouseY,
-                            pointXValue: pointXValue
-                      });
-
-                      //If user double clicks the layer, fire a elementDblclick dispatch.
-                      if (d3.event.type === "dblclick") {
-                        dispatch.elementDblclick({
-                            mouseX: mouseX,
-                            mouseY: mouseY,
-                            pointXValue: pointXValue
-                        });
-                      }
-                }
-
-				svgContainer
-				      .on("mousemove",mouseHandler, true)
-				      .on("mouseout" ,mouseHandler,true)
-                      .on("dblclick" ,mouseHandler)
-				      ;
-
-				 //Draws a vertical guideline at the given X postion.
-				layer.renderGuideLine = function(x) {
-				 	if (!showGuideLine) return;
-				 	var line = wrap.select(".nv-interactiveGuideLine")
-				 	      .selectAll("line")
-				 	      .data((x != null) ? [nv.utils.NaNtoZero(x)] : [], String);
-
-				 	line.enter()
-				 		.append("line")
-				 		.attr("class", "nv-guideline")
-				 		.attr("x1", function(d) { return d;})
-				 		.attr("x2", function(d) { return d;})
-				 		.attr("y1", availableHeight)
-				 		.attr("y2",0)
-				 		;
-				 	line.exit().remove();
-
-				}
-		});
-	}
-
-	layer.dispatch = dispatch;
-	layer.tooltip = tooltip;
-
-	layer.margin = function(_) {
-	    if (!arguments.length) return margin;
-	    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-	    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-	    return layer;
-    };
-
-	layer.width = function(_) {
-		if (!arguments.length) return width;
-		width = _;
-		return layer;
-	};
-
-	layer.height = function(_) {
-		if (!arguments.length) return height;
-		height = _;
-		return layer;
-	};
-
-	layer.xScale = function(_) {
-		if (!arguments.length) return xScale;
-		xScale = _;
-		return layer;
-	};
-
-	layer.showGuideLine = function(_) {
-		if (!arguments.length) return showGuideLine;
-		showGuideLine = _;
-		return layer;
-	};
-
-	layer.svgContainer = function(_) {
-		if (!arguments.length) return svgContainer;
-		svgContainer = _;
-		return layer;
-	};
-
-
-	return layer;
-};
-
-/* Utility class that uses d3.bisect to find the index in a given array, where a search value can be inserted.
-This is different from normal bisectLeft; this function finds the nearest index to insert the search value.
-
-For instance, lets say your array is [1,2,3,5,10,30], and you search for 28. 
-Normal d3.bisectLeft will return 4, because 28 is inserted after the number 10.  But interactiveBisect will return 5
-because 28 is closer to 30 than 10.
-
-Unit tests can be found in: interactiveBisectTest.html
-
-Has the following known issues:
-   * Will not work if the data points move backwards (ie, 10,9,8,7, etc) or if the data points are in random order.
-   * Won't work if there are duplicate x coordinate values.
-*/
-nv.interactiveBisect = function (values, searchVal, xAccessor) {
-	  "use strict";
-      if (! values instanceof Array) return null;
-      if (typeof xAccessor !== 'function') xAccessor = function(d,i) { return d.x;}
-
-      var bisect = d3.bisector(xAccessor).left;
-      var index = d3.max([0, bisect(values,searchVal) - 1]);
-      var currentValue = xAccessor(values[index], index);
-      if (typeof currentValue === 'undefined') currentValue = index;
-
-      if (currentValue === searchVal) return index;  //found exact match
-
-      var nextIndex = d3.min([index+1, values.length - 1]);
-      var nextValue = xAccessor(values[nextIndex], nextIndex);
-      if (typeof nextValue === 'undefined') nextValue = nextIndex;
-
-      if (Math.abs(nextValue - searchVal) >= Math.abs(currentValue - searchVal))
-          return index;
-      else
-          return nextIndex
-};/* Tooltip rendering model for nvd3 charts.
-window.nv.models.tooltip is the updated,new way to render tooltips.
-
-window.nv.tooltip.show is the old tooltip code.
-window.nv.tooltip.* also has various helper methods.
-*/
-(function() {
-  "use strict";
-  window.nv.tooltip = {};
-
-  /* Model which can be instantiated to handle tooltip rendering.
-    Example usage: 
-    var tip = nv.models.tooltip().gravity('w').distance(23)
-                .data(myDataObject);
-
-        tip();    //just invoke the returned function to render tooltip.
-  */
-  window.nv.models.tooltip = function() {
-        var content = null    //HTML contents of the tooltip.  If null, the content is generated via the data variable.
-        ,   data = null     /* Tooltip data. If data is given in the proper format, a consistent tooltip is generated.
-        Format of data:
-        {
-            key: "Date",
-            value: "August 2009", 
-            series: [
-                    {
-                        key: "Series 1",
-                        value: "Value 1",
-                        color: "#000"
-                    },
-                    {
-                        key: "Series 2",
-                        value: "Value 2",
-                        color: "#00f"
-                    }
-            ]
-
-        }
-
-        */
-        ,   gravity = 'w'   //Can be 'n','s','e','w'. Determines how tooltip is positioned.
-        ,   distance = 50   //Distance to offset tooltip from the mouse location.
-        ,   snapDistance = 25   //Tolerance allowed before tooltip is moved from its current position (creates 'snapping' effect)
-        ,   fixedTop = null //If not null, this fixes the top position of the tooltip.
-        ,   classes = null  //Attaches additional CSS classes to the tooltip DIV that is created.
-        ,   chartContainer = null   //Parent DIV, of the SVG Container that holds the chart.
-        ,   tooltipElem = null  //actual DOM element representing the tooltip.
-        ,   position = {left: null, top: null}      //Relative position of the tooltip inside chartContainer.
-        ,   enabled = true  //True -> tooltips are rendered. False -> don't render tooltips.
-        //Generates a unique id when you create a new tooltip() object
-        ,   id = "nvtooltip-" + Math.floor(Math.random() * 100000)
-        ;
-
-        //CSS class to specify whether element should not have mouse events.
-        var  nvPointerEventsClass = "nv-pointer-events-none";
-
-        //Format function for the tooltip values column
-        var valueFormatter = function(d,i) {
-            return d;
-        };
-
-        //Format function for the tooltip header value.
-        var headerFormatter = function(d) {
-            return d;
-        };
-
-        //By default, the tooltip model renders a beautiful table inside a DIV.
-        //You can override this function if a custom tooltip is desired.
-        var contentGenerator = function(d) {
-            if (content != null) return content;
-
-            if (d == null) return '';
-
-            var table = d3.select(document.createElement("table"));
-            var theadEnter = table.selectAll("thead")
-                .data([d])
-                .enter().append("thead");
-            theadEnter.append("tr")
-                .append("td")
-                .attr("colspan",3)
-                .append("strong")
-                    .classed("x-value",true)
-                    .text(headerFormatter(d.value));
-
-            var tbodyEnter = table.selectAll("tbody")
-                .data([d])
-                .enter().append("tbody");
-            var trowEnter = tbodyEnter.selectAll("tr")
-                .data(function(p) { return p.series})
-                .enter()
-                .append("tr")
-                .classed("highlight", function(p) { return p.highlight})
-                ;
-
-            trowEnter.append("td")
-                .classed("legend-color-guide",true)
-                .append("div")
-                    .style("background-color", function(p) { return p.color});
-            trowEnter.append("td")
-                .classed("key",true)
-                .text(function(p) {return p.key});
-            trowEnter.append("td")
-                .classed("value",true)
-                .text(function(p,i) { return valueFormatter(p.value,i) });
-
-            trowEnter.selectAll("td").each(function(p) {
-                if (p.highlight)
-                    d3.select(this)
-                        .style("border-bottom","solid 1px " + p.color)
-                        .style("border-top","solid 1px " + p.color)
-                        ;
-            });
-
-            var html = table.node().outerHTML;
-            if (d.footer !== undefined)
-                html += "<div class='footer'>" + d.footer + "</div>";
-            return html;
-
-        };
-
-        var dataSeriesExists = function(d) {
-            if (d && d.series && d.series.length > 0) return true;
-
-            return false;
-        };
-
-        //In situations where the chart is in a 'viewBox', re-position the tooltip based on how far chart is zoomed.
-        function convertViewBoxRatio() {
-            if (chartContainer) {
-              var svg = d3.select(chartContainer);
-              if (svg.node().tagName !== "svg") {
-                 svg = svg.select("svg");
-              }
-              var viewBox = (svg.node()) ? svg.attr('viewBox') : null;
-              if (viewBox) {
-                viewBox = viewBox.split(' ');
-                var ratio = parseInt(svg.style('width')) / viewBox[2];
-                
-                position.left = position.left * ratio;
-                position.top  = position.top * ratio;
-              }
-            }
-        }
-
-        //Creates new tooltip container, or uses existing one on DOM.
-        function getTooltipContainer(newContent) {
-            var body;
-            if (chartContainer)
-                body = d3.select(chartContainer);
-            else
-                body = d3.select("body");
-
-            var container = body.select(".nvtooltip");
-            if (container.node() === null) {
-                //Create new tooltip div if it doesn't exist on DOM.
-                container = body.append("div")
-                    .attr("class", "nvtooltip " + (classes? classes: "xy-tooltip"))
-                    .attr("id",id)
-                    ;
-            }
-        
-
-            container.node().innerHTML = newContent;
-            container.style("top",0).style("left",0).style("opacity",0);
-            container.selectAll("div, table, td, tr").classed(nvPointerEventsClass,true)
-            container.classed(nvPointerEventsClass,true);
-            return container.node();
-        }
-
-        
-
-        //Draw the tooltip onto the DOM.
-        function nvtooltip() {
-            if (!enabled) return;
-            if (!dataSeriesExists(data)) return;
-
-            convertViewBoxRatio();
-
-            var left = position.left;
-            var top = (fixedTop != null) ? fixedTop : position.top;
-            var container = getTooltipContainer(contentGenerator(data));
-            tooltipElem = container;
-            if (chartContainer) {
-                var svgComp = chartContainer.getElementsByTagName("svg")[0];
-                var boundRect = (svgComp) ? svgComp.getBoundingClientRect() : chartContainer.getBoundingClientRect();
-                var svgOffset = {left:0,top:0};
-                if (svgComp) {
-                    var svgBound = svgComp.getBoundingClientRect();
-                    var chartBound = chartContainer.getBoundingClientRect();
-                    var svgBoundTop = svgBound.top;
-                    
-                    //Defensive code. Sometimes, svgBoundTop can be a really negative
-                    //  number, like -134254. That's a bug. 
-                    //  If such a number is found, use zero instead. FireFox bug only
-                    if (svgBoundTop < 0) {
-                        var containerBound = chartContainer.getBoundingClientRect();
-                        svgBoundTop = (Math.abs(svgBoundTop) > containerBound.height) ? 0 : svgBoundTop;
-                    } 
-                    svgOffset.top = Math.abs(svgBoundTop - chartBound.top);
-                    svgOffset.left = Math.abs(svgBound.left - chartBound.left);
-                }
-                //If the parent container is an overflow <div> with scrollbars, subtract the scroll offsets.
-                //You need to also add any offset between the <svg> element and its containing <div>
-                //Finally, add any offset of the containing <div> on the whole page.
-                left += chartContainer.offsetLeft + svgOffset.left - 2*chartContainer.scrollLeft;
-                top += chartContainer.offsetTop + svgOffset.top - 2*chartContainer.scrollTop;
-            }
-
-            if (snapDistance && snapDistance > 0) {
-                top = Math.floor(top/snapDistance) * snapDistance;
-            }
-
-            nv.tooltip.calcTooltipPosition([left,top], gravity, distance, container);
-            return nvtooltip;
-        };
-
-        nvtooltip.nvPointerEventsClass = nvPointerEventsClass;
-        
-        nvtooltip.content = function(_) {
-            if (!arguments.length) return content;
-            content = _;
-            return nvtooltip;
-        };
-
-        //Returns tooltipElem...not able to set it.
-        nvtooltip.tooltipElem = function() {
-            return tooltipElem;
-        };
-
-        nvtooltip.contentGenerator = function(_) {
-            if (!arguments.length) return contentGenerator;
-            if (typeof _ === 'function') {
-                contentGenerator = _;
-            }
-            return nvtooltip;
-        };
-
-        nvtooltip.data = function(_) {
-            if (!arguments.length) return data;
-            data = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.gravity = function(_) {
-            if (!arguments.length) return gravity;
-            gravity = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.distance = function(_) {
-            if (!arguments.length) return distance;
-            distance = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.snapDistance = function(_) {
-            if (!arguments.length) return snapDistance;
-            snapDistance = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.classes = function(_) {
-            if (!arguments.length) return classes;
-            classes = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.chartContainer = function(_) {
-            if (!arguments.length) return chartContainer;
-            chartContainer = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.position = function(_) {
-            if (!arguments.length) return position;
-            position.left = (typeof _.left !== 'undefined') ? _.left : position.left;
-            position.top = (typeof _.top !== 'undefined') ? _.top : position.top;
-            return nvtooltip;
-        };
-
-        nvtooltip.fixedTop = function(_) {
-            if (!arguments.length) return fixedTop;
-            fixedTop = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.enabled = function(_) {
-            if (!arguments.length) return enabled;
-            enabled = _;
-            return nvtooltip;
-        };
-
-        nvtooltip.valueFormatter = function(_) {
-            if (!arguments.length) return valueFormatter;
-            if (typeof _ === 'function') {
-                valueFormatter = _;
-            }
-            return nvtooltip;
-        };
-
-        nvtooltip.headerFormatter = function(_) {
-            if (!arguments.length) return headerFormatter;
-            if (typeof _ === 'function') {
-                headerFormatter = _;
-            }
-            return nvtooltip;
-        };
-
-        //id() is a read-only function. You can't use it to set the id.
-        nvtooltip.id = function() {
-            return id;
-        };
-
-
-        return nvtooltip;
-  };
-
-
-  //Original tooltip.show function. Kept for backward compatibility.
-  // pos = [left,top]
-  nv.tooltip.show = function(pos, content, gravity, dist, parentContainer, classes) {
-      
-        //Create new tooltip div if it doesn't exist on DOM.
-        var   container = document.createElement('div');
-        container.className = 'nvtooltip ' + (classes ? classes : 'xy-tooltip');
-
-        var body = parentContainer;
-        if ( !parentContainer || parentContainer.tagName.match(/g|svg/i)) {
-            //If the parent element is an SVG element, place tooltip in the <body> element.
-            body = document.getElementsByTagName('body')[0];
-        }
-   
-        container.style.left = 0;
-        container.style.top = 0;
-        container.style.opacity = 0;
-        container.innerHTML = content;
-        body.appendChild(container);
-        
-        //If the parent container is an overflow <div> with scrollbars, subtract the scroll offsets.
-        if (parentContainer) {
-           pos[0] = pos[0] - parentContainer.scrollLeft;
-           pos[1] = pos[1] - parentContainer.scrollTop;
-        }
-        nv.tooltip.calcTooltipPosition(pos, gravity, dist, container);
-  };
-
-  //Looks up the ancestry of a DOM element, and returns the first NON-svg node.
-  nv.tooltip.findFirstNonSVGParent = function(Elem) {
-            while(Elem.tagName.match(/^g|svg$/i) !== null) {
-                Elem = Elem.parentNode;
-            }
-            return Elem;
-  };
-
-  //Finds the total offsetTop of a given DOM element.
-  //Looks up the entire ancestry of an element, up to the first relatively positioned element.
-  nv.tooltip.findTotalOffsetTop = function ( Elem, initialTop ) {
-                var offsetTop = initialTop;
-                
-                do {
-                    if( !isNaN( Elem.offsetTop ) ) {
-                        offsetTop += (Elem.offsetTop);
-                    }
-                } while( Elem = Elem.offsetParent );
-                return offsetTop;
-  };
-
-  //Finds the total offsetLeft of a given DOM element.
-  //Looks up the entire ancestry of an element, up to the first relatively positioned element.
-  nv.tooltip.findTotalOffsetLeft = function ( Elem, initialLeft) {
-                var offsetLeft = initialLeft;
-                
-                do {
-                    if( !isNaN( Elem.offsetLeft ) ) {
-                        offsetLeft += (Elem.offsetLeft);
-                    }
-                } while( Elem = Elem.offsetParent );
-                return offsetLeft;
-  };
-
-  //Global utility function to render a tooltip on the DOM.
-  //pos = [left,top] coordinates of where to place the tooltip, relative to the SVG chart container.
-  //gravity = how to orient the tooltip
-  //dist = how far away from the mouse to place tooltip
-  //container = tooltip DIV
-  nv.tooltip.calcTooltipPosition = function(pos, gravity, dist, container) {
-
-            var height = parseInt(container.offsetHeight),
-                width = parseInt(container.offsetWidth),
-                windowWidth = nv.utils.windowSize().width,
-                windowHeight = nv.utils.windowSize().height,
-                scrollTop = window.pageYOffset,
-                scrollLeft = window.pageXOffset,
-                left, top;
-
-            windowHeight = window.innerWidth >= document.body.scrollWidth ? windowHeight : windowHeight - 16;
-            windowWidth = window.innerHeight >= document.body.scrollHeight ? windowWidth : windowWidth - 16;
-
-            gravity = gravity || 's';
-            dist = dist || 20;
-
-            var tooltipTop = function ( Elem ) {
-                return nv.tooltip.findTotalOffsetTop(Elem, top);
-            };
-
-            var tooltipLeft = function ( Elem ) {
-                return nv.tooltip.findTotalOffsetLeft(Elem,left);
-            };
-
-            switch (gravity) {
-              case 'e':
-                left = pos[0] - width - dist;
-                top = pos[1] - (height / 2);
-                var tLeft = tooltipLeft(container);
-                var tTop = tooltipTop(container);
-                if (tLeft < scrollLeft) left = pos[0] + dist > scrollLeft ? pos[0] + dist : scrollLeft - tLeft + left;
-                if (tTop < scrollTop) top = scrollTop - tTop + top;
-                if (tTop + height > scrollTop + windowHeight) top = scrollTop + windowHeight - tTop + top - height;
-                break;
-              case 'w':
-                left = pos[0] + dist;
-                top = pos[1] - (height / 2);
-                var tLeft = tooltipLeft(container);
-                var tTop = tooltipTop(container);
-                if (tLeft + width > windowWidth) left = pos[0] - width - dist;
-                if (tTop < scrollTop) top = scrollTop + 5;
-                if (tTop + height > scrollTop + windowHeight) top = scrollTop + windowHeight - tTop + top - height;
-                break;
-              case 'n':
-                left = pos[0] - (width / 2) - 5;
-                top = pos[1] + dist;
-                var tLeft = tooltipLeft(container);
-                var tTop = tooltipTop(container);
-                if (tLeft < scrollLeft) left = scrollLeft + 5;
-                if (tLeft + width > windowWidth) left = left - width/2 + 5;
-                if (tTop + height > scrollTop + windowHeight) top = scrollTop + windowHeight - tTop + top - height;
-                break;
-              case 's':
-                left = pos[0] - (width / 2);
-                top = pos[1] - height - dist;
-                var tLeft = tooltipLeft(container);
-                var tTop = tooltipTop(container);
-                if (tLeft < scrollLeft) left = scrollLeft + 5;
-                if (tLeft + width > windowWidth) left = left - width/2 + 5;
-                if (scrollTop > tTop) top = scrollTop;
-                break;
-              case 'none':
-                left = pos[0];
-                top = pos[1] - dist;
-                var tLeft = tooltipLeft(container);
-                var tTop = tooltipTop(container);
-                break;
-            }
-
-
-            container.style.left = left+'px';
-            container.style.top = top+'px';
-            container.style.opacity = 1;
-            container.style.position = 'absolute'; 
-
-            return container;
-    };
-
-    //Global utility function to remove tooltips from the DOM.
-    nv.tooltip.cleanup = function() {
-
-              // Find the tooltips, mark them for removal by this class (so others cleanups won't find it)
-              var tooltips = document.getElementsByClassName('nvtooltip');
-              var purging = [];
-              while(tooltips.length) {
-                purging.push(tooltips[0]);
-                tooltips[0].style.transitionDelay = '0 !important';
-                tooltips[0].style.opacity = 0;
-                tooltips[0].className = 'nvtooltip-pending-removal';
-              }
-
-              setTimeout(function() {
-
-                  while (purging.length) {
-                     var removeMe = purging.pop();
-                      removeMe.parentNode.removeChild(removeMe);
-                  }
-            }, 500);
-    };
-
-})();
-
-nv.utils.windowSize = function() {
-    // Sane defaults
-    var size = {width: 640, height: 480};
-
-    // Earlier IE uses Doc.body
-    if (document.body && document.body.offsetWidth) {
-        size.width = document.body.offsetWidth;
-        size.height = document.body.offsetHeight;
-    }
-
-    // IE can use depending on mode it is in
-    if (document.compatMode=='CSS1Compat' &&
-        document.documentElement &&
-        document.documentElement.offsetWidth ) {
-        size.width = document.documentElement.offsetWidth;
-        size.height = document.documentElement.offsetHeight;
-    }
-
-    // Most recent browsers use
-    if (window.innerWidth && window.innerHeight) {
-        size.width = window.innerWidth;
-        size.height = window.innerHeight;
-    }
-    return (size);
-};
-
-
-
-// Easy way to bind multiple functions to window.onresize
-// TODO: give a way to remove a function after its bound, other than removing all of them
-nv.utils.windowResize = function(fun){
-  if (fun === undefined) return;
-  var oldresize = window.onresize;
-
-  window.onresize = function(e) {
-    if (typeof oldresize == 'function') oldresize(e);
-    fun(e);
-  }
-}
-
-// Backwards compatible way to implement more d3-like coloring of graphs.
-// If passed an array, wrap it in a function which implements the old default
-// behavior
-nv.utils.getColor = function(color) {
-    if (!arguments.length) return nv.utils.defaultColor(); //if you pass in nothing, get default colors back
-
-    if( Object.prototype.toString.call( color ) === '[object Array]' )
-        return function(d, i) { return d.color || color[i % color.length]; };
-    else
-        return color;
-        //can't really help it if someone passes rubbish as color
-}
-
-// Default color chooser uses the index of an object as before.
-nv.utils.defaultColor = function() {
-    var colors = d3.scale.category20().range();
-    return function(d, i) { return d.color || colors[i % colors.length] };
-}
-
-
-// Returns a color function that takes the result of 'getKey' for each series and
-// looks for a corresponding color from the dictionary,
-nv.utils.customTheme = function(dictionary, getKey, defaultColors) {
-  getKey = getKey || function(series) { return series.key }; // use default series.key if getKey is undefined
-  defaultColors = defaultColors || d3.scale.category20().range(); //default color function
-
-  var defIndex = defaultColors.length; //current default color (going in reverse)
-
-  return function(series, index) {
-    var key = getKey(series);
-
-    if (!defIndex) defIndex = defaultColors.length; //used all the default colors, start over
-
-    if (typeof dictionary[key] !== "undefined")
-      return (typeof dictionary[key] === "function") ? dictionary[key]() : dictionary[key];
-    else
-      return defaultColors[--defIndex]; // no match in dictionary, use default color
-  }
-}
-
-
-
-// From the PJAX example on d3js.org, while this is not really directly needed
-// it's a very cool method for doing pjax, I may expand upon it a little bit,
-// open to suggestions on anything that may be useful
-nv.utils.pjax = function(links, content) {
-  d3.selectAll(links).on("click", function() {
-    history.pushState(this.href, this.textContent, this.href);
-    load(this.href);
-    d3.event.preventDefault();
-  });
-
-  function load(href) {
-    d3.html(href, function(fragment) {
-      var target = d3.select(content).node();
-      target.parentNode.replaceChild(d3.select(fragment).select(content).node(), target);
-      nv.utils.pjax(links, content);
-    });
-  }
-
-  d3.select(window).on("popstate", function() {
-    if (d3.event.state) load(d3.event.state);
-  });
-}
-
-/* For situations where we want to approximate the width in pixels for an SVG:text element.
-Most common instance is when the element is in a display:none; container. 
-Forumla is : text.length * font-size * constant_factor
-*/
-nv.utils.calcApproxTextWidth = function (svgTextElem) {
-    if (svgTextElem instanceof d3.selection) {
-        var fontSize = parseInt(svgTextElem.style("font-size").replace("px",""));
-        var textLength = svgTextElem.text().length;
-
-        return textLength * fontSize * 0.5; 
-    }
-    return 0;
-};
-
-/* Numbers that are undefined, null or NaN, convert them to zeros.
-*/
-nv.utils.NaNtoZero = function(n) {
-    if (typeof n !== 'number' 
-        || isNaN(n) 
-        || n === null
-        || n === Infinity) return 0;
-
-    return n;
-};
-
-/*
-Snippet of code you can insert into each nv.models.* to give you the ability to
-do things like:
-chart.options({
-  showXAxis: true,
-  tooltips: true
-});
-
-To enable in the chart:
-chart.options = nv.utils.optionsFunc.bind(chart);
-*/
-nv.utils.optionsFunc = function(args) {
-    if (args) {
-      d3.map(args).forEach((function(key,value) {
-        if (typeof this[key] === "function") {
-           this[key](value);
-        }
-      }).bind(this));
-    }
-    return this;
-};nv.models.axis = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var axis = d3.svg.axis()
-    ;
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 75 //only used for tickLabel currently
-    , height = 60 //only used for tickLabel currently
-    , scale = d3.scale.linear()
-    , axisLabelText = null
-    , showMaxMin = true //TODO: showMaxMin should be disabled on all ordinal scaled axes
-    , highlightZero = true
-    , rotateLabels = 0
-    , rotateYLabel = true
-    , staggerLabels = false
-    , isOrdinal = false
-    , ticks = null
-    , axisLabelDistance = 12 //The larger this number is, the closer the axis label is to the axis.
-    ;
-
-  axis
-    .scale(scale)
-    .orient('bottom')
-    .tickFormat(function(d) { return d })
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var scale0;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-axis').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-axis');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g')
-
-      //------------------------------------------------------------
-
-
-      if (ticks !== null)
-        axis.ticks(ticks);
-      else if (axis.orient() == 'top' || axis.orient() == 'bottom')
-        axis.ticks(Math.abs(scale.range()[1] - scale.range()[0]) / 100);
-
-
-      //TODO: consider calculating width/height based on whether or not label is added, for reference in charts using this component
-
-
-      g.transition().call(axis);
-
-      scale0 = scale0 || axis.scale();
-
-      var fmt = axis.tickFormat();
-      if (fmt == null) {
-        fmt = scale0.tickFormat();
-      }
-
-      var axisLabel = g.selectAll('text.nv-axislabel')
-          .data([axisLabelText || null]);
-      axisLabel.exit().remove();
-      switch (axis.orient()) {
-        case 'top':
-          axisLabel.enter().append('text').attr('class', 'nv-axislabel');
-          var w = (scale.range().length==2) ? scale.range()[1] : (scale.range()[scale.range().length-1]+(scale.range()[1]-scale.range()[0]));
-          axisLabel
-              .attr('text-anchor', 'middle')
-              .attr('y', 0)
-              .attr('x', w/2);
-          if (showMaxMin) {
-            var axisMaxMin = wrap.selectAll('g.nv-axisMaxMin')
-                           .data(scale.domain());
-            axisMaxMin.enter().append('g').attr('class', 'nv-axisMaxMin').append('text');
-            axisMaxMin.exit().remove();
-            axisMaxMin
-                .attr('transform', function(d,i) {
-                  return 'translate(' + scale(d) + ',0)'
-                })
-              .select('text')
-                .attr('dy', '0em')
-                .attr('y', -axis.tickPadding())
-                .attr('text-anchor', 'middle')
-                .text(function(d,i) {
-                  var v = fmt(d);
-                  return ('' + v).match('NaN') ? '' : v;
-                });
-            axisMaxMin.transition()
-                .attr('transform', function(d,i) {
-                  return 'translate(' + scale.range()[i] + ',0)'
-                });
-          }
-          break;
-        case 'bottom':
-          var xLabelMargin = 36;
-          var maxTextWidth = 30;
-          var xTicks = g.selectAll('g').select("text");
-          if (rotateLabels%360) {
-            //Calculate the longest xTick width
-            xTicks.each(function(d,i){
-              var width = this.getBBox().width;
-              if(width > maxTextWidth) maxTextWidth = width;
-            });
-            //Convert to radians before calculating sin. Add 30 to margin for healthy padding.
-            var sin = Math.abs(Math.sin(rotateLabels*Math.PI/180));
-            var xLabelMargin = (sin ? sin*maxTextWidth : maxTextWidth)+30;
-            //Rotate all xTicks
-            xTicks
-              .attr('transform', function(d,i,j) { return 'rotate(' + rotateLabels + ' 0,0)' })
-              .style('text-anchor', rotateLabels%360 > 0 ? 'start' : 'end');
-          }
-          axisLabel.enter().append('text').attr('class', 'nv-axislabel');
-          var w = (scale.range().length==2) ? scale.range()[1] : (scale.range()[scale.range().length-1]+(scale.range()[1]-scale.range()[0]));
-          axisLabel
-              .attr('text-anchor', 'middle')
-              .attr('y', xLabelMargin)
-              .attr('x', w/2);
-          if (showMaxMin) {
-          //if (showMaxMin && !isOrdinal) {
-            var axisMaxMin = wrap.selectAll('g.nv-axisMaxMin')
-                           //.data(scale.domain())
-                           .data([scale.domain()[0], scale.domain()[scale.domain().length - 1]]);
-            axisMaxMin.enter().append('g').attr('class', 'nv-axisMaxMin').append('text');
-            axisMaxMin.exit().remove();
-            axisMaxMin
-                .attr('transform', function(d,i) {
-                  return 'translate(' + (scale(d) + (isOrdinal ? scale.rangeBand() / 2 : 0)) + ',0)'
-                })
-              .select('text')
-                .attr('dy', '.71em')
-                .attr('y', axis.tickPadding())
-                .attr('transform', function(d,i,j) { return 'rotate(' + rotateLabels + ' 0,0)' })
-                .style('text-anchor', rotateLabels ? (rotateLabels%360 > 0 ? 'start' : 'end') : 'middle')
-                .text(function(d,i) {
-                  var v = fmt(d);
-                  return ('' + v).match('NaN') ? '' : v;
-                });
-            axisMaxMin.transition()
-                .attr('transform', function(d,i) {
-                  //return 'translate(' + scale.range()[i] + ',0)'
-                  //return 'translate(' + scale(d) + ',0)'
-                  return 'translate(' + (scale(d) + (isOrdinal ? scale.rangeBand() / 2 : 0)) + ',0)'
-                });
-          }
-          if (staggerLabels)
-            xTicks
-                .attr('transform', function(d,i) { return 'translate(0,' + (i % 2 == 0 ? '0' : '12') + ')' });
-
-          break;
-        case 'right':
-          axisLabel.enter().append('text').attr('class', 'nv-axislabel');
-          axisLabel
-              .style('text-anchor', rotateYLabel ? 'middle' : 'begin')
-              .attr('transform', rotateYLabel ? 'rotate(90)' : '')
-              .attr('y', rotateYLabel ? (-Math.max(margin.right,width) + 12) : -10) //TODO: consider calculating this based on largest tick width... OR at least expose this on chart
-              .attr('x', rotateYLabel ? (scale.range()[0] / 2) : axis.tickPadding());
-          if (showMaxMin) {
-            var axisMaxMin = wrap.selectAll('g.nv-axisMaxMin')
-                           .data(scale.domain());
-            axisMaxMin.enter().append('g').attr('class', 'nv-axisMaxMin').append('text')
-                .style('opacity', 0);
-            axisMaxMin.exit().remove();
-            axisMaxMin
-                .attr('transform', function(d,i) {
-                  return 'translate(0,' + scale(d) + ')'
-                })
-              .select('text')
-                .attr('dy', '.32em')
-                .attr('y', 0)
-                .attr('x', axis.tickPadding())
-                .style('text-anchor', 'start')
-                .text(function(d,i) {
-                  var v = fmt(d);
-                  return ('' + v).match('NaN') ? '' : v;
-                });
-            axisMaxMin.transition()
-                .attr('transform', function(d,i) {
-                  return 'translate(0,' + scale.range()[i] + ')'
-                })
-              .select('text')
-                .style('opacity', 1);
-          }
-          break;
-        case 'left':
-          /*
-          //For dynamically placing the label. Can be used with dynamically-sized chart axis margins
-          var yTicks = g.selectAll('g').select("text");
-          yTicks.each(function(d,i){
-            var labelPadding = this.getBBox().width + axis.tickPadding() + 16;
-            if(labelPadding > width) width = labelPadding;
-          });
-          */
-          axisLabel.enter().append('text').attr('class', 'nv-axislabel');
-          axisLabel
-              .style('text-anchor', rotateYLabel ? 'middle' : 'end')
-              .attr('transform', rotateYLabel ? 'rotate(-90)' : '')
-              .attr('y', rotateYLabel ? (-Math.max(margin.left,width) + axisLabelDistance) : -10) //TODO: consider calculating this based on largest tick width... OR at least expose this on chart
-              .attr('x', rotateYLabel ? (-scale.range()[0] / 2) : -axis.tickPadding());
-          if (showMaxMin) {
-            var axisMaxMin = wrap.selectAll('g.nv-axisMaxMin')
-                           .data(scale.domain());
-            axisMaxMin.enter().append('g').attr('class', 'nv-axisMaxMin').append('text')
-                .style('opacity', 0);
-            axisMaxMin.exit().remove();
-            axisMaxMin
-                .attr('transform', function(d,i) {
-                  return 'translate(0,' + scale0(d) + ')'
-                })
-              .select('text')
-                .attr('dy', '.32em')
-                .attr('y', 0)
-                .attr('x', -axis.tickPadding())
-                .attr('text-anchor', 'end')
-                .text(function(d,i) {
-                  var v = fmt(d);
-                  return ('' + v).match('NaN') ? '' : v;
-                });
-            axisMaxMin.transition()
-                .attr('transform', function(d,i) {
-                  return 'translate(0,' + scale.range()[i] + ')'
-                })
-              .select('text')
-                .style('opacity', 1);
-          }
-          break;
-      }
-      axisLabel
-          .text(function(d) { return d });
-
-
-      if (showMaxMin && (axis.orient() === 'left' || axis.orient() === 'right')) {
-        //check if max and min overlap other values, if so, hide the values that overlap
-        g.selectAll('g') // the g's wrapping each tick
-            .each(function(d,i) {
-              d3.select(this).select('text').attr('opacity', 1);
-              if (scale(d) < scale.range()[1] + 10 || scale(d) > scale.range()[0] - 10) { // 10 is assuming text height is 16... if d is 0, leave it!
-                if (d > 1e-10 || d < -1e-10) // accounts for minor floating point errors... though could be problematic if the scale is EXTREMELY SMALL
-                  d3.select(this).attr('opacity', 0);
-                
-                d3.select(this).select('text').attr('opacity', 0); // Don't remove the ZERO line!!
-              }
-            });
-
-        //if Max and Min = 0 only show min, Issue #281
-        if (scale.domain()[0] == scale.domain()[1] && scale.domain()[0] == 0)
-          wrap.selectAll('g.nv-axisMaxMin')
-            .style('opacity', function(d,i) { return !i ? 1 : 0 });
-
-      }
-
-      if (showMaxMin && (axis.orient() === 'top' || axis.orient() === 'bottom')) {
-        var maxMinRange = [];
-        wrap.selectAll('g.nv-axisMaxMin')
-            .each(function(d,i) {
-              try {
-                  if (i) // i== 1, max position
-                      maxMinRange.push(scale(d) - this.getBBox().width - 4)  //assuming the max and min labels are as wide as the next tick (with an extra 4 pixels just in case)
-                  else // i==0, min position
-                      maxMinRange.push(scale(d) + this.getBBox().width + 4)
-              }catch (err) {
-                  if (i) // i== 1, max position
-                      maxMinRange.push(scale(d) - 4)  //assuming the max and min labels are as wide as the next tick (with an extra 4 pixels just in case)
-                  else // i==0, min position
-                      maxMinRange.push(scale(d) + 4)
-              }
-            });
-        g.selectAll('g') // the g's wrapping each tick
-            .each(function(d,i) {
-              if (scale(d) < maxMinRange[0] || scale(d) > maxMinRange[1]) {
-                if (d > 1e-10 || d < -1e-10) // accounts for minor floating point errors... though could be problematic if the scale is EXTREMELY SMALL
-                  d3.select(this).remove();
-                else
-                  d3.select(this).select('text').remove(); // Don't remove the ZERO line!!
-              }
-            });
-      }
-
-
-      //highlight zero line ... Maybe should not be an option and should just be in CSS?
-      if (highlightZero)
-        g.selectAll('.tick')
-          .filter(function(d) { return !parseFloat(Math.round(d.__data__*100000)/1000000) && (d.__data__ !== undefined) }) //this is because sometimes the 0 tick is a very small fraction, TODO: think of cleaner technique
-            .classed('zero', true);
-
-      //store old scales for use in transitions on update
-      scale0 = scale.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.axis = axis;
-
-  d3.rebind(chart, axis, 'orient', 'tickValues', 'tickSubdivide', 'tickSize', 'tickPadding', 'tickFormat');
-  d3.rebind(chart, scale, 'domain', 'range', 'rangeBand', 'rangeBands'); //these are also accessible by chart.scale(), but added common ones directly for ease of use
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if(!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  }
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.ticks = function(_) {
-    if (!arguments.length) return ticks;
-    ticks = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.axisLabel = function(_) {
-    if (!arguments.length) return axisLabelText;
-    axisLabelText = _;
-    return chart;
-  }
-
-  chart.showMaxMin = function(_) {
-    if (!arguments.length) return showMaxMin;
-    showMaxMin = _;
-    return chart;
-  }
-
-  chart.highlightZero = function(_) {
-    if (!arguments.length) return highlightZero;
-    highlightZero = _;
-    return chart;
-  }
-
-  chart.scale = function(_) {
-    if (!arguments.length) return scale;
-    scale = _;
-    axis.scale(scale);
-    isOrdinal = typeof scale.rangeBands === 'function';
-    d3.rebind(chart, scale, 'domain', 'range', 'rangeBand', 'rangeBands');
-    return chart;
-  }
-
-  chart.rotateYLabel = function(_) {
-    if(!arguments.length) return rotateYLabel;
-    rotateYLabel = _;
-    return chart;
-  }
-
-  chart.rotateLabels = function(_) {
-    if(!arguments.length) return rotateLabels;
-    rotateLabels = _;
-    return chart;
-  }
-
-  chart.staggerLabels = function(_) {
-    if (!arguments.length) return staggerLabels;
-    staggerLabels = _;
-    return chart;
-  };
-
-  chart.axisLabelDistance = function(_) {
-    if (!arguments.length) return axisLabelDistance;
-    axisLabelDistance = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-// Chart design based on the recommendations of Stephen Few. Implementation
-// based on the work of Clint Ivy, Jamie Love, and Jason Davies.
-// http://projects.instantcognition.com/protovis/bulletchart/
-
-nv.models.bullet = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , orient = 'left' // TODO top & bottom
-    , reverse = false
-    , ranges = function(d) { return d.ranges }
-    , markers = function(d) { return d.markers }
-    , measures = function(d) { return d.measures }
-    , rangeLabels = function(d) { return d.rangeLabels ? d.rangeLabels : [] }
-    , markerLabels = function(d) { return d.markerLabels ? d.markerLabels : []  }
-    , measureLabels = function(d) { return d.measureLabels ? d.measureLabels : []  }
-    , forceX = [0] // List of numbers to Force into the X scale (ie. 0, or a max / min, etc.)
-    , width = 380
-    , height = 30
-    , tickFormat = null
-    , color = nv.utils.getColor(['#1f77b4'])
-    , dispatch = d3.dispatch('elementMouseover', 'elementMouseout')
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(d, i) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-      var rangez = ranges.call(this, d, i).slice().sort(d3.descending),
-          markerz = markers.call(this, d, i).slice().sort(d3.descending),
-          measurez = measures.call(this, d, i).slice().sort(d3.descending),
-          rangeLabelz = rangeLabels.call(this, d, i).slice(),
-          markerLabelz = markerLabels.call(this, d, i).slice(),
-          measureLabelz = measureLabels.call(this, d, i).slice();
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      // Compute the new x-scale.
-      var x1 = d3.scale.linear()
-          .domain( d3.extent(d3.merge([forceX, rangez])) )
-          .range(reverse ? [availableWidth, 0] : [0, availableWidth]);
-
-      // Retrieve the old x-scale, if this is an update.
-      var x0 = this.__chart__ || d3.scale.linear()
-          .domain([0, Infinity])
-          .range(x1.range());
-
-      // Stash the new scale.
-      this.__chart__ = x1;
-
-
-      var rangeMin = d3.min(rangez), //rangez[2]
-          rangeMax = d3.max(rangez), //rangez[0]
-          rangeAvg = rangez[1];
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-bullet').data([d]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-bullet');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('rect').attr('class', 'nv-range nv-rangeMax');
-      gEnter.append('rect').attr('class', 'nv-range nv-rangeAvg');
-      gEnter.append('rect').attr('class', 'nv-range nv-rangeMin');
-      gEnter.append('rect').attr('class', 'nv-measure');
-      gEnter.append('path').attr('class', 'nv-markerTriangle');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-
-      var w0 = function(d) { return Math.abs(x0(d) - x0(0)) }, // TODO: could optimize by precalculating x0(0) and x1(0)
-          w1 = function(d) { return Math.abs(x1(d) - x1(0)) };
-      var xp0 = function(d) { return d < 0 ? x0(d) : x0(0) },
-          xp1 = function(d) { return d < 0 ? x1(d) : x1(0) };
-
-
-      g.select('rect.nv-rangeMax')
-          .attr('height', availableHeight)
-          .attr('width', w1(rangeMax > 0 ? rangeMax : rangeMin))
-          .attr('x', xp1(rangeMax > 0 ? rangeMax : rangeMin))
-          .datum(rangeMax > 0 ? rangeMax : rangeMin)
-          /*
-          .attr('x', rangeMin < 0 ?
-                         rangeMax > 0 ?
-                             x1(rangeMin)
-                           : x1(rangeMax)
-                       : x1(0))
-                      */
-
-      g.select('rect.nv-rangeAvg')
-          .attr('height', availableHeight)
-          .attr('width', w1(rangeAvg))
-          .attr('x', xp1(rangeAvg))
-          .datum(rangeAvg)
-          /*
-          .attr('width', rangeMax <= 0 ?
-                             x1(rangeMax) - x1(rangeAvg)
-                           : x1(rangeAvg) - x1(rangeMin))
-          .attr('x', rangeMax <= 0 ?
-                         x1(rangeAvg)
-                       : x1(rangeMin))
-                      */
-
-      g.select('rect.nv-rangeMin')
-          .attr('height', availableHeight)
-          .attr('width', w1(rangeMax))
-          .attr('x', xp1(rangeMax))
-          .attr('width', w1(rangeMax > 0 ? rangeMin : rangeMax))
-          .attr('x', xp1(rangeMax > 0 ? rangeMin : rangeMax))
-          .datum(rangeMax > 0 ? rangeMin : rangeMax)
-          /*
-          .attr('width', rangeMax <= 0 ?
-                             x1(rangeAvg) - x1(rangeMin)
-                           : x1(rangeMax) - x1(rangeAvg))
-          .attr('x', rangeMax <= 0 ?
-                         x1(rangeMin)
-                       : x1(rangeAvg))
-                      */
-
-      g.select('rect.nv-measure')
-          .style('fill', color)
-          .attr('height', availableHeight / 3)
-          .attr('y', availableHeight / 3)
-          .attr('width', measurez < 0 ?
-                             x1(0) - x1(measurez[0])
-                           : x1(measurez[0]) - x1(0))
-          .attr('x', xp1(measurez))
-          .on('mouseover', function() {
-              dispatch.elementMouseover({
-                value: measurez[0],
-                label: measureLabelz[0] || 'Current',
-                pos: [x1(measurez[0]), availableHeight/2]
-              })
-          })
-          .on('mouseout', function() {
-              dispatch.elementMouseout({
-                value: measurez[0],
-                label: measureLabelz[0] || 'Current'
-              })
-          })
-
-      var h3 =  availableHeight / 6;
-      if (markerz[0]) {
-        g.selectAll('path.nv-markerTriangle')
-            .attr('transform', function(d) { return 'translate(' + x1(markerz[0]) + ',' + (availableHeight / 2) + ')' })
-            .attr('d', 'M0,' + h3 + 'L' + h3 + ',' + (-h3) + ' ' + (-h3) + ',' + (-h3) + 'Z')
-            .on('mouseover', function() {
-              dispatch.elementMouseover({
-                value: markerz[0],
-                label: markerLabelz[0] || 'Previous',
-                pos: [x1(markerz[0]), availableHeight/2]
-              })
-            })
-            .on('mouseout', function() {
-              dispatch.elementMouseout({
-                value: markerz[0],
-                label: markerLabelz[0] || 'Previous'
-              })
-            });
-      } else {
-        g.selectAll('path.nv-markerTriangle').remove();
-      }
-
-
-      wrap.selectAll('.nv-range')
-          .on('mouseover', function(d,i) {
-            var label = rangeLabelz[i] || (!i ? "Maximum" : i == 1 ? "Mean" : "Minimum");
-
-            dispatch.elementMouseover({
-              value: d,
-              label: label,
-              pos: [x1(d), availableHeight/2]
-            })
-          })
-          .on('mouseout', function(d,i) {
-            var label = rangeLabelz[i] || (!i ? "Maximum" : i == 1 ? "Mean" : "Minimum");
-
-            dispatch.elementMouseout({
-              value: d,
-              label: label
-            })
-          })
-
-/* // THIS IS THE PREVIOUS BULLET IMPLEMENTATION, WILL REMOVE SHORTLY
-      // Update the range rects.
-      var range = g.selectAll('rect.nv-range')
-          .data(rangez);
-
-      range.enter().append('rect')
-          .attr('class', function(d, i) { return 'nv-range nv-s' + i; })
-          .attr('width', w0)
-          .attr('height', availableHeight)
-          .attr('x', reverse ? x0 : 0)
-          .on('mouseover', function(d,i) { 
-              dispatch.elementMouseover({
-                value: d,
-                label: (i <= 0) ? 'Maximum' : (i > 1) ? 'Minimum' : 'Mean', //TODO: make these labels a variable
-                pos: [x1(d), availableHeight/2]
-              })
-          })
-          .on('mouseout', function(d,i) { 
-              dispatch.elementMouseout({
-                value: d,
-                label: (i <= 0) ? 'Minimum' : (i >=1) ? 'Maximum' : 'Mean' //TODO: make these labels a variable
-              })
-          })
-
-      d3.transition(range)
-          .attr('x', reverse ? x1 : 0)
-          .attr('width', w1)
-          .attr('height', availableHeight);
-
-
-      // Update the measure rects.
-      var measure = g.selectAll('rect.nv-measure')
-          .data(measurez);
-
-      measure.enter().append('rect')
-          .attr('class', function(d, i) { return 'nv-measure nv-s' + i; })
-          .style('fill', function(d,i) { return color(d,i ) })
-          .attr('width', w0)
-          .attr('height', availableHeight / 3)
-          .attr('x', reverse ? x0 : 0)
-          .attr('y', availableHeight / 3)
-          .on('mouseover', function(d) { 
-              dispatch.elementMouseover({
-                value: d,
-                label: 'Current', //TODO: make these labels a variable
-                pos: [x1(d), availableHeight/2]
-              })
-          })
-          .on('mouseout', function(d) { 
-              dispatch.elementMouseout({
-                value: d,
-                label: 'Current' //TODO: make these labels a variable
-              })
-          })
-
-      d3.transition(measure)
-          .attr('width', w1)
-          .attr('height', availableHeight / 3)
-          .attr('x', reverse ? x1 : 0)
-          .attr('y', availableHeight / 3);
-
-
-
-      // Update the marker lines.
-      var marker = g.selectAll('path.nv-markerTriangle')
-          .data(markerz);
-
-      var h3 =  availableHeight / 6;
-      marker.enter().append('path')
-          .attr('class', 'nv-markerTriangle')
-          .attr('transform', function(d) { return 'translate(' + x0(d) + ',' + (availableHeight / 2) + ')' })
-          .attr('d', 'M0,' + h3 + 'L' + h3 + ',' + (-h3) + ' ' + (-h3) + ',' + (-h3) + 'Z')
-          .on('mouseover', function(d,i) {
-              dispatch.elementMouseover({
-                value: d,
-                label: 'Previous',
-                pos: [x1(d), availableHeight/2]
-              })
-          })
-          .on('mouseout', function(d,i) {
-              dispatch.elementMouseout({
-                value: d,
-                label: 'Previous'
-              })
-          });
-
-      d3.transition(marker)
-          .attr('transform', function(d) { return 'translate(' + (x1(d) - x1(0)) + ',' + (availableHeight / 2) + ')' });
-
-      marker.exit().remove();
-*/
-
-    });
-
-    // d3.timer.flush();  // Not needed?
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  // left, right, top, bottom
-  chart.orient = function(_) {
-    if (!arguments.length) return orient;
-    orient = _;
-    reverse = orient == 'right' || orient == 'bottom';
-    return chart;
-  };
-
-  // ranges (bad, satisfactory, good)
-  chart.ranges = function(_) {
-    if (!arguments.length) return ranges;
-    ranges = _;
-    return chart;
-  };
-
-  // markers (previous, goal)
-  chart.markers = function(_) {
-    if (!arguments.length) return markers;
-    markers = _;
-    return chart;
-  };
-
-  // measures (actual, forecast)
-  chart.measures = function(_) {
-    if (!arguments.length) return measures;
-    measures = _;
-    return chart;
-  };
-
-  chart.forceX = function(_) {
-    if (!arguments.length) return forceX;
-    forceX = _;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.tickFormat = function(_) {
-    if (!arguments.length) return tickFormat;
-    tickFormat = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-};
-
-
-
-// Chart design based on the recommendations of Stephen Few. Implementation
-// based on the work of Clint Ivy, Jamie Love, and Jason Davies.
-// http://projects.instantcognition.com/protovis/bulletchart/
-nv.models.bulletChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var bullet = nv.models.bullet()
-    ;
-
-  var orient = 'left' // TODO top & bottom
-    , reverse = false
-    , margin = {top: 5, right: 40, bottom: 20, left: 120}
-    , ranges = function(d) { return d.ranges }
-    , markers = function(d) { return d.markers }
-    , measures = function(d) { return d.measures }
-    , width = null
-    , height = 55
-    , tickFormat = null
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + x + '</h3>' +
-               '<p>' + y + '</p>'
-      }
-    , noData = 'No Data Available.'
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ) + margin.left,
-        top = e.pos[1] + ( offsetElement.offsetTop || 0) + margin.top,
-        content = tooltip(e.key, e.label, e.value, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'e' : 'w', null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(d, i) {
-      var container = d3.select(this);
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          that = this;
-
-
-      chart.update = function() { chart(selection) };
-      chart.container = this;
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!d || !ranges.call(this, d, i)) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', 18 + margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-
-      var rangez = ranges.call(this, d, i).slice().sort(d3.descending),
-          markerz = markers.call(this, d, i).slice().sort(d3.descending),
-          measurez = measures.call(this, d, i).slice().sort(d3.descending);
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-bulletChart').data([d]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-bulletChart');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-bulletWrap');
-      gEnter.append('g').attr('class', 'nv-titles');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      // Compute the new x-scale.
-      var x1 = d3.scale.linear()
-          .domain([0, Math.max(rangez[0], markerz[0], measurez[0])])  // TODO: need to allow forceX and forceY, and xDomain, yDomain
-          .range(reverse ? [availableWidth, 0] : [0, availableWidth]);
-
-      // Retrieve the old x-scale, if this is an update.
-      var x0 = this.__chart__ || d3.scale.linear()
-          .domain([0, Infinity])
-          .range(x1.range());
-
-      // Stash the new scale.
-      this.__chart__ = x1;
-
-      /*
-      // Derive width-scales from the x-scales.
-      var w0 = bulletWidth(x0),
-          w1 = bulletWidth(x1);
-
-      function bulletWidth(x) {
-        var x0 = x(0);
-        return function(d) {
-          return Math.abs(x(d) - x(0));
-        };
-      }
-
-      function bulletTranslate(x) {
-        return function(d) {
-          return 'translate(' + x(d) + ',0)';
-        };
-      }
-      */
-
-      var w0 = function(d) { return Math.abs(x0(d) - x0(0)) }, // TODO: could optimize by precalculating x0(0) and x1(0)
-          w1 = function(d) { return Math.abs(x1(d) - x1(0)) };
-
-
-      var title = gEnter.select('.nv-titles').append('g')
-          .attr('text-anchor', 'end')
-          .attr('transform', 'translate(-6,' + (height - margin.top - margin.bottom) / 2 + ')');
-      title.append('text')
-          .attr('class', 'nv-title')
-          .text(function(d) { return d.title; });
-
-      title.append('text')
-          .attr('class', 'nv-subtitle')
-          .attr('dy', '1em')
-          .text(function(d) { return d.subtitle; });
-
-
-
-      bullet
-        .width(availableWidth)
-        .height(availableHeight)
-
-      var bulletWrap = g.select('.nv-bulletWrap');
-
-      d3.transition(bulletWrap).call(bullet);
-
-
-
-      // Compute the tick format.
-      var format = tickFormat || x1.tickFormat( availableWidth / 100 );
-
-      // Update the tick groups.
-      var tick = g.selectAll('g.nv-tick')
-          .data(x1.ticks( availableWidth / 50 ), function(d) {
-            return this.textContent || format(d);
-          });
-
-      // Initialize the ticks with the old scale, x0.
-      var tickEnter = tick.enter().append('g')
-          .attr('class', 'nv-tick')
-          .attr('transform', function(d) { return 'translate(' + x0(d) + ',0)' })
-          .style('opacity', 1e-6);
-
-      tickEnter.append('line')
-          .attr('y1', availableHeight)
-          .attr('y2', availableHeight * 7 / 6);
-
-      tickEnter.append('text')
-          .attr('text-anchor', 'middle')
-          .attr('dy', '1em')
-          .attr('y', availableHeight * 7 / 6)
-          .text(format);
-
-
-      // Transition the updating ticks to the new scale, x1.
-      var tickUpdate = d3.transition(tick)
-          .attr('transform', function(d) { return 'translate(' + x1(d) + ',0)' })
-          .style('opacity', 1);
-
-      tickUpdate.select('line')
-          .attr('y1', availableHeight)
-          .attr('y2', availableHeight * 7 / 6);
-
-      tickUpdate.select('text')
-          .attr('y', availableHeight * 7 / 6);
-
-      // Transition the exiting ticks to the new scale, x1.
-      d3.transition(tick.exit())
-          .attr('transform', function(d) { return 'translate(' + x1(d) + ',0)' })
-          .style('opacity', 1e-6)
-          .remove();
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      dispatch.on('tooltipShow', function(e) {
-        e.key = d.title;
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      //============================================================
-
-    });
-
-    d3.timer.flush();
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  bullet.dispatch.on('elementMouseover.tooltip', function(e) {
-    dispatch.tooltipShow(e);
-  });
-
-  bullet.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-  chart.bullet = bullet;
-
-  d3.rebind(chart, bullet, 'color');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  // left, right, top, bottom
-  chart.orient = function(x) {
-    if (!arguments.length) return orient;
-    orient = x;
-    reverse = orient == 'right' || orient == 'bottom';
-    return chart;
-  };
-
-  // ranges (bad, satisfactory, good)
-  chart.ranges = function(x) {
-    if (!arguments.length) return ranges;
-    ranges = x;
-    return chart;
-  };
-
-  // markers (previous, goal)
-  chart.markers = function(x) {
-    if (!arguments.length) return markers;
-    markers = x;
-    return chart;
-  };
-
-  // measures (actual, forecast)
-  chart.measures = function(x) {
-    if (!arguments.length) return measures;
-    measures = x;
-    return chart;
-  };
-
-  chart.width = function(x) {
-    if (!arguments.length) return width;
-    width = x;
-    return chart;
-  };
-
-  chart.height = function(x) {
-    if (!arguments.length) return height;
-    height = x;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.tickFormat = function(x) {
-    if (!arguments.length) return tickFormat;
-    tickFormat = x;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-};
-
-
-
-nv.models.cumulativeLineChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var lines = nv.models.line()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , legend = nv.models.legend()
-    , controls = nv.models.legend()
-    , interactiveLayer = nv.interactiveGuideline()
-    ;
-
-  var margin = {top: 30, right: 30, bottom: 50, left: 60}
-    , color = nv.utils.defaultColor()
-    , width = null
-    , height = null
-    , showLegend = true
-    , showXAxis = true
-    , showYAxis = true
-    , rightAlignYAxis = false
-    , tooltips = true
-    , showControls = true
-    , useInteractiveGuideline = false
-    , rescaleY = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>'
-      }
-    , x //can be accessed via chart.xScale()
-    , y //can be accessed via chart.yScale()
-    , id = lines.id()
-    , state = { index: 0, rescaleY: rescaleY }
-    , defaultState = null
-    , noData = 'No Data Available.'
-    , average = function(d) { return d.average }
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , transitionDuration = 250
-    ;
-
-  xAxis
-    .orient('bottom')
-    .tickPadding(7)
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    ;
-
-  //============================================================
-  controls.updateState(false);
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-   var dx = d3.scale.linear()
-     , index = {i: 0, x: 0}
-     ;
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(lines.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(lines.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, null, null, offsetElement);
-  };
-
-/*
-  //Moved to see if we can get better behavior to fix issue #315
-  var indexDrag = d3.behavior.drag()
-                    .on('dragstart', dragStart)
-                    .on('drag', dragMove)
-                    .on('dragend', dragEnd);
-
-  function dragStart(d,i) {
-    d3.select(chart.container)
-        .style('cursor', 'ew-resize');
-  }
-
-  function dragMove(d,i) {
-    d.x += d3.event.dx;
-    d.i = Math.round(dx.invert(d.x));
-
-    d3.select(this).attr('transform', 'translate(' + dx(d.i) + ',0)');
-    chart.update();
-  }
-
-  function dragEnd(d,i) {
-    d3.select(chart.container)
-        .style('cursor', 'auto');
-    chart.update();
-  }
-*/
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this).classed('nv-chart-' + id, true),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart) };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      var indexDrag = d3.behavior.drag()
-                        .on('dragstart', dragStart)
-                        .on('drag', dragMove)
-                        .on('dragend', dragEnd);
-
-
-      function dragStart(d,i) {
-        d3.select(chart.container)
-            .style('cursor', 'ew-resize');
-      }
-
-      function dragMove(d,i) {
-        index.x = d3.event.x;
-        index.i = Math.round(dx.invert(index.x));
-        updateZero();
-      }
-
-      function dragEnd(d,i) {
-        d3.select(chart.container)
-            .style('cursor', 'auto');
-
-        // update state and send stateChange with new index
-        state.index = index.i;
-        dispatch.stateChange(state);
-      }
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = lines.xScale();
-      y = lines.yScale();
-
-
-      if (!rescaleY) {
-        var seriesDomains = data
-          .filter(function(series) { return !series.disabled })
-          .map(function(series,i) {
-            var initialDomain = d3.extent(series.values, lines.y());
-
-            //account for series being disabled when losing 95% or more
-            if (initialDomain[0] < -.95) initialDomain[0] = -.95;
-
-            return [
-              (initialDomain[0] - initialDomain[1]) / (1 + initialDomain[1]),
-              (initialDomain[1] - initialDomain[0]) / (1 + initialDomain[0])
-            ];
-          });
-
-        var completeDomain = [
-          d3.min(seriesDomains, function(d) { return d[0] }),
-          d3.max(seriesDomains, function(d) { return d[1] })
-        ]
-
-        lines.yDomain(completeDomain);
-      } else {
-        lines.yDomain(null);
-      }
-
-
-      dx  .domain([0, data[0].values.length - 1]) //Assumes all series have same length
-          .range([0, availableWidth])
-          .clamp(true);
-
-      //------------------------------------------------------------
-
-
-      var data = indexify(index.i, data);
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-      var interactivePointerEvents = (useInteractiveGuideline) ? "none" : "all";
-      var wrap = container.selectAll('g.nv-wrap.nv-cumulativeLine').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-cumulativeLine').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-interactive');
-      gEnter.append('g').attr('class', 'nv-x nv-axis').style("pointer-events","none");
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-background');
-      gEnter.append('g').attr('class', 'nv-linesWrap').style("pointer-events",interactivePointerEvents);
-      gEnter.append('g').attr('class', 'nv-avgLinesWrap').style("pointer-events","none");
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-controlsWrap');
-      
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width(availableWidth);
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Controls
-
-      if (showControls) {
-        var controlsData = [
-          { key: 'Re-scale y-axis', disabled: !rescaleY }
-        ];
-
-        controls.width(140).color(['#444', '#444', '#444']);
-        g.select('.nv-controlsWrap')
-            .datum(controlsData)
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-            .call(controls);
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      // Show error if series goes below 100%
-      var tempDisabled = data.filter(function(d) { return d.tempDisabled });
-
-      wrap.select('.tempDisabled').remove(); //clean-up and prevent duplicates
-      if (tempDisabled.length) {
-        wrap.append('text').attr('class', 'tempDisabled')
-            .attr('x', availableWidth / 2)
-            .attr('y', '-.71em')
-            .style('text-anchor', 'end')
-            .text(tempDisabled.map(function(d) { return d.key }).join(', ') + ' values cannot be calculated for this time period.');
-      }
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-      
-      //------------------------------------------------------------
-      //Set up interactive layer
-      if (useInteractiveGuideline) {
-        interactiveLayer
-          .width(availableWidth)
-          .height(availableHeight)
-          .margin({left:margin.left,top:margin.top})
-          .svgContainer(container)
-          .xScale(x);
-        wrap.select(".nv-interactive").call(interactiveLayer);
-      }
-
-      gEnter.select('.nv-background')
-        .append('rect');
-
-      g.select('.nv-background rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      lines
-        //.x(function(d) { return d.x })
-        .y(function(d) { return d.display.y })
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && !data[i].tempDisabled; }));
-
-
-
-      var linesWrap = g.select('.nv-linesWrap')
-          .datum(data.filter(function(d) { return  !d.disabled && !d.tempDisabled }));
-
-      //d3.transition(linesWrap).call(lines);
-      linesWrap.call(lines);
-
-      /*Handle average lines [AN-612] ----------------------------*/
-
-      //Store a series index number in the data array.
-      data.forEach(function(d,i) {
-            d.seriesIndex = i;
-      });
-
-      var avgLineData = data.filter(function(d) {
-          return !d.disabled && !!average(d);
-      });
-
-      var avgLines = g.select(".nv-avgLinesWrap").selectAll("line")
-              .data(avgLineData, function(d) { return d.key; });
-
-      var getAvgLineY = function(d) {
-          //If average lines go off the svg element, clamp them to the svg bounds.
-          var yVal = y(average(d));
-          if (yVal < 0) return 0;
-          if (yVal > availableHeight) return availableHeight;
-          return yVal;
-      };
-
-      avgLines.enter()
-              .append('line')
-              .style('stroke-width',2)
-              .style('stroke-dasharray','10,10')
-              .style('stroke',function (d,i) {
-                  return lines.color()(d,d.seriesIndex);
-              })
-              .attr('x1',0)
-              .attr('x2',availableWidth)
-              .attr('y1', getAvgLineY)
-              .attr('y2', getAvgLineY);
-
-      avgLines
-              .style('stroke-opacity',function(d){
-                  //If average lines go offscreen, make them transparent
-                  var yVal = y(average(d));
-                  if (yVal < 0 || yVal > availableHeight) return 0;
-                  return 1;
-              })
-              .attr('x1',0)
-              .attr('x2',availableWidth)
-              .attr('y1', getAvgLineY)
-              .attr('y2', getAvgLineY);
-
-      avgLines.exit().remove();
-
-      //Create index line -----------------------------------------
-
-      var indexLine = linesWrap.selectAll('.nv-indexLine')
-          .data([index]);
-      indexLine.enter().append('rect').attr('class', 'nv-indexLine')
-          .attr('width', 3)
-          .attr('x', -2)
-          .attr('fill', 'red')
-          .attr('fill-opacity', .5)
-          .style("pointer-events","all")
-          .call(indexDrag)
-
-      indexLine
-          .attr('transform', function(d) { return 'translate(' + dx(d.i) + ',0)' })
-          .attr('height', availableHeight)
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-        xAxis
-          .scale(x)
-          //Suggest how many ticks based on the chart width and D3 should listen (70 is the optimal number for MM/DD/YY dates)
-          .ticks( Math.min(data[0].values.length,availableWidth/70) )
-          .tickSize(-availableHeight, 0);
-
-        g.select('.nv-x.nv-axis')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')');
-        d3.transition(g.select('.nv-x.nv-axis'))
-            .call(xAxis);
-      }
-
-
-      if (showYAxis) {
-        yAxis
-          .scale(y)
-          .ticks( availableHeight / 36 )
-          .tickSize( -availableWidth, 0);
-
-        d3.transition(g.select('.nv-y.nv-axis'))
-            .call(yAxis);
-      }
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-
-      function updateZero() {
-        indexLine
-          .data([index]);
-        
-        //When dragging the index line, turn off line transitions.
-        // Then turn them back on when done dragging.
-        var oldDuration = chart.transitionDuration();
-        chart.transitionDuration(0);
-        chart.update();
-        chart.transitionDuration(oldDuration);
-      }
-
-      g.select('.nv-background rect')
-          .on('click', function() {
-            index.x = d3.mouse(this)[0];
-            index.i = Math.round(dx.invert(index.x));
-
-            // update state and send stateChange with new index
-            state.index = index.i;
-            dispatch.stateChange(state);
-
-            updateZero();
-          });
-
-      lines.dispatch.on('elementClick', function(e) {
-        index.i = e.pointIndex;
-        index.x = dx(index.i);
-
-        // update state and send stateChange with new index
-        state.index = index.i;
-        dispatch.stateChange(state);
-
-        updateZero();
-      });
-
-      controls.dispatch.on('legendClick', function(d,i) { 
-        d.disabled = !d.disabled;
-        rescaleY = !d.disabled;
-
-        state.rescaleY = rescaleY;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-
-      legend.dispatch.on('stateChange', function(newState) {
-        state.disabled = newState.disabled; 
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      interactiveLayer.dispatch.on('elementMousemove', function(e) {
-          lines.clearHighlights();
-          var singlePoint, pointIndex, pointXLocation, allData = [];
-          
-          
-          data
-          .filter(function(series, i) { 
-            series.seriesIndex = i;
-            return !series.disabled; 
-          })
-          .forEach(function(series,i) {
-              pointIndex = nv.interactiveBisect(series.values, e.pointXValue, chart.x());
-              lines.highlightPoint(i, pointIndex, true);
-              var point = series.values[pointIndex];
-              if (typeof point === 'undefined') return;
-              if (typeof singlePoint === 'undefined') singlePoint = point;
-              if (typeof pointXLocation === 'undefined') pointXLocation = chart.xScale()(chart.x()(point,pointIndex));
-              allData.push({
-                  key: series.key,
-                  value: chart.y()(point, pointIndex),
-                  color: color(series,series.seriesIndex)
-              });
-          });
-
-          //Highlight the tooltip entry based on which point the mouse is closest to.
-          if (allData.length > 2) {
-            var yValue = chart.yScale().invert(e.mouseY);
-            var yDistMax = Infinity, indexToHighlight = null;
-            allData.forEach(function(series,i) {
-               var delta = Math.abs(yValue - series.value);
-               if ( delta < yDistMax) {
-                  yDistMax = delta;
-                  indexToHighlight = i;
-               }
-            });
-            allData[indexToHighlight].highlight = true;
-          }
-
-          var xValue = xAxis.tickFormat()(chart.x()(singlePoint,pointIndex), pointIndex);
-          interactiveLayer.tooltip
-                  .position({left: pointXLocation + margin.left, top: e.mouseY + margin.top})
-                  .chartContainer(that.parentNode)
-                  .enabled(tooltips)
-                  .valueFormatter(function(d,i) {
-                     return yAxis.tickFormat()(d);
-                  })
-                  .data(
-                      {
-                        value: xValue,
-                        series: allData
-                      }
-                  )();
-
-          interactiveLayer.renderGuideLine(pointXLocation);
-
-      });
-
-      interactiveLayer.dispatch.on("elementMouseout",function(e) {
-          dispatch.tooltipHide();
-          lines.clearHighlights();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-
-        if (typeof e.index !== 'undefined') {
-          index.i = e.index;
-          index.x = dx(index.i);
-
-          state.index = e.index;
-
-          indexLine
-            .data([index]);
-        }
-
-
-        if (typeof e.rescaleY !== 'undefined') {
-          rescaleY = e.rescaleY;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  lines.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.lines = lines;
-  chart.legend = legend;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-  chart.interactiveLayer = interactiveLayer;
-
-  d3.rebind(chart, lines, 'defined', 'isArea', 'x', 'y', 'xScale','yScale', 'size', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'interactive', 'clipEdge', 'clipVoronoi','useVoronoi',  'id');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.rescaleY = function(_) {
-    if (!arguments.length) return rescaleY;
-    rescaleY = _
-    return rescaleY;
-  };
-
-  chart.showControls = function(_) {
-    if (!arguments.length) return showControls;
-    showControls = _;
-    return chart;
-  };
-
-  chart.useInteractiveGuideline = function(_) {
-    if(!arguments.length) return useInteractiveGuideline;
-    useInteractiveGuideline = _;
-    if (_ === true) {
-       chart.interactive(false);
-       chart.useVoronoi(false);
-    }
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.average = function(_) {
-     if(!arguments.length) return average;
-     average = _;
-     return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  //============================================================
-  // Functions
-  //------------------------------------------------------------
-
-  /* Normalize the data according to an index point. */
-  function indexify(idx, data) {
-    return data.map(function(line, i) {
-      if (!line.values) {
-         return line;
-      }
-      var v = lines.y()(line.values[idx], idx);
-
-      //TODO: implement check below, and disable series if series loses 100% or more cause divide by 0 issue
-      if (v < -.95) {
-        //if a series loses more than 100%, calculations fail.. anything close can cause major distortion (but is mathematically correct till it hits 100)
-        line.tempDisabled = true;
-        return line;
-      }
-
-      line.tempDisabled = false;
-
-      line.values = line.values.map(function(point, pointIndex) {
-        point.display = {'y': (lines.y()(point, pointIndex) - v) / (1 + v) };
-        return point;
-      })
-
-      return line;
-    })
-  }
-
-  //============================================================
-
-
-  return chart;
-}
-//TODO: consider deprecating by adding necessary features to multiBar model
-nv.models.discreteBar = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , id = Math.floor(Math.random() * 10000) //Create semi-unique ID in case user doesn't select one
-    , x = d3.scale.ordinal()
-    , y = d3.scale.linear()
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , forceY = [0] // 0 is forced by default.. this makes sense for the majority of bar graphs... user can always do chart.forceY([]) to remove
-    , color = nv.utils.defaultColor()
-    , showValues = false
-    , valueFormat = d3.format(',.2f')
-    , xDomain
-    , yDomain
-    , xRange
-    , yRange
-    , dispatch = d3.dispatch('chartClick', 'elementClick', 'elementDblClick', 'elementMouseover', 'elementMouseout')
-    , rectClass = 'discreteBar'
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-
-      //add series index to each data point for reference
-      data = data.map(function(series, i) {
-        series.values = series.values.map(function(point) {
-          point.series = i;
-          return point;
-        });
-        return series;
-      });
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      // remap and flatten the data for use in calculating the scales' domains
-      var seriesData = (xDomain && yDomain) ? [] : // if we know xDomain and yDomain, no need to calculate
-            data.map(function(d) {
-              return d.values.map(function(d,i) {
-                return { x: getX(d,i), y: getY(d,i), y0: d.y0 }
-              })
-            });
-
-      x   .domain(xDomain || d3.merge(seriesData).map(function(d) { return d.x }))
-          .rangeBands(xRange || [0, availableWidth], .1);
-
-      y   .domain(yDomain || d3.extent(d3.merge(seriesData).map(function(d) { return d.y }).concat(forceY)));
-
-
-      // If showValues, pad the Y axis range to account for label height
-      if (showValues) y.range(yRange || [availableHeight - (y.domain()[0] < 0 ? 12 : 0), y.domain()[1] > 0 ? 12 : 0]);
-      else y.range(yRange || [availableHeight, 0]);
-
-      //store old scales if they exist
-      x0 = x0 || x;
-      y0 = y0 || y.copy().range([y(0),y(0)]);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-discretebar').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-discretebar');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-groups');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-
-      //TODO: by definition, the discrete bar should not have multiple groups, will modify/remove later
-      var groups = wrap.select('.nv-groups').selectAll('.nv-group')
-          .data(function(d) { return d }, function(d) { return d.key });
-      groups.enter().append('g')
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6);
-      groups.exit()
-          .transition()
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6)
-          .remove();
-      groups
-          .attr('class', function(d,i) { return 'nv-group nv-series-' + i })
-          .classed('hover', function(d) { return d.hover });
-      groups
-          .transition()
-          .style('stroke-opacity', 1)
-          .style('fill-opacity', .75);
-
-
-      var bars = groups.selectAll('g.nv-bar')
-          .data(function(d) { return d.values });
-
-      bars.exit().remove();
-
-
-      var barsEnter = bars.enter().append('g')
-          .attr('transform', function(d,i,j) {
-              return 'translate(' + (x(getX(d,i)) + x.rangeBand() * .05 ) + ', ' + y(0) + ')' 
-          })
-          .on('mouseover', function(d,i) { //TODO: figure out why j works above, but not here
-            d3.select(this).classed('hover', true);
-            dispatch.elementMouseover({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (d.series + .5) / data.length), y(getY(d,i))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-          })
-          .on('mouseout', function(d,i) {
-            d3.select(this).classed('hover', false);
-            dispatch.elementMouseout({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-          })
-          .on('click', function(d,i) {
-            dispatch.elementClick({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (d.series + .5) / data.length), y(getY(d,i))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-            d3.event.stopPropagation();
-          })
-          .on('dblclick', function(d,i) {
-            dispatch.elementDblClick({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (d.series + .5) / data.length), y(getY(d,i))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-            d3.event.stopPropagation();
-          });
-
-      barsEnter.append('rect')
-          .attr('height', 0)
-          .attr('width', x.rangeBand() * .9 / data.length )
-
-      if (showValues) {
-        barsEnter.append('text')
-          .attr('text-anchor', 'middle')
-          ;
-
-        bars.select('text')
-          .text(function(d,i) { return valueFormat(getY(d,i)) })
-          .transition()
-          .attr('x', x.rangeBand() * .9 / 2)
-          .attr('y', function(d,i) { return getY(d,i) < 0 ? y(getY(d,i)) - y(0) + 12 : -4 })
-          
-          ;
-      } else {
-        bars.selectAll('text').remove();
-      }
-
-      bars
-          .attr('class', function(d,i) { return getY(d,i) < 0 ? 'nv-bar negative' : 'nv-bar positive' })
-          .style('fill', function(d,i) { return d.color || color(d,i) })
-          .style('stroke', function(d,i) { return d.color || color(d,i) })
-        .select('rect')
-          .attr('class', rectClass)
-          .transition()
-          .attr('width', x.rangeBand() * .9 / data.length);
-      bars.transition()
-        //.delay(function(d,i) { return i * 1200 / data[0].values.length })
-          .attr('transform', function(d,i) {
-            var left = x(getX(d,i)) + x.rangeBand() * .05,
-                top = getY(d,i) < 0 ?
-                        y(0) :
-                        y(0) - y(getY(d,i)) < 1 ?
-                          y(0) - 1 : //make 1 px positive bars show up above y=0
-                          y(getY(d,i));
-
-              return 'translate(' + left + ', ' + top + ')'
-          })
-        .select('rect')
-          .attr('height', function(d,i) {
-            return  Math.max(Math.abs(y(getY(d,i)) - y((yDomain && yDomain[0]) || 0)) || 1)
-          });
-
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.forceY = function(_) {
-    if (!arguments.length) return forceY;
-    forceY = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.showValues = function(_) {
-    if (!arguments.length) return showValues;
-    showValues = _;
-    return chart;
-  };
-
-  chart.valueFormat= function(_) {
-    if (!arguments.length) return valueFormat;
-    valueFormat = _;
-    return chart;
-  };
-
-  chart.rectClass= function(_) {
-    if (!arguments.length) return rectClass;
-    rectClass = _;
-    return chart;
-  };
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.discreteBarChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var discretebar = nv.models.discreteBar()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    ;
-
-  var margin = {top: 15, right: 10, bottom: 50, left: 60}
-    , width = null
-    , height = null
-    , color = nv.utils.getColor()
-    , showXAxis = true
-    , showYAxis = true
-    , rightAlignYAxis = false
-    , staggerLabels = false
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + x + '</h3>' +
-               '<p>' +  y + '</p>'
-      }
-    , x
-    , y
-    , noData = "No Data Available."
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'beforeUpdate')
-    , transitionDuration = 250
-    ;
-
-  xAxis
-    .orient('bottom')
-    .highlightZero(false)
-    .showMaxMin(false)
-    .tickFormat(function(d) { return d })
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    .tickFormat(d3.format(',.1f'))
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(discretebar.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(discretebar.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-
-      chart.update = function() { 
-        dispatch.beforeUpdate(); 
-        container.transition().duration(transitionDuration).call(chart); 
-      };
-      chart.container = this;
-
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = discretebar.xScale();
-      y = discretebar.yScale().clamp(true);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-discreteBarWithAxes').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-discreteBarWithAxes').append('g');
-      var defsEnter = gEnter.append('defs');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-barsWrap');
-
-      g.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      discretebar
-        .width(availableWidth)
-        .height(availableHeight);
-
-
-      var barsWrap = g.select('.nv-barsWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      barsWrap.transition().call(discretebar);
-
-      //------------------------------------------------------------
-
-
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-x-label-clip-' + discretebar.id())
-        .append('rect');
-
-      g.select('#nv-x-label-clip-' + discretebar.id() + ' rect')
-          .attr('width', x.rangeBand() * (staggerLabels ? 2 : 1))
-          .attr('height', 16)
-          .attr('x', -x.rangeBand() / (staggerLabels ? 1 : 2 ));
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-          xAxis
-            .scale(x)
-            .ticks( availableWidth / 100 )
-            .tickSize(-availableHeight, 0);
-
-          g.select('.nv-x.nv-axis')
-              .attr('transform', 'translate(0,' + (y.range()[0] + ((discretebar.showValues() && y.domain()[0] < 0) ? 16 : 0)) + ')');
-          //d3.transition(g.select('.nv-x.nv-axis'))
-          g.select('.nv-x.nv-axis').transition()
-              .call(xAxis);
-
-
-          var xTicks = g.select('.nv-x.nv-axis').selectAll('g');
-
-          if (staggerLabels) {
-            xTicks
-                .selectAll('text')
-                .attr('transform', function(d,i,j) { return 'translate(0,' + (j % 2 == 0 ? '5' : '17') + ')' })
-          }
-      }
-
-      if (showYAxis) {
-          yAxis
-            .scale(y)
-            .ticks( availableHeight / 36 )
-            .tickSize( -availableWidth, 0);
-
-          g.select('.nv-y.nv-axis').transition()
-              .call(yAxis);
-      }
-
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      //============================================================
-
-
-    });
-
-    return chart;
-  }
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  discretebar.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  discretebar.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.discretebar = discretebar;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-
-  d3.rebind(chart, discretebar, 'x', 'y', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'id', 'showValues', 'valueFormat');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    discretebar.color(color);
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.staggerLabels = function(_) {
-    if (!arguments.length) return staggerLabels;
-    staggerLabels = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.distribution = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 400 //technically width or height depending on x or y....
-    , size = 8
-    , axis = 'x' // 'x' or 'y'... horizontal or vertical
-    , getData = function(d) { return d[axis] }  // defaults d.x or d.y
-    , color = nv.utils.defaultColor()
-    , scale = d3.scale.linear()
-    , domain
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var scale0;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableLength = width - (axis === 'x' ? margin.left + margin.right : margin.top + margin.bottom),
-          naxis = axis == 'x' ? 'y' : 'x',
-          container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      scale0 = scale0 || scale;
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-distribution').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-distribution');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')')
-
-      //------------------------------------------------------------
-
-
-      var distWrap = g.selectAll('g.nv-dist')
-          .data(function(d) { return d }, function(d) { return d.key });
-
-      distWrap.enter().append('g');
-      distWrap
-          .attr('class', function(d,i) { return 'nv-dist nv-series-' + i })
-          .style('stroke', function(d,i) { return color(d, i) });
-
-      var dist = distWrap.selectAll('line.nv-dist' + axis)
-          .data(function(d) { return d.values })
-      dist.enter().append('line')
-          .attr(axis + '1', function(d,i) { return scale0(getData(d,i)) })
-          .attr(axis + '2', function(d,i) { return scale0(getData(d,i)) })
-      distWrap.exit().selectAll('line.nv-dist' + axis)
-          .transition()
-          .attr(axis + '1', function(d,i) { return scale(getData(d,i)) })
-          .attr(axis + '2', function(d,i) { return scale(getData(d,i)) })
-          .style('stroke-opacity', 0)
-          .remove();
-      dist
-          .attr('class', function(d,i) { return 'nv-dist' + axis + ' nv-dist' + axis + '-' + i })
-          .attr(naxis + '1', 0)
-          .attr(naxis + '2', size);
-      dist
-          .transition()
-          .attr(axis + '1', function(d,i) { return scale(getData(d,i)) })
-          .attr(axis + '2', function(d,i) { return scale(getData(d,i)) })
-
-
-      scale0 = scale.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.axis = function(_) {
-    if (!arguments.length) return axis;
-    axis = _;
-    return chart;
-  };
-
-  chart.size = function(_) {
-    if (!arguments.length) return size;
-    size = _;
-    return chart;
-  };
-
-  chart.getData = function(_) {
-    if (!arguments.length) return getData;
-    getData = d3.functor(_);
-    return chart;
-  };
-
-  chart.scale = function(_) {
-    if (!arguments.length) return scale;
-    scale = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-  //============================================================
-
-
-  return chart;
-}
-//TODO: consider deprecating and using multibar with single series for this
-nv.models.historicalBar = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , id = Math.floor(Math.random() * 10000) //Create semi-unique ID in case user doesn't select one
-    , x = d3.scale.linear()
-    , y = d3.scale.linear()
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , forceX = []
-    , forceY = [0]
-    , padData = false
-    , clipEdge = true
-    , color = nv.utils.defaultColor()
-    , xDomain
-    , yDomain
-    , xRange
-    , yRange
-    , dispatch = d3.dispatch('chartClick', 'elementClick', 'elementDblClick', 'elementMouseover', 'elementMouseout')
-    , interactive = true
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x   .domain(xDomain || d3.extent(data[0].values.map(getX).concat(forceX) ))
-
-      if (padData)
-        x.range(xRange || [availableWidth * .5 / data[0].values.length, availableWidth * (data[0].values.length - .5)  / data[0].values.length ]);
-      else
-        x.range(xRange || [0, availableWidth]);
-
-      y   .domain(yDomain || d3.extent(data[0].values.map(getY).concat(forceY) ))
-          .range(yRange || [availableHeight, 0]);
-
-      // If scale's domain don't have a range, slightly adjust to make one... so a chart can show a single data point
-
-      if (x.domain()[0] === x.domain()[1])
-        x.domain()[0] ?
-            x.domain([x.domain()[0] - x.domain()[0] * 0.01, x.domain()[1] + x.domain()[1] * 0.01])
-          : x.domain([-1,1]);
-
-      if (y.domain()[0] === y.domain()[1])
-        y.domain()[0] ?
-            y.domain([y.domain()[0] + y.domain()[0] * 0.01, y.domain()[1] - y.domain()[1] * 0.01])
-          : y.domain([-1,1]);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-historicalBar-' + id).data([data[0].values]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-historicalBar-' + id);
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-bars');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      container
-          .on('click', function(d,i) {
-            dispatch.chartClick({
-                data: d,
-                index: i,
-                pos: d3.event,
-                id: id
-            });
-          });
-
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-chart-clip-path-' + id)
-        .append('rect');
-
-      wrap.select('#nv-chart-clip-path-' + id + ' rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      g   .attr('clip-path', clipEdge ? 'url(#nv-chart-clip-path-' + id + ')' : '');
-
-
-
-      var bars = wrap.select('.nv-bars').selectAll('.nv-bar')
-          .data(function(d) { return d }, function(d,i) {return getX(d,i)});
-
-      bars.exit().remove();
-
-
-      var barsEnter = bars.enter().append('rect')
-          //.attr('class', function(d,i,j) { return (getY(d,i) < 0 ? 'nv-bar negative' : 'nv-bar positive') + ' nv-bar-' + j + '-' + i })
-          .attr('x', 0 )
-          .attr('y', function(d,i) {  return nv.utils.NaNtoZero(y(Math.max(0, getY(d,i)))) })
-          .attr('height', function(d,i) { return nv.utils.NaNtoZero(Math.abs(y(getY(d,i)) - y(0))) })
-          .attr('transform', function(d,i) { return 'translate(' + (x(getX(d,i)) - availableWidth / data[0].values.length * .45) + ',0)'; }) 
-          .on('mouseover', function(d,i) {
-            if (!interactive) return;
-            d3.select(this).classed('hover', true);
-            dispatch.elementMouseover({
-                point: d,
-                series: data[0],
-                pos: [x(getX(d,i)), y(getY(d,i))],  // TODO: Figure out why the value appears to be shifted
-                pointIndex: i,
-                seriesIndex: 0,
-                e: d3.event
-            });
-
-          })
-          .on('mouseout', function(d,i) {
-                if (!interactive) return;
-                d3.select(this).classed('hover', false);
-                dispatch.elementMouseout({
-                    point: d,
-                    series: data[0],
-                    pointIndex: i,
-                    seriesIndex: 0,
-                    e: d3.event
-                });
-          })
-          .on('click', function(d,i) {
-                if (!interactive) return;
-                dispatch.elementClick({
-                    //label: d[label],
-                    value: getY(d,i),
-                    data: d,
-                    index: i,
-                    pos: [x(getX(d,i)), y(getY(d,i))],
-                    e: d3.event,
-                    id: id
-                });
-              d3.event.stopPropagation();
-          })
-          .on('dblclick', function(d,i) {
-              if (!interactive) return;
-              dispatch.elementDblClick({
-                  //label: d[label],
-                  value: getY(d,i),
-                  data: d,
-                  index: i,
-                  pos: [x(getX(d,i)), y(getY(d,i))],
-                  e: d3.event,
-                  id: id
-              });
-              d3.event.stopPropagation();
-          });
-
-      bars
-          .attr('fill', function(d,i) { return color(d, i); })
-          .attr('class', function(d,i,j) { return (getY(d,i) < 0 ? 'nv-bar negative' : 'nv-bar positive') + ' nv-bar-' + j + '-' + i })
-          .transition()
-          .attr('transform', function(d,i) { return 'translate(' + (x(getX(d,i)) - availableWidth / data[0].values.length * .45) + ',0)'; }) 
-           //TODO: better width calculations that don't assume always uniform data spacing;w
-          .attr('width', (availableWidth / data[0].values.length) * .9 );
-
-
-      bars.transition()
-          .attr('y', function(d,i) {
-            var rval = getY(d,i) < 0 ?
-                    y(0) :
-                    y(0) - y(getY(d,i)) < 1 ?
-                      y(0) - 1 :
-                      y(getY(d,i));
-            return nv.utils.NaNtoZero(rval);
-          })
-          .attr('height', function(d,i) { return nv.utils.NaNtoZero(Math.max(Math.abs(y(getY(d,i)) - y(0)),1)) });
-
-    });
-
-    return chart;
-  }
-
-  //Create methods to allow outside functions to highlight a specific bar.
-  chart.highlightPoint = function(pointIndex, isHoverOver) {
-      d3.select(".nv-historicalBar-" + id)
-        .select(".nv-bars .nv-bar-0-" + pointIndex)
-              .classed("hover", isHoverOver)
-               ;
-  };
-
-  chart.clearHighlights = function() {
-      d3.select(".nv-historicalBar-" + id)
-        .select(".nv-bars .nv-bar.hover")
-              .classed("hover", false)
-               ;
-  };
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.forceX = function(_) {
-    if (!arguments.length) return forceX;
-    forceX = _;
-    return chart;
-  };
-
-  chart.forceY = function(_) {
-    if (!arguments.length) return forceY;
-    forceY = _;
-    return chart;
-  };
-
-  chart.padData = function(_) {
-    if (!arguments.length) return padData;
-    padData = _;
-    return chart;
-  };
-
-  chart.clipEdge = function(_) {
-    if (!arguments.length) return clipEdge;
-    clipEdge = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.interactive = function(_) {
-    if(!arguments.length) return interactive;
-    interactive = false;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.historicalBarChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var bars = nv.models.historicalBar()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , legend = nv.models.legend()
-    ;
-
-
-  var margin = {top: 30, right: 90, bottom: 50, left: 90}
-    , color = nv.utils.defaultColor()
-    , width = null
-    , height = null
-    , showLegend = false
-    , showXAxis = true
-    , showYAxis = true
-    , rightAlignYAxis = false
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>'
-      }
-    , x
-    , y
-    , state = {}
-    , defaultState = null
-    , noData = 'No Data Available.'
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , transitionDuration = 250
-    ;
-
-  xAxis
-    .orient('bottom')
-    .tickPadding(7)
-    ;
-  yAxis
-    .orient( (rightAlignYAxis) ? 'right' : 'left')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-
-    // New addition to calculate position if SVG is scaled with viewBox, may move TODO: consider implementing everywhere else
-    if (offsetElement) {
-      var svg = d3.select(offsetElement).select('svg');
-      var viewBox = (svg.node()) ? svg.attr('viewBox') : null;
-      if (viewBox) {
-        viewBox = viewBox.split(' ');
-        var ratio = parseInt(svg.style('width')) / viewBox[2];
-        e.pos[0] = e.pos[0] * ratio;
-        e.pos[1] = e.pos[1] * ratio;
-      }
-    }
-
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(bars.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(bars.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, null, null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart) };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display noData message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = bars.xScale();
-      y = bars.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-historicalBarChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-historicalBarChart').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-barsWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width(availableWidth);
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        wrap.select('.nv-legendWrap')
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-      }
-
-      //------------------------------------------------------------
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-        g.select(".nv-y.nv-axis")
-            .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      bars
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled }));
-
-
-      var barsWrap = g.select('.nv-barsWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      barsWrap.transition().call(bars);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-        xAxis
-          .scale(x)
-          .tickSize(-availableHeight, 0);
-
-        g.select('.nv-x.nv-axis')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')');
-        g.select('.nv-x.nv-axis')
-            .transition()
-            .call(xAxis);
-      }
-
-      if (showYAxis) {
-        yAxis
-          .scale(y)
-          .ticks( availableHeight / 36 )
-          .tickSize( -availableWidth, 0);
-
-        g.select('.nv-y.nv-axis')
-          .transition()
-            .call(yAxis);
-      }
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('legendClick', function(d,i) { 
-        d.disabled = !d.disabled;
-
-        if (!data.filter(function(d) { return !d.disabled }).length) {
-          data.map(function(d) {
-            d.disabled = false;
-            wrap.selectAll('.nv-series').classed('disabled', false);
-            return d;
-          });
-        }
-
-        state.disabled = data.map(function(d) { return !!d.disabled });
-        dispatch.stateChange(state);
-
-        selection.transition().call(chart);
-      });
-
-      legend.dispatch.on('legendDblclick', function(d) {
-          //Double clicking should always enable current series, and disabled all others.
-          data.forEach(function(d) {
-             d.disabled = true;
-          });
-          d.disabled = false;  
-
-          state.disabled = data.map(function(d) { return !!d.disabled });
-          dispatch.stateChange(state);
-          chart.update();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        selection.call(chart);
-      });
-
-      //============================================================
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  bars.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  bars.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.bars = bars;
-  chart.legend = legend;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-
-  d3.rebind(chart, bars, 'defined', 'isArea', 'x', 'y', 'size', 'xScale', 'yScale', 
-    'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'interactive', 'clipEdge', 'clipVoronoi', 'id', 'interpolate','highlightPoint','clearHighlights', 'interactive');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-nv.models.indentedTree = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0} //TODO: implement, maybe as margin on the containing div
-    , width = 960
-    , height = 500
-    , color = nv.utils.defaultColor()
-    , id = Math.floor(Math.random() * 10000)
-    , header = true
-    , filterZero = false
-    , noData = "No Data Available."
-    , childIndent = 20
-    , columns = [{key:'key', label: 'Name', type:'text'}] //TODO: consider functions like chart.addColumn, chart.removeColumn, instead of a block like this
-    , tableClass = null
-    , iconOpen = 'images/grey-plus.png' //TODO: consider removing this and replacing with a '+' or '-' unless user defines images
-    , iconClose = 'images/grey-minus.png'
-    , dispatch = d3.dispatch('elementClick', 'elementDblclick', 'elementMouseover', 'elementMouseout')
-    , getUrl = function(d) { return d.url }
-    ;
-
-  //============================================================
-
-  var idx = 0;
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var depth = 1,
-          container = d3.select(this);
-
-      var tree = d3.layout.tree()
-          .children(function(d) { return d.values })
-          .size([height, childIndent]); //Not sure if this is needed now that the result is HTML
-
-      chart.update = function() { container.transition().duration(600).call(chart) };
-
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-      if (!data[0]) data[0] = {key: noData};
-
-      //------------------------------------------------------------
-
-
-      var nodes = tree.nodes(data[0]);
-
-      // nodes.map(function(d) {
-      //   d.id = i++;
-      // })
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = d3.select(this).selectAll('div').data([[nodes]]);
-      var wrapEnter = wrap.enter().append('div').attr('class', 'nvd3 nv-wrap nv-indentedtree');
-      var tableEnter = wrapEnter.append('table');
-      var table = wrap.select('table').attr('width', '100%').attr('class', tableClass);
-
-      //------------------------------------------------------------
-
-
-      if (header) {
-        var thead = tableEnter.append('thead');
-
-        var theadRow1 = thead.append('tr');
-
-        columns.forEach(function(column) {
-          theadRow1
-            .append('th')
-              .attr('width', column.width ? column.width : '10%')
-              .style('text-align', column.type == 'numeric' ? 'right' : 'left')
-            .append('span')
-              .text(column.label);
-        });
-      }
-
-
-      var tbody = table.selectAll('tbody')
-                    .data(function(d) { return d });
-      tbody.enter().append('tbody');
-
-
-
-      //compute max generations
-      depth = d3.max(nodes, function(node) { return node.depth });
-      tree.size([height, depth * childIndent]); //TODO: see if this is necessary at all
-
-
-      // Update the nodes…
-      var node = tbody.selectAll('tr')
-          // .data(function(d) { return d; }, function(d) { return d.id || (d.id == ++i)});
-          .data(function(d) { return d.filter(function(d) { return (filterZero && !d.children) ? filterZero(d) :  true; } )}, function(d,i) { return d.id || (d.id || ++idx)});
-          //.style('display', 'table-row'); //TODO: see if this does anything
-
-      node.exit().remove();
-
-      node.select('img.nv-treeicon')
-          .attr('src', icon)
-          .classed('folded', folded);
-
-      var nodeEnter = node.enter().append('tr');
-
-
-      columns.forEach(function(column, index) {
-
-        var nodeName = nodeEnter.append('td')
-            .style('padding-left', function(d) { return (index ? 0 : d.depth * childIndent + 12 + (icon(d) ? 0 : 16)) + 'px' }, 'important') //TODO: check why I did the ternary here
-            .style('text-align', column.type == 'numeric' ? 'right' : 'left');
-
-
-        if (index == 0) {
-          nodeName.append('img')
-              .classed('nv-treeicon', true)
-              .classed('nv-folded', folded)
-              .attr('src', icon)
-              .style('width', '14px')
-              .style('height', '14px')
-              .style('padding', '0 1px')
-              .style('display', function(d) { return icon(d) ? 'inline-block' : 'none'; })
-              .on('click', click);
-        }
-
-
-        nodeName.each(function(d) {
-          if (!index && getUrl(d))
-            d3.select(this)
-              .append('a')
-              .attr('href',getUrl)
-              .attr('class', d3.functor(column.classes))
-              .append('span')
-          else
-            d3.select(this)
-              .append('span')
-
-            d3.select(this).select('span')
-              .attr('class', d3.functor(column.classes) )
-              .text(function(d) { return column.format ? column.format(d) :
-                                        (d[column.key] || '-') });
-          });
-
-        if  (column.showCount) {
-          nodeName.append('span')
-              .attr('class', 'nv-childrenCount');
-
-          node.selectAll('span.nv-childrenCount').text(function(d) {
-                return ((d.values && d.values.length) || (d._values && d._values.length)) ?                                   //If this is a parent
-                    '(' + ((d.values && (d.values.filter(function(d) { return filterZero ? filterZero(d) :  true; }).length)) //If children are in values check its children and filter
-                    || (d._values && d._values.filter(function(d) { return filterZero ? filterZero(d) :  true; }).length)     //Otherwise, do the same, but with the other name, _values...
-                    || 0) + ')'                                                                                               //This is the catch-all in case there are no children after a filter
-                    : ''                                                                                                     //If this is not a parent, just give an empty string
-            });
-        }
-
-        // if (column.click)
-        //   nodeName.select('span').on('click', column.click);
-
-      });
-
-      node
-        .order()
-        .on('click', function(d) { 
-          dispatch.elementClick({
-            row: this, //TODO: decide whether or not this should be consistent with scatter/line events or should be an html link (a href)
-            data: d,
-            pos: [d.x, d.y]
-          });
-        })
-        .on('dblclick', function(d) { 
-          dispatch.elementDblclick({
-            row: this,
-            data: d,
-            pos: [d.x, d.y]
-          });
-        })
-        .on('mouseover', function(d) { 
-          dispatch.elementMouseover({
-            row: this,
-            data: d,
-            pos: [d.x, d.y]
-          });
-        })
-        .on('mouseout', function(d) { 
-          dispatch.elementMouseout({
-            row: this,
-            data: d,
-            pos: [d.x, d.y]
-          });
-        });
-
-
-
-
-      // Toggle children on click.
-      function click(d, _, unshift) {
-        d3.event.stopPropagation();
-
-        if(d3.event.shiftKey && !unshift) {
-          //If you shift-click, it'll toggle fold all the children, instead of itself
-          d3.event.shiftKey = false;
-          d.values && d.values.forEach(function(node){
-            if (node.values || node._values) {
-              click(node, 0, true);
-            }
-          });
-          return true;
-        }
-        if(!hasChildren(d)) {
-          //download file
-          //window.location.href = d.url;
-          return true;
-        }
-        if (d.values) {
-          d._values = d.values;
-          d.values = null;
-        } else {
-          d.values = d._values;
-          d._values = null;
-        }
-        chart.update();
-      }
-
-
-      function icon(d) {
-        return (d._values && d._values.length) ? iconOpen : (d.values && d.values.length) ? iconClose : '';
-      }
-
-      function folded(d) {
-        return (d._values && d._values.length);
-      }
-
-      function hasChildren(d) {
-        var values = d.values || d._values;
-
-        return (values && values.length);
-      }
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    scatter.color(color);
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.header = function(_) {
-    if (!arguments.length) return header;
-    header = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.filterZero = function(_) {
-    if (!arguments.length) return filterZero;
-    filterZero = _;
-    return chart;
-  };
-
-  chart.columns = function(_) {
-    if (!arguments.length) return columns;
-    columns = _;
-    return chart;
-  };
-
-  chart.tableClass = function(_) {
-    if (!arguments.length) return tableClass;
-    tableClass = _;
-    return chart;
-  };
-
-  chart.iconOpen = function(_){
-     if (!arguments.length) return iconOpen;
-    iconOpen = _;
-    return chart;
-  }
-
-  chart.iconClose = function(_){
-     if (!arguments.length) return iconClose;
-    iconClose = _;
-    return chart;
-  }
-
-  chart.getUrl = function(_){
-     if (!arguments.length) return getUrl;
-    getUrl = _;
-    return chart;
-  }
-
-  //============================================================
-
-
-  return chart;
-};nv.models.legend = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 5, right: 0, bottom: 5, left: 0}
-    , width = 400
-    , height = 20
-    , getKey = function(d) { return d.key }
-    , color = nv.utils.defaultColor()
-    , align = true
-    , rightAlign = true
-    , updateState = true   //If true, legend will update data.disabled and trigger a 'stateChange' dispatch.
-    , radioButtonMode = false   //If true, clicking legend items will cause it to behave like a radio button. (only one can be selected at a time)
-    , dispatch = d3.dispatch('legendClick', 'legendDblclick', 'legendMouseover', 'legendMouseout', 'stateChange')
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-legend').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-legend').append('g');
-      var g = wrap.select('g');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      var series = g.selectAll('.nv-series')
-          .data(function(d) { return d });
-      var seriesEnter = series.enter().append('g').attr('class', 'nv-series')
-          .on('mouseover', function(d,i) {
-            dispatch.legendMouseover(d,i);  //TODO: Make consistent with other event objects
-          })
-          .on('mouseout', function(d,i) {
-            dispatch.legendMouseout(d,i);
-          })
-          .on('click', function(d,i) {
-            dispatch.legendClick(d,i);
-            if (updateState) {
-               if (radioButtonMode) {
-                   //Radio button mode: set every series to disabled,
-                   //  and enable the clicked series.
-                   data.forEach(function(series) { series.disabled = true});
-                   d.disabled = false;
-               }
-               else {
-                   d.disabled = !d.disabled;
-                   if (data.every(function(series) { return series.disabled})) {
-                       //the default behavior of NVD3 legends is, if every single series
-                       // is disabled, turn all series' back on.
-                       data.forEach(function(series) { series.disabled = false});
-                   }
-               }
-               dispatch.stateChange({
-                  disabled: data.map(function(d) { return !!d.disabled })
-               });
-            }
-          })
-          .on('dblclick', function(d,i) {
-            dispatch.legendDblclick(d,i);
-            if (updateState) {
-                //the default behavior of NVD3 legends, when double clicking one,
-                // is to set all other series' to false, and make the double clicked series enabled.
-                data.forEach(function(series) {
-                   series.disabled = true;
-                });
-                d.disabled = false; 
-                dispatch.stateChange({
-                    disabled: data.map(function(d) { return !!d.disabled })
-                });
-            }
-          });
-      seriesEnter.append('circle')
-          .style('stroke-width', 2)
-          .attr('class','nv-legend-symbol')
-          .attr('r', 5);
-      seriesEnter.append('text')
-          .attr('text-anchor', 'start')
-          .attr('class','nv-legend-text')
-          .attr('dy', '.32em')
-          .attr('dx', '8');
-      series.classed('disabled', function(d) { return d.disabled });
-      series.exit().remove();
-      series.select('circle')
-          .style('fill', function(d,i) { return d.color || color(d,i)})
-          .style('stroke', function(d,i) { return d.color || color(d, i) });
-      series.select('text').text(getKey);
-
-
-      //TODO: implement fixed-width and max-width options (max-width is especially useful with the align option)
-
-      // NEW ALIGNING CODE, TODO: clean up
-      if (align) {
-
-        var seriesWidths = [];
-        series.each(function(d,i) {
-              var legendText = d3.select(this).select('text');
-              var nodeTextLength;
-              try {
-                nodeTextLength = legendText.node().getComputedTextLength();
-              }
-              catch(e) {
-                nodeTextLength = nv.utils.calcApproxTextWidth(legendText);
-              }
-             
-              seriesWidths.push(nodeTextLength + 28); // 28 is ~ the width of the circle plus some padding
-            });
-
-        var seriesPerRow = 0;
-        var legendWidth = 0;
-        var columnWidths = [];
-
-        while ( legendWidth < availableWidth && seriesPerRow < seriesWidths.length) {
-          columnWidths[seriesPerRow] = seriesWidths[seriesPerRow];
-          legendWidth += seriesWidths[seriesPerRow++];
-        }
-        if (seriesPerRow === 0) seriesPerRow = 1; //minimum of one series per row
-
-
-        while ( legendWidth > availableWidth && seriesPerRow > 1 ) {
-          columnWidths = [];
-          seriesPerRow--;
-
-          for (var k = 0; k < seriesWidths.length; k++) {
-            if (seriesWidths[k] > (columnWidths[k % seriesPerRow] || 0) )
-              columnWidths[k % seriesPerRow] = seriesWidths[k];
-          }
-
-          legendWidth = columnWidths.reduce(function(prev, cur, index, array) {
-                          return prev + cur;
-                        });
-        }
-
-        var xPositions = [];
-        for (var i = 0, curX = 0; i < seriesPerRow; i++) {
-            xPositions[i] = curX;
-            curX += columnWidths[i];
-        }
-
-        series
-            .attr('transform', function(d, i) {
-              return 'translate(' + xPositions[i % seriesPerRow] + ',' + (5 + Math.floor(i / seriesPerRow) * 20) + ')';
-            });
-
-        //position legend as far right as possible within the total width
-        if (rightAlign) {
-           g.attr('transform', 'translate(' + (width - margin.right - legendWidth) + ',' + margin.top + ')');
-        }
-        else {
-           g.attr('transform', 'translate(0' + ',' + margin.top + ')');
-        }
-
-        height = margin.top + margin.bottom + (Math.ceil(seriesWidths.length / seriesPerRow) * 20);
-
-      } else {
-
-        var ypos = 5,
-            newxpos = 5,
-            maxwidth = 0,
-            xpos;
-        series
-            .attr('transform', function(d, i) {
-              var length = d3.select(this).select('text').node().getComputedTextLength() + 28;
-              xpos = newxpos;
-
-              if (width < margin.left + margin.right + xpos + length) {
-                newxpos = xpos = 5;
-                ypos += 20;
-              }
-
-              newxpos += length;
-              if (newxpos > maxwidth) maxwidth = newxpos;
-
-              return 'translate(' + xpos + ',' + ypos + ')';
-            });
-
-        //position legend as far right as possible within the total width
-        g.attr('transform', 'translate(' + (width - margin.right - maxwidth) + ',' + margin.top + ')');
-
-        height = margin.top + margin.bottom + ypos + 15;
-
-      }
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.key = function(_) {
-    if (!arguments.length) return getKey;
-    getKey = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.align = function(_) {
-    if (!arguments.length) return align;
-    align = _;
-    return chart;
-  };
-
-  chart.rightAlign = function(_) {
-    if (!arguments.length) return rightAlign;
-    rightAlign = _;
-    return chart;
-  };
-
-  chart.updateState = function(_) {
-    if (!arguments.length) return updateState;
-    updateState = _;
-    return chart;
-  };
-
-  chart.radioButtonMode = function(_) {
-    if (!arguments.length) return radioButtonMode;
-    radioButtonMode = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.line = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var  scatter = nv.models.scatter()
-    ;
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , color = nv.utils.defaultColor() // a function that returns a color
-    , getX = function(d) { return d.x } // accessor to get the x value from a data point
-    , getY = function(d) { return d.y } // accessor to get the y value from a data point
-    , defined = function(d,i) { return !isNaN(getY(d,i)) && getY(d,i) !== null } // allows a line to be not continuous when it is not defined
-    , isArea = function(d) { return d.area } // decides if a line is an area or just a line
-    , clipEdge = false // if true, masks lines within x and y scale
-    , x //can be accessed via chart.xScale()
-    , y //can be accessed via chart.yScale()
-    , interpolate = "linear" // controls the line interpolation
-    ;
-
-  scatter
-    .size(16) // default size
-    .sizeDomain([16,256]) //set to speed up calculation, needs to be unset if there is a custom size accessor
-    ;
-    
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0 //used to store previous scales
-      ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = scatter.xScale();
-      y = scatter.yScale();
-
-      x0 = x0 || x;
-      y0 = y0 || y;
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-line').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-line');
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g')
-
-      gEnter.append('g').attr('class', 'nv-groups');
-      gEnter.append('g').attr('class', 'nv-scatterWrap');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-
-
-      scatter
-        .width(availableWidth)
-        .height(availableHeight)
-
-      var scatterWrap = wrap.select('.nv-scatterWrap');
-          //.datum(data); // Data automatically trickles down from the wrap
-
-      scatterWrap.transition().call(scatter);
-
-
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-edge-clip-' + scatter.id())
-        .append('rect');
-
-      wrap.select('#nv-edge-clip-' + scatter.id() + ' rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      g   .attr('clip-path', clipEdge ? 'url(#nv-edge-clip-' + scatter.id() + ')' : '');
-      scatterWrap
-          .attr('clip-path', clipEdge ? 'url(#nv-edge-clip-' + scatter.id() + ')' : '');
-
-
-
-
-      var groups = wrap.select('.nv-groups').selectAll('.nv-group')
-          .data(function(d) { return d }, function(d) { return d.key });
-      groups.enter().append('g')
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6);
-      groups.exit()
-          .transition()
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6)
-          .remove();
-      groups
-          .attr('class', function(d,i) { return 'nv-group nv-series-' + i })
-          .classed('hover', function(d) { return d.hover })
-          .style('fill', function(d,i){ return color(d, i) })
-          .style('stroke', function(d,i){ return color(d, i)});
-      groups
-          .transition()
-          .style('stroke-opacity', 1)
-          .style('fill-opacity', .5);
-
-
-
-      var areaPaths = groups.selectAll('path.nv-area')
-          .data(function(d) { return isArea(d) ? [d] : [] }); // this is done differently than lines because I need to check if series is an area
-      areaPaths.enter().append('path')
-          .attr('class', 'nv-area')
-          .attr('d', function(d) {
-            return d3.svg.area()
-                .interpolate(interpolate)
-                .defined(defined)
-                .x(function(d,i) { return nv.utils.NaNtoZero(x0(getX(d,i))) })
-                .y0(function(d,i) { return nv.utils.NaNtoZero(y0(getY(d,i))) })
-                .y1(function(d,i) { return y0( y.domain()[0] <= 0 ? y.domain()[1] >= 0 ? 0 : y.domain()[1] : y.domain()[0] ) })
-                //.y1(function(d,i) { return y0(0) }) //assuming 0 is within y domain.. may need to tweak this
-                .apply(this, [d.values])
-          });
-      groups.exit().selectAll('path.nv-area')
-           .remove();
-           
-      areaPaths
-          .transition()
-          .attr('d', function(d) {
-            return d3.svg.area()
-                .interpolate(interpolate)
-                .defined(defined)
-                .x(function(d,i) { return nv.utils.NaNtoZero(x(getX(d,i))) })
-                .y0(function(d,i) { return nv.utils.NaNtoZero(y(getY(d,i))) })
-                .y1(function(d,i) { return y( y.domain()[0] <= 0 ? y.domain()[1] >= 0 ? 0 : y.domain()[1] : y.domain()[0] ) })
-                //.y1(function(d,i) { return y0(0) }) //assuming 0 is within y domain.. may need to tweak this
-                .apply(this, [d.values])
-          });
-
-
-
-      var linePaths = groups.selectAll('path.nv-line')
-          .data(function(d) { return [d.values] });
-      linePaths.enter().append('path')
-          .attr('class', 'nv-line')
-          .attr('d',
-            d3.svg.line()
-              .interpolate(interpolate)
-              .defined(defined)
-              .x(function(d,i) { return nv.utils.NaNtoZero(x0(getX(d,i))) })
-              .y(function(d,i) { return nv.utils.NaNtoZero(y0(getY(d,i))) })
-          );
-      groups.exit().selectAll('path.nv-line')
-          .transition()
-          .attr('d',
-            d3.svg.line()
-              .interpolate(interpolate)
-              .defined(defined)
-              .x(function(d,i) { return nv.utils.NaNtoZero(x(getX(d,i))) })
-              .y(function(d,i) { return nv.utils.NaNtoZero(y(getY(d,i))) })
-          );
-      linePaths
-          .transition()
-          .attr('d',
-            d3.svg.line()
-              .interpolate(interpolate)
-              .defined(defined)
-              .x(function(d,i) { return nv.utils.NaNtoZero(x(getX(d,i))) })
-              .y(function(d,i) { return nv.utils.NaNtoZero(y(getY(d,i))) })
-          );
-
-
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = scatter.dispatch;
-  chart.scatter = scatter;
-
-  d3.rebind(chart, scatter, 'id', 'interactive', 'size', 'xScale', 'yScale', 'zScale', 'xDomain', 'yDomain', 'xRange', 'yRange', 
-    'sizeDomain', 'forceX', 'forceY', 'forceSize', 'clipVoronoi', 'useVoronoi', 'clipRadius', 'padData','highlightPoint','clearHighlights');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    scatter.x(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    scatter.y(_);
-    return chart;
-  };
-
-  chart.clipEdge = function(_) {
-    if (!arguments.length) return clipEdge;
-    clipEdge = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    scatter.color(color);
-    return chart;
-  };
-
-  chart.interpolate = function(_) {
-    if (!arguments.length) return interpolate;
-    interpolate = _;
-    return chart;
-  };
-
-  chart.defined = function(_) {
-    if (!arguments.length) return defined;
-    defined = _;
-    return chart;
-  };
-
-  chart.isArea = function(_) {
-    if (!arguments.length) return isArea;
-    isArea = d3.functor(_);
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.lineChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var lines = nv.models.line()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , legend = nv.models.legend()
-    , interactiveLayer = nv.interactiveGuideline()
-    ;
-
-  var margin = {top: 30, right: 20, bottom: 50, left: 60}
-    , color = nv.utils.defaultColor()
-    , width = null
-    , height = null
-    , showLegend = true
-    , showXAxis = true
-    , showYAxis = true
-    , rightAlignYAxis = false
-    , useInteractiveGuideline = false
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>'
-      }
-    , x
-    , y
-    , state = {}
-    , defaultState = null
-    , noData = 'No Data Available.'
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , transitionDuration = 250
-    ;
-
-  xAxis
-    .orient('bottom')
-    .tickPadding(7)
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(lines.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(lines.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, null, null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart) };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-    
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display noData message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = lines.xScale();
-      y = lines.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-lineChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-lineChart').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append("rect").style("opacity",0);
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-linesWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-interactive');
-
-      g.select("rect").attr("width",availableWidth).attr("height",availableHeight);
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width(availableWidth);
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        wrap.select('.nv-legendWrap')
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-      }
-
-      //------------------------------------------------------------
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      
-      //------------------------------------------------------------
-      //Set up interactive layer
-      if (useInteractiveGuideline) {
-        interactiveLayer
-           .width(availableWidth)
-           .height(availableHeight)
-           .margin({left:margin.left, top:margin.top})
-           .svgContainer(container)
-           .xScale(x);
-        wrap.select(".nv-interactive").call(interactiveLayer);
-      }
-
-
-      lines
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled }));
-
-
-      var linesWrap = g.select('.nv-linesWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      linesWrap.transition().call(lines);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-        xAxis
-          .scale(x)
-          .ticks( availableWidth / 100 )
-          .tickSize(-availableHeight, 0);
-
-        g.select('.nv-x.nv-axis')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')');
-        g.select('.nv-x.nv-axis')
-            .transition()
-            .call(xAxis);
-      }
-
-      if (showYAxis) {
-        yAxis
-          .scale(y)
-          .ticks( availableHeight / 36 )
-          .tickSize( -availableWidth, 0);
-
-        g.select('.nv-y.nv-axis')
-            .transition()
-            .call(yAxis);
-      }
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) {
-          state = newState;
-          dispatch.stateChange(state);
-          chart.update();
-      });
-
-      interactiveLayer.dispatch.on('elementMousemove', function(e) {
-          lines.clearHighlights();
-          var singlePoint, pointIndex, pointXLocation, allData = [];
-          data
-          .filter(function(series, i) { 
-            series.seriesIndex = i;
-            return !series.disabled; 
-          })
-          .forEach(function(series,i) {
-              pointIndex = nv.interactiveBisect(series.values, e.pointXValue, chart.x());
-              lines.highlightPoint(i, pointIndex, true);
-              var point = series.values[pointIndex];
-              if (typeof point === 'undefined') return;
-              if (typeof singlePoint === 'undefined') singlePoint = point;
-              if (typeof pointXLocation === 'undefined') pointXLocation = chart.xScale()(chart.x()(point,pointIndex));
-              allData.push({
-                  key: series.key,
-                  value: chart.y()(point, pointIndex),
-                  color: color(series,series.seriesIndex)
-              });
-          });
-          //Highlight the tooltip entry based on which point the mouse is closest to.
-          if (allData.length > 2) {
-            var yValue = chart.yScale().invert(e.mouseY);
-            var yDistMax = Infinity, indexToHighlight = null;
-            allData.forEach(function(series,i) {
-               var delta = Math.abs(yValue - series.value);
-               if ( delta < yDistMax) {
-                  yDistMax = delta;
-                  indexToHighlight = i;
-               }
-            });
-            allData[indexToHighlight].highlight = true;
-          }
-
-          var xValue = xAxis.tickFormat()(chart.x()(singlePoint,pointIndex));
-          interactiveLayer.tooltip
-                  .position({left: pointXLocation + margin.left, top: e.mouseY + margin.top})
-                  .chartContainer(that.parentNode)
-                  .enabled(tooltips)
-                  .valueFormatter(function(d,i) {
-                     return yAxis.tickFormat()(d);
-                  })
-                  .data(
-                      {
-                        value: xValue,
-                        series: allData
-                      }
-                  )();
-
-          interactiveLayer.renderGuideLine(pointXLocation);
-
-      });
-
-      interactiveLayer.dispatch.on("elementMouseout",function(e) {
-          dispatch.tooltipHide();
-          lines.clearHighlights();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  lines.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.lines = lines;
-  chart.legend = legend;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-  chart.interactiveLayer = interactiveLayer;
-
-  d3.rebind(chart, lines, 'defined', 'isArea', 'x', 'y', 'size', 'xScale', 'yScale', 'xDomain', 'yDomain', 'xRange', 'yRange'
-    , 'forceX', 'forceY', 'interactive', 'clipEdge', 'clipVoronoi', 'useVoronoi','id', 'interpolate');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.useInteractiveGuideline = function(_) {
-    if(!arguments.length) return useInteractiveGuideline;
-    useInteractiveGuideline = _;
-    if (_ === true) {
-       chart.interactive(false);
-       chart.useVoronoi(false);
-    }
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.linePlusBarChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var lines = nv.models.line()
-    , bars = nv.models.historicalBar()
-    , xAxis = nv.models.axis()
-    , y1Axis = nv.models.axis()
-    , y2Axis = nv.models.axis()
-    , legend = nv.models.legend()
-    ;
-
-  var margin = {top: 30, right: 60, bottom: 50, left: 60}
-    , width = null
-    , height = null
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , color = nv.utils.defaultColor()
-    , showLegend = true
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>';
-      }
-    , x
-    , y1
-    , y2
-    , state = {}
-    , defaultState = null
-    , noData = "No Data Available."
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    ;
-
-  bars
-    .padData(true)
-    ;
-  lines
-    .clipEdge(false)
-    .padData(true)
-    ;
-  xAxis
-    .orient('bottom')
-    .tickPadding(7)
-    .highlightZero(false)
-    ;
-  y1Axis
-    .orient('left')
-    ;
-  y2Axis
-    .orient('right')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-      var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-          top = e.pos[1] + ( offsetElement.offsetTop || 0),
-          x = xAxis.tickFormat()(lines.x()(e.point, e.pointIndex)),
-          y = (e.series.bar ? y1Axis : y2Axis).tickFormat()(lines.y()(e.point, e.pointIndex)),
-          content = tooltip(e.series.key, x, y, e, chart);
-
-      nv.tooltip.show([left, top], content, e.value < 0 ? 'n' : 's', null, offsetElement);
-    }
-    ;
-
-  //------------------------------------------------------------
-
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().call(chart); };
-      // chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      var dataBars = data.filter(function(d) { return !d.disabled && d.bar });
-      var dataLines = data.filter(function(d) { return !d.bar }); // removed the !d.disabled clause here to fix Issue #240
-
-      //x = xAxis.scale();
-       x = dataLines.filter(function(d) { return !d.disabled; }).length && dataLines.filter(function(d) { return !d.disabled; })[0].values.length ? lines.xScale() : bars.xScale();
-      //x = dataLines.filter(function(d) { return !d.disabled; }).length ? lines.xScale() : bars.xScale(); //old code before change above
-      y1 = bars.yScale();
-      y2 = lines.yScale();
-
-      //------------------------------------------------------------
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = d3.select(this).selectAll('g.nv-wrap.nv-linePlusBar').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-linePlusBar').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y1 nv-axis');
-      gEnter.append('g').attr('class', 'nv-y2 nv-axis');
-      gEnter.append('g').attr('class', 'nv-barsWrap');
-      gEnter.append('g').attr('class', 'nv-linesWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width( availableWidth / 2 );
-
-        g.select('.nv-legendWrap')
-            .datum(data.map(function(series) {
-              series.originalKey = series.originalKey === undefined ? series.key : series.originalKey;
-              series.key = series.originalKey + (series.bar ? ' (left axis)' : ' (right axis)');
-              return series;
-            }))
-          .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + ( availableWidth / 2 ) + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-
-      lines
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && !data[i].bar }))
-
-      bars
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && data[i].bar }))
-
-
-
-      var barsWrap = g.select('.nv-barsWrap')
-          .datum(dataBars.length ? dataBars : [{values:[]}])
-
-      var linesWrap = g.select('.nv-linesWrap')
-          .datum(dataLines[0] && !dataLines[0].disabled ? dataLines : [{values:[]}] );
-          //.datum(!dataLines[0].disabled ? dataLines : [{values:dataLines[0].values.map(function(d) { return [d[0], null] }) }] );
-
-      d3.transition(barsWrap).call(bars);
-      d3.transition(linesWrap).call(lines);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      xAxis
-        .scale(x)
-        .ticks( availableWidth / 100 )
-        .tickSize(-availableHeight, 0);
-
-      g.select('.nv-x.nv-axis')
-          .attr('transform', 'translate(0,' + y1.range()[0] + ')');
-      d3.transition(g.select('.nv-x.nv-axis'))
-          .call(xAxis);
-
-
-      y1Axis
-        .scale(y1)
-        .ticks( availableHeight / 36 )
-        .tickSize(-availableWidth, 0);
-
-      d3.transition(g.select('.nv-y1.nv-axis'))
-          .style('opacity', dataBars.length ? 1 : 0)
-          .call(y1Axis);
-
-
-      y2Axis
-        .scale(y2)
-        .ticks( availableHeight / 36 )
-        .tickSize(dataBars.length ? 0 : -availableWidth, 0); // Show the y2 rules only if y1 has none
-
-      g.select('.nv-y2.nv-axis')
-          .style('opacity', dataLines.length ? 1 : 0)
-          .attr('transform', 'translate(' + availableWidth + ',0)');
-          //.attr('transform', 'translate(' + x.range()[1] + ',0)');
-
-      d3.transition(g.select('.nv-y2.nv-axis'))
-          .call(y2Axis);
-
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        state = newState;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  lines.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  bars.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  bars.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.legend = legend;
-  chart.lines = lines;
-  chart.bars = bars;
-  chart.xAxis = xAxis;
-  chart.y1Axis = y1Axis;
-  chart.y2Axis = y2Axis;
-
-  d3.rebind(chart, lines, 'defined', 'size', 'clipVoronoi', 'interpolate');
-  //TODO: consider rebinding x, y and some other stuff, and simply do soemthign lile bars.x(lines.x()), etc.
-  //d3.rebind(chart, lines, 'x', 'y', 'size', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'interactive', 'clipEdge', 'clipVoronoi', 'id');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    lines.x(_);
-    bars.x(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    lines.y(_);
-    bars.y(_);
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-nv.models.lineWithFocusChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var lines = nv.models.line()
-    , lines2 = nv.models.line()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , x2Axis = nv.models.axis()
-    , y2Axis = nv.models.axis()
-    , legend = nv.models.legend()
-    , brush = d3.svg.brush()
-    ;
-
-  var margin = {top: 30, right: 30, bottom: 30, left: 60}
-    , margin2 = {top: 0, right: 30, bottom: 20, left: 60}
-    , color = nv.utils.defaultColor()
-    , width = null
-    , height = null
-    , height2 = 100
-    , x
-    , y
-    , x2
-    , y2
-    , showLegend = true
-    , brushExtent = null
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>'
-      }
-    , noData = "No Data Available."
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'brush')
-    , transitionDuration = 250
-    ;
-
-  lines
-    .clipEdge(true)
-    ;
-  lines2
-    .interactive(false)
-    ;
-  xAxis
-    .orient('bottom')
-    .tickPadding(5)
-    ;
-  yAxis
-    .orient('left')
-    ;
-  x2Axis
-    .orient('bottom')
-    .tickPadding(5)
-    ;
-  y2Axis
-    .orient('left')
-    ;
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(lines.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(lines.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, null, null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight1 = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom - height2,
-          availableHeight2 = height2 - margin2.top - margin2.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart) };
-      chart.container = this;
-
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight1 / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = lines.xScale();
-      y = lines.yScale();
-      x2 = lines2.xScale();
-      y2 = lines2.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-lineWithFocusChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-lineWithFocusChart').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-
-      var focusEnter = gEnter.append('g').attr('class', 'nv-focus');
-      focusEnter.append('g').attr('class', 'nv-x nv-axis');
-      focusEnter.append('g').attr('class', 'nv-y nv-axis');
-      focusEnter.append('g').attr('class', 'nv-linesWrap');
-
-      var contextEnter = gEnter.append('g').attr('class', 'nv-context');
-      contextEnter.append('g').attr('class', 'nv-x nv-axis');
-      contextEnter.append('g').attr('class', 'nv-y nv-axis');
-      contextEnter.append('g').attr('class', 'nv-linesWrap');
-      contextEnter.append('g').attr('class', 'nv-brushBackground');
-      contextEnter.append('g').attr('class', 'nv-x nv-brush');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width(availableWidth);
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight1 = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom - height2;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      lines
-        .width(availableWidth)
-        .height(availableHeight1)
-        .color(
-          data
-            .map(function(d,i) {
-              return d.color || color(d, i);
-            })
-            .filter(function(d,i) {
-              return !data[i].disabled;
-          })
-        );
-
-      lines2
-        .defined(lines.defined())
-        .width(availableWidth)
-        .height(availableHeight2)
-        .color(
-          data
-            .map(function(d,i) {
-              return d.color || color(d, i);
-            })
-            .filter(function(d,i) {
-              return !data[i].disabled;
-          })
-        );
-
-      g.select('.nv-context')
-          .attr('transform', 'translate(0,' + ( availableHeight1 + margin.bottom + margin2.top) + ')')
-
-      var contextLinesWrap = g.select('.nv-context .nv-linesWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      d3.transition(contextLinesWrap).call(lines2);
-
-      //------------------------------------------------------------
-
-
-      /*
-      var focusLinesWrap = g.select('.nv-focus .nv-linesWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      d3.transition(focusLinesWrap).call(lines);
-     */
-
-
-      //------------------------------------------------------------
-      // Setup Main (Focus) Axes
-
-      xAxis
-        .scale(x)
-        .ticks( availableWidth / 100 )
-        .tickSize(-availableHeight1, 0);
-
-      yAxis
-        .scale(y)
-        .ticks( availableHeight1 / 36 )
-        .tickSize( -availableWidth, 0);
-
-      g.select('.nv-focus .nv-x.nv-axis')
-          .attr('transform', 'translate(0,' + availableHeight1 + ')');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Brush
-
-      brush
-        .x(x2)
-        .on('brush', function() {
-            //When brushing, turn off transitions because chart needs to change immediately.
-            var oldTransition = chart.transitionDuration();
-            chart.transitionDuration(0); 
-            onBrush();
-            chart.transitionDuration(oldTransition);
-        });
-
-      if (brushExtent) brush.extent(brushExtent);
-
-      var brushBG = g.select('.nv-brushBackground').selectAll('g')
-          .data([brushExtent || brush.extent()])
-
-      var brushBGenter = brushBG.enter()
-          .append('g');
-
-      brushBGenter.append('rect')
-          .attr('class', 'left')
-          .attr('x', 0)
-          .attr('y', 0)
-          .attr('height', availableHeight2);
-
-      brushBGenter.append('rect')
-          .attr('class', 'right')
-          .attr('x', 0)
-          .attr('y', 0)
-          .attr('height', availableHeight2);
-
-      var gBrush = g.select('.nv-x.nv-brush')
-          .call(brush);
-      gBrush.selectAll('rect')
-          //.attr('y', -5)
-          .attr('height', availableHeight2);
-      gBrush.selectAll('.resize').append('path').attr('d', resizePath);
-
-      onBrush();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Secondary (Context) Axes
-
-      x2Axis
-        .scale(x2)
-        .ticks( availableWidth / 100 )
-        .tickSize(-availableHeight2, 0);
-
-      g.select('.nv-context .nv-x.nv-axis')
-          .attr('transform', 'translate(0,' + y2.range()[0] + ')');
-      d3.transition(g.select('.nv-context .nv-x.nv-axis'))
-          .call(x2Axis);
-
-
-      y2Axis
-        .scale(y2)
-        .ticks( availableHeight2 / 36 )
-        .tickSize( -availableWidth, 0);
-
-      d3.transition(g.select('.nv-context .nv-y.nv-axis'))
-          .call(y2Axis);
-
-      g.select('.nv-context .nv-x.nv-axis')
-          .attr('transform', 'translate(0,' + y2.range()[0] + ')');
-
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        chart.update();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      //============================================================
-
-
-      //============================================================
-      // Functions
-      //------------------------------------------------------------
-
-      // Taken from crossfilter (http://square.github.com/crossfilter/)
-      function resizePath(d) {
-        var e = +(d == 'e'),
-            x = e ? 1 : -1,
-            y = availableHeight2 / 3;
-        return 'M' + (.5 * x) + ',' + y
-            + 'A6,6 0 0 ' + e + ' ' + (6.5 * x) + ',' + (y + 6)
-            + 'V' + (2 * y - 6)
-            + 'A6,6 0 0 ' + e + ' ' + (.5 * x) + ',' + (2 * y)
-            + 'Z'
-            + 'M' + (2.5 * x) + ',' + (y + 8)
-            + 'V' + (2 * y - 8)
-            + 'M' + (4.5 * x) + ',' + (y + 8)
-            + 'V' + (2 * y - 8);
-      }
-
-
-      function updateBrushBG() {
-        if (!brush.empty()) brush.extent(brushExtent);
-        brushBG
-            .data([brush.empty() ? x2.domain() : brushExtent])
-            .each(function(d,i) {
-              var leftWidth = x2(d[0]) - x.range()[0],
-                  rightWidth = x.range()[1] - x2(d[1]);
-              d3.select(this).select('.left')
-                .attr('width',  leftWidth < 0 ? 0 : leftWidth);
-
-              d3.select(this).select('.right')
-                .attr('x', x2(d[1]))
-                .attr('width', rightWidth < 0 ? 0 : rightWidth);
-            });
-      }
-
-
-      function onBrush() {
-        brushExtent = brush.empty() ? null : brush.extent();
-        var extent = brush.empty() ? x2.domain() : brush.extent();
-
-        //The brush extent cannot be less than one.  If it is, don't update the line chart.
-        if (Math.abs(extent[0] - extent[1]) <= 1) {
-          return;
-        }
-
-        dispatch.brush({extent: extent, brush: brush});
-
-
-        updateBrushBG();
-
-        // Update Main (Focus)
-        var focusLinesWrap = g.select('.nv-focus .nv-linesWrap')
-            .datum(
-              data
-                .filter(function(d) { return !d.disabled })
-                .map(function(d,i) {
-                  return {
-                    key: d.key,
-                    values: d.values.filter(function(d,i) {
-                      return lines.x()(d,i) >= extent[0] && lines.x()(d,i) <= extent[1];
-                    })
-                  }
-                })
-            );
-        focusLinesWrap.transition().duration(transitionDuration).call(lines);
-
-
-        // Update Main (Focus) Axes
-        g.select('.nv-focus .nv-x.nv-axis').transition().duration(transitionDuration)
-            .call(xAxis);
-        g.select('.nv-focus .nv-y.nv-axis').transition().duration(transitionDuration)
-            .call(yAxis);
-      }
-
-      //============================================================
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  lines.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.legend = legend;
-  chart.lines = lines;
-  chart.lines2 = lines2;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-  chart.x2Axis = x2Axis;
-  chart.y2Axis = y2Axis;
-
-  d3.rebind(chart, lines, 'defined', 'isArea', 'size', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'interactive', 'clipEdge', 'clipVoronoi', 'id');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return lines.x;
-    lines.x(_);
-    lines2.x(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return lines.y;
-    lines.y(_);
-    lines2.y(_);
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.margin2 = function(_) {
-    if (!arguments.length) return margin2;
-    margin2 = _;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.height2 = function(_) {
-    if (!arguments.length) return height2;
-    height2 = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color =nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.interpolate = function(_) {
-    if (!arguments.length) return lines.interpolate();
-    lines.interpolate(_);
-    lines2.interpolate(_);
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  // Chart has multiple similar Axes, to prevent code duplication, probably need to link all axis functions manually like below
-  chart.xTickFormat = function(_) {
-    if (!arguments.length) return xAxis.tickFormat();
-    xAxis.tickFormat(_);
-    x2Axis.tickFormat(_);
-    return chart;
-  };
-
-  chart.yTickFormat = function(_) {
-    if (!arguments.length) return yAxis.tickFormat();
-    yAxis.tickFormat(_);
-    y2Axis.tickFormat(_);
-    return chart;
-  };
-  
-  chart.brushExtent = function(_) {
-    if (!arguments.length) return brushExtent;
-    brushExtent = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.linePlusBarWithFocusChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var lines = nv.models.line()
-    , lines2 = nv.models.line()
-    , bars = nv.models.historicalBar()
-    , bars2 = nv.models.historicalBar()
-    , xAxis = nv.models.axis()
-    , x2Axis = nv.models.axis()
-    , y1Axis = nv.models.axis()
-    , y2Axis = nv.models.axis()
-    , y3Axis = nv.models.axis()
-    , y4Axis = nv.models.axis()
-    , legend = nv.models.legend()
-    , brush = d3.svg.brush()
-    ;
-
-  var margin = {top: 30, right: 30, bottom: 30, left: 60}
-    , margin2 = {top: 0, right: 30, bottom: 20, left: 60}
-    , width = null
-    , height = null
-    , height2 = 100
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , color = nv.utils.defaultColor()
-    , showLegend = true
-    , extent
-    , brushExtent = null
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>';
-      }
-    , x
-    , x2
-    , y1
-    , y2
-    , y3
-    , y4
-    , noData = "No Data Available."
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'brush')
-    , transitionDuration = 0
-    ;
-
-  lines
-    .clipEdge(true)
-    ;
-  lines2
-    .interactive(false)
-    ;
-  xAxis
-    .orient('bottom')
-    .tickPadding(5)
-    ;
-  y1Axis
-    .orient('left')
-    ;
-  y2Axis
-    .orient('right')
-    ;
-  x2Axis
-    .orient('bottom')
-    .tickPadding(5)
-    ;
-  y3Axis
-    .orient('left')
-    ;
-  y4Axis
-    .orient('right')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    if (extent) {
-        e.pointIndex += Math.ceil(extent[0]);
-    }
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(lines.x()(e.point, e.pointIndex)),
-        y = (e.series.bar ? y1Axis : y2Axis).tickFormat()(lines.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  //------------------------------------------------------------
-
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight1 = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom - height2,
-          availableHeight2 = height2 - margin2.top - margin2.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart); };
-      chart.container = this;
-
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight1 / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      var dataBars = data.filter(function(d) { return !d.disabled && d.bar });
-      var dataLines = data.filter(function(d) { return !d.bar }); // removed the !d.disabled clause here to fix Issue #240
-
-      x = bars.xScale();
-      x2 = x2Axis.scale();
-      y1 = bars.yScale();
-      y2 = lines.yScale();
-      y3 = bars2.yScale();
-      y4 = lines2.yScale();
-
-      var series1 = data
-        .filter(function(d) { return !d.disabled && d.bar })
-        .map(function(d) {
-          return d.values.map(function(d,i) {
-            return { x: getX(d,i), y: getY(d,i) }
-          })
-        });
-
-      var series2 = data
-        .filter(function(d) { return !d.disabled && !d.bar })
-        .map(function(d) {
-          return d.values.map(function(d,i) {
-            return { x: getX(d,i), y: getY(d,i) }
-          })
-        });
-
-      x   .range([0, availableWidth]);
-      
-      x2  .domain(d3.extent(d3.merge(series1.concat(series2)), function(d) { return d.x } ))
-          .range([0, availableWidth]);
-
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-linePlusBar').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-linePlusBar').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      
-      var focusEnter = gEnter.append('g').attr('class', 'nv-focus');
-      focusEnter.append('g').attr('class', 'nv-x nv-axis');
-      focusEnter.append('g').attr('class', 'nv-y1 nv-axis');
-      focusEnter.append('g').attr('class', 'nv-y2 nv-axis');
-      focusEnter.append('g').attr('class', 'nv-barsWrap');
-      focusEnter.append('g').attr('class', 'nv-linesWrap');
-
-      var contextEnter = gEnter.append('g').attr('class', 'nv-context');
-      contextEnter.append('g').attr('class', 'nv-x nv-axis');
-      contextEnter.append('g').attr('class', 'nv-y1 nv-axis');
-      contextEnter.append('g').attr('class', 'nv-y2 nv-axis');
-      contextEnter.append('g').attr('class', 'nv-barsWrap');
-      contextEnter.append('g').attr('class', 'nv-linesWrap');
-      contextEnter.append('g').attr('class', 'nv-brushBackground');
-      contextEnter.append('g').attr('class', 'nv-x nv-brush');
-
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width( availableWidth / 2 );
-
-        g.select('.nv-legendWrap')
-            .datum(data.map(function(series) {
-              series.originalKey = series.originalKey === undefined ? series.key : series.originalKey;
-              series.key = series.originalKey + (series.bar ? ' (left axis)' : ' (right axis)');
-              return series;
-            }))
-          .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight1 = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom - height2;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + ( availableWidth / 2 ) + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-
-      //------------------------------------------------------------
-      // Context Components
-
-      bars2
-        .width(availableWidth)
-        .height(availableHeight2)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && data[i].bar }));
-
-      lines2
-        .width(availableWidth)
-        .height(availableHeight2)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && !data[i].bar }));
-        
-      var bars2Wrap = g.select('.nv-context .nv-barsWrap')
-          .datum(dataBars.length ? dataBars : [{values:[]}]);
-
-      var lines2Wrap = g.select('.nv-context .nv-linesWrap')
-          .datum(!dataLines[0].disabled ? dataLines : [{values:[]}]);
-          
-      g.select('.nv-context')
-          .attr('transform', 'translate(0,' + ( availableHeight1 + margin.bottom + margin2.top) + ')')
-
-      bars2Wrap.transition().call(bars2);
-      lines2Wrap.transition().call(lines2);
-
-      //------------------------------------------------------------
-
-
-
-      //------------------------------------------------------------
-      // Setup Brush
-
-      brush
-        .x(x2)
-        .on('brush', onBrush);
-
-      if (brushExtent) brush.extent(brushExtent);
-
-      var brushBG = g.select('.nv-brushBackground').selectAll('g')
-          .data([brushExtent || brush.extent()])
-
-      var brushBGenter = brushBG.enter()
-          .append('g');
-
-      brushBGenter.append('rect')
-          .attr('class', 'left')
-          .attr('x', 0)
-          .attr('y', 0)
-          .attr('height', availableHeight2);
-
-      brushBGenter.append('rect')
-          .attr('class', 'right')
-          .attr('x', 0)
-          .attr('y', 0)
-          .attr('height', availableHeight2);
-
-      var gBrush = g.select('.nv-x.nv-brush')
-          .call(brush);
-      gBrush.selectAll('rect')
-          //.attr('y', -5)
-          .attr('height', availableHeight2);
-      gBrush.selectAll('.resize').append('path').attr('d', resizePath);
-
-      //------------------------------------------------------------
-
-      //------------------------------------------------------------
-      // Setup Secondary (Context) Axes
-
-      x2Axis
-        .ticks( availableWidth / 100 )
-        .tickSize(-availableHeight2, 0);
-
-      g.select('.nv-context .nv-x.nv-axis')
-          .attr('transform', 'translate(0,' + y3.range()[0] + ')');
-      g.select('.nv-context .nv-x.nv-axis').transition()
-          .call(x2Axis);
-
-
-      y3Axis
-        .scale(y3)
-        .ticks( availableHeight2 / 36 )
-        .tickSize( -availableWidth, 0);
-
-      g.select('.nv-context .nv-y1.nv-axis')
-          .style('opacity', dataBars.length ? 1 : 0)
-          .attr('transform', 'translate(0,' + x2.range()[0] + ')');
-          
-      g.select('.nv-context .nv-y1.nv-axis').transition()
-          .call(y3Axis);
-          
-
-      y4Axis
-        .scale(y4)
-        .ticks( availableHeight2 / 36 )
-        .tickSize(dataBars.length ? 0 : -availableWidth, 0); // Show the y2 rules only if y1 has none
-
-      g.select('.nv-context .nv-y2.nv-axis')
-          .style('opacity', dataLines.length ? 1 : 0)
-          .attr('transform', 'translate(' + x2.range()[1] + ',0)');
-
-      g.select('.nv-context .nv-y2.nv-axis').transition()
-          .call(y4Axis);
-          
-      //------------------------------------------------------------
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        chart.update();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      //============================================================
-
-
-      //============================================================
-      // Functions
-      //------------------------------------------------------------
-
-      // Taken from crossfilter (http://square.github.com/crossfilter/)
-      function resizePath(d) {
-        var e = +(d == 'e'),
-            x = e ? 1 : -1,
-            y = availableHeight2 / 3;
-        return 'M' + (.5 * x) + ',' + y
-            + 'A6,6 0 0 ' + e + ' ' + (6.5 * x) + ',' + (y + 6)
-            + 'V' + (2 * y - 6)
-            + 'A6,6 0 0 ' + e + ' ' + (.5 * x) + ',' + (2 * y)
-            + 'Z'
-            + 'M' + (2.5 * x) + ',' + (y + 8)
-            + 'V' + (2 * y - 8)
-            + 'M' + (4.5 * x) + ',' + (y + 8)
-            + 'V' + (2 * y - 8);
-      }
-
-
-      function updateBrushBG() {
-        if (!brush.empty()) brush.extent(brushExtent);
-        brushBG
-            .data([brush.empty() ? x2.domain() : brushExtent])
-            .each(function(d,i) {
-              var leftWidth = x2(d[0]) - x2.range()[0],
-                  rightWidth = x2.range()[1] - x2(d[1]);
-              d3.select(this).select('.left')
-                .attr('width',  leftWidth < 0 ? 0 : leftWidth);
-
-              d3.select(this).select('.right')
-                .attr('x', x2(d[1]))
-                .attr('width', rightWidth < 0 ? 0 : rightWidth);
-            });
-      }
-
-
-      function onBrush() {
-        brushExtent = brush.empty() ? null : brush.extent();
-        extent = brush.empty() ? x2.domain() : brush.extent();
-
-
-        dispatch.brush({extent: extent, brush: brush});
-
-        updateBrushBG();
-
-
-        //------------------------------------------------------------
-        // Prepare Main (Focus) Bars and Lines
-        
-        bars
-        .width(availableWidth)
-        .height(availableHeight1)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && data[i].bar }));
-
-
-        lines
-        .width(availableWidth)
-        .height(availableHeight1)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled && !data[i].bar }));
-
-        var focusBarsWrap = g.select('.nv-focus .nv-barsWrap')
-            .datum(!dataBars.length ? [{values:[]}] :
-              dataBars
-                .map(function(d,i) {
-                  return {
-                    key: d.key,
-                    values: d.values.filter(function(d,i) {
-                      return bars.x()(d,i) >= extent[0] && bars.x()(d,i) <= extent[1];
-                    })
-                  }
-                })
-            );
-        
-        var focusLinesWrap = g.select('.nv-focus .nv-linesWrap')
-            .datum(dataLines[0].disabled ? [{values:[]}] :
-              dataLines
-                .map(function(d,i) {
-                  return {
-                    key: d.key,
-                    values: d.values.filter(function(d,i) {
-                      return lines.x()(d,i) >= extent[0] && lines.x()(d,i) <= extent[1];
-                    })
-                  }
-                })
-             );
-                 
-        //------------------------------------------------------------
-        
-        
-        //------------------------------------------------------------
-        // Update Main (Focus) X Axis
-
-        if (dataBars.length) {
-            x = bars.xScale();
-        } else {
-            x = lines.xScale();
-        }
-        
-        xAxis
-        .scale(x)
-        .ticks( availableWidth / 100 )
-        .tickSize(-availableHeight1, 0);
-
-        xAxis.domain([Math.ceil(extent[0]), Math.floor(extent[1])]);
-        
-        g.select('.nv-x.nv-axis').transition().duration(transitionDuration)
-          .call(xAxis);
-        //------------------------------------------------------------
-        
-        
-        //------------------------------------------------------------
-        // Update Main (Focus) Bars and Lines
-
-        focusBarsWrap.transition().duration(transitionDuration).call(bars);
-        focusLinesWrap.transition().duration(transitionDuration).call(lines);
-        
-        //------------------------------------------------------------
-        
-          
-        //------------------------------------------------------------
-        // Setup and Update Main (Focus) Y Axes
-        
-        g.select('.nv-focus .nv-x.nv-axis')
-          .attr('transform', 'translate(0,' + y1.range()[0] + ')');
-
-
-        y1Axis
-        .scale(y1)
-        .ticks( availableHeight1 / 36 )
-        .tickSize(-availableWidth, 0);
-
-        g.select('.nv-focus .nv-y1.nv-axis')
-          .style('opacity', dataBars.length ? 1 : 0);
-
-
-        y2Axis
-        .scale(y2)
-        .ticks( availableHeight1 / 36 )
-        .tickSize(dataBars.length ? 0 : -availableWidth, 0); // Show the y2 rules only if y1 has none
-
-        g.select('.nv-focus .nv-y2.nv-axis')
-          .style('opacity', dataLines.length ? 1 : 0)
-          .attr('transform', 'translate(' + x.range()[1] + ',0)');
-
-        g.select('.nv-focus .nv-y1.nv-axis').transition().duration(transitionDuration)
-            .call(y1Axis);
-        g.select('.nv-focus .nv-y2.nv-axis').transition().duration(transitionDuration)
-            .call(y2Axis);
-      }
-
-      //============================================================
-
-      onBrush();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  lines.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  bars.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  bars.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.legend = legend;
-  chart.lines = lines;
-  chart.lines2 = lines2;
-  chart.bars = bars;
-  chart.bars2 = bars2;
-  chart.xAxis = xAxis;
-  chart.x2Axis = x2Axis;
-  chart.y1Axis = y1Axis;
-  chart.y2Axis = y2Axis;
-  chart.y3Axis = y3Axis;
-  chart.y4Axis = y4Axis;
-
-  d3.rebind(chart, lines, 'defined', 'size', 'clipVoronoi', 'interpolate');
-  //TODO: consider rebinding x, y and some other stuff, and simply do soemthign lile bars.x(lines.x()), etc.
-  //d3.rebind(chart, lines, 'x', 'y', 'size', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'interactive', 'clipEdge', 'clipVoronoi', 'id');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    lines.x(_);
-    bars.x(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    lines.y(_);
-    bars.y(_);
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.brushExtent = function(_) {
-    if (!arguments.length) return brushExtent;
-    brushExtent = _;
-    return chart;
-  };
-
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.multiBar = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , x = d3.scale.ordinal()
-    , y = d3.scale.linear()
-    , id = Math.floor(Math.random() * 10000) //Create semi-unique ID in case user doesn't select one
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , forceY = [0] // 0 is forced by default.. this makes sense for the majority of bar graphs... user can always do chart.forceY([]) to remove
-    , clipEdge = true
-    , stacked = false
-    , color = nv.utils.defaultColor()
-    , hideable = false
-    , barColor = null // adding the ability to set the color for each rather than the whole group
-    , disabled // used in conjunction with barColor to communicate from multiBarHorizontalChart what series are disabled
-    , delay = 1200
-    , xDomain
-    , yDomain
-    , xRange
-    , yRange
-    , groupSpacing = 0.1
-    , dispatch = d3.dispatch('chartClick', 'elementClick', 'elementDblClick', 'elementMouseover', 'elementMouseout')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0 //used to store previous scales
-      ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-      if(hideable && data.length) hideable = [{
-        values: data[0].values.map(function(d) {
-        return {
-          x: d.x,
-          y: 0,
-          series: d.series,
-          size: 0.01
-        };}
-      )}];
-
-      if (stacked)
-        data = d3.layout.stack()
-                 .offset('zero')
-                 .values(function(d){ return d.values })
-                 .y(getY)
-                 (!data.length && hideable ? hideable : data);
-
-
-      //add series index to each data point for reference
-      data = data.map(function(series, i) {
-        series.values = series.values.map(function(point) {
-          point.series = i;
-          return point;
-        });
-        return series;
-      });
-
-
-      //------------------------------------------------------------
-      // HACK for negative value stacking
-      if (stacked)
-        data[0].values.map(function(d,i) {
-          var posBase = 0, negBase = 0;
-          data.map(function(d) {
-            var f = d.values[i]
-            f.size = Math.abs(f.y);
-            if (f.y<0)  {
-              f.y1 = negBase;
-              negBase = negBase - f.size;
-            } else
-            { 
-              f.y1 = f.size + posBase;
-              posBase = posBase + f.size;
-            }
-          });
-        });
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      // remap and flatten the data for use in calculating the scales' domains
-      var seriesData = (xDomain && yDomain) ? [] : // if we know xDomain and yDomain, no need to calculate
-            data.map(function(d) {
-              return d.values.map(function(d,i) {
-                return { x: getX(d,i), y: getY(d,i), y0: d.y0, y1: d.y1 }
-              })
-            });
-
-      x   .domain(xDomain || d3.merge(seriesData).map(function(d) { return d.x }))
-          .rangeBands(xRange || [0, availableWidth], groupSpacing);
-
-      //y   .domain(yDomain || d3.extent(d3.merge(seriesData).map(function(d) { return d.y + (stacked ? d.y1 : 0) }).concat(forceY)))
-      y   .domain(yDomain || d3.extent(d3.merge(seriesData).map(function(d) { return stacked ? (d.y > 0 ? d.y1 : d.y1 + d.y ) : d.y }).concat(forceY)))
-          .range(yRange || [availableHeight, 0]);
-
-      // If scale's domain don't have a range, slightly adjust to make one... so a chart can show a single data point
-      if (x.domain()[0] === x.domain()[1])
-        x.domain()[0] ?
-            x.domain([x.domain()[0] - x.domain()[0] * 0.01, x.domain()[1] + x.domain()[1] * 0.01])
-          : x.domain([-1,1]);
-
-      if (y.domain()[0] === y.domain()[1])
-        y.domain()[0] ?
-            y.domain([y.domain()[0] + y.domain()[0] * 0.01, y.domain()[1] - y.domain()[1] * 0.01])
-          : y.domain([-1,1]);
-
-
-      x0 = x0 || x;
-      y0 = y0 || y;
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-multibar').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-multibar');
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g')
-
-      gEnter.append('g').attr('class', 'nv-groups');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-edge-clip-' + id)
-        .append('rect');
-      wrap.select('#nv-edge-clip-' + id + ' rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      g   .attr('clip-path', clipEdge ? 'url(#nv-edge-clip-' + id + ')' : '');
-
-
-
-      var groups = wrap.select('.nv-groups').selectAll('.nv-group')
-          .data(function(d) { return d }, function(d,i) { return i });
-      groups.enter().append('g')
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6);
-      groups.exit()
-        .transition()
-        .selectAll('rect.nv-bar')
-        .delay(function(d,i) { 
-             return i * delay/ data[0].values.length;
-        })
-          .attr('y', function(d) { return stacked ? y0(d.y0) : y0(0) })
-          .attr('height', 0)
-          .remove();
-      groups
-          .attr('class', function(d,i) { return 'nv-group nv-series-' + i })
-          .classed('hover', function(d) { return d.hover })
-          .style('fill', function(d,i){ return color(d, i) })
-          .style('stroke', function(d,i){ return color(d, i) });
-      groups
-          .transition()
-          .style('stroke-opacity', 1)
-          .style('fill-opacity', .75);
-
-
-      var bars = groups.selectAll('rect.nv-bar')
-          .data(function(d) { return (hideable && !data.length) ? hideable.values : d.values });
-
-      bars.exit().remove();
-
-
-      var barsEnter = bars.enter().append('rect')
-          .attr('class', function(d,i) { return getY(d,i) < 0 ? 'nv-bar negative' : 'nv-bar positive'})
-          .attr('x', function(d,i,j) {
-              return stacked ? 0 : (j * x.rangeBand() / data.length )
-          })
-          .attr('y', function(d) { return y0(stacked ? d.y0 : 0) })
-          .attr('height', 0)
-          .attr('width', x.rangeBand() / (stacked ? 1 : data.length) )
-          .attr('transform', function(d,i) { return 'translate(' + x(getX(d,i)) + ',0)'; })
-          ;
-      bars
-          .style('fill', function(d,i,j){ return color(d, j, i);  })
-          .style('stroke', function(d,i,j){ return color(d, j, i); })
-          .on('mouseover', function(d,i) { //TODO: figure out why j works above, but not here
-            d3.select(this).classed('hover', true);
-            dispatch.elementMouseover({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (stacked ? data.length / 2 : d.series + .5) / data.length), y(getY(d,i) + (stacked ? d.y0 : 0))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-          })
-          .on('mouseout', function(d,i) {
-            d3.select(this).classed('hover', false);
-            dispatch.elementMouseout({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-          })
-          .on('click', function(d,i) {
-            dispatch.elementClick({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (stacked ? data.length / 2 : d.series + .5) / data.length), y(getY(d,i) + (stacked ? d.y0 : 0))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-            d3.event.stopPropagation();
-          })
-          .on('dblclick', function(d,i) {
-            dispatch.elementDblClick({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (stacked ? data.length / 2 : d.series + .5) / data.length), y(getY(d,i) + (stacked ? d.y0 : 0))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-            d3.event.stopPropagation();
-          });
-      bars
-          .attr('class', function(d,i) { return getY(d,i) < 0 ? 'nv-bar negative' : 'nv-bar positive'})
-          .transition()
-          .attr('transform', function(d,i) { return 'translate(' + x(getX(d,i)) + ',0)'; })
-
-      if (barColor) {
-        if (!disabled) disabled = data.map(function() { return true });
-        bars
-          .style('fill', function(d,i,j) { return d3.rgb(barColor(d,i)).darker(  disabled.map(function(d,i) { return i }).filter(function(d,i){ return !disabled[i]  })[j]   ).toString(); })
-          .style('stroke', function(d,i,j) { return d3.rgb(barColor(d,i)).darker(  disabled.map(function(d,i) { return i }).filter(function(d,i){ return !disabled[i]  })[j]   ).toString(); });
-      }
-
-
-      if (stacked)
-          bars.transition()
-            .delay(function(d,i) { 
-
-                  return i * delay / data[0].values.length;
-            })
-            .attr('y', function(d,i) {
-
-              return y((stacked ? d.y1 : 0));
-            })
-            .attr('height', function(d,i) {
-              return Math.max(Math.abs(y(d.y + (stacked ? d.y0 : 0)) - y((stacked ? d.y0 : 0))),1);
-            })
-            .attr('x', function(d,i) {
-                  return stacked ? 0 : (d.series * x.rangeBand() / data.length )
-            })
-            .attr('width', x.rangeBand() / (stacked ? 1 : data.length) );
-      else
-          bars.transition()
-            .delay(function(d,i) { 
-                return i * delay/ data[0].values.length;
-            })
-            .attr('x', function(d,i) {
-              return d.series * x.rangeBand() / data.length
-            })
-            .attr('width', x.rangeBand() / data.length)
-            .attr('y', function(d,i) {
-                return getY(d,i) < 0 ?
-                        y(0) :
-                        y(0) - y(getY(d,i)) < 1 ?
-                          y(0) - 1 :
-                        y(getY(d,i)) || 0;
-            })
-            .attr('height', function(d,i) {
-                return Math.max(Math.abs(y(getY(d,i)) - y(0)),1) || 0;
-            });
-
-
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.forceY = function(_) {
-    if (!arguments.length) return forceY;
-    forceY = _;
-    return chart;
-  };
-
-  chart.stacked = function(_) {
-    if (!arguments.length) return stacked;
-    stacked = _;
-    return chart;
-  };
-
-  chart.clipEdge = function(_) {
-    if (!arguments.length) return clipEdge;
-    clipEdge = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.barColor = function(_) {
-    if (!arguments.length) return barColor;
-    barColor = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.disabled = function(_) {
-    if (!arguments.length) return disabled;
-    disabled = _;
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.hideable = function(_) {
-    if (!arguments.length) return hideable;
-    hideable = _;
-    return chart;
-  };
-
-  chart.delay = function(_) {
-    if (!arguments.length) return delay;
-    delay = _;
-    return chart;
-  };
-
-  chart.groupSpacing = function(_) {
-    if (!arguments.length) return groupSpacing;
-    groupSpacing = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.multiBarChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var multibar = nv.models.multiBar()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , legend = nv.models.legend()
-    , controls = nv.models.legend()
-    ;
-
-  var margin = {top: 30, right: 20, bottom: 50, left: 60}
-    , width = null
-    , height = null
-    , color = nv.utils.defaultColor()
-    , showControls = true
-    , showLegend = true
-    , showXAxis = true
-    , showYAxis = true
-    , rightAlignYAxis = false
-    , reduceXTicks = true // if false a tick will show for every data point
-    , staggerLabels = false
-    , rotateLabels = 0
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' on ' + x + '</p>'
-      }
-    , x //can be accessed via chart.xScale()
-    , y //can be accessed via chart.yScale()
-    , state = { stacked: false }
-    , defaultState = null
-    , noData = "No Data Available."
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , controlWidth = function() { return showControls ? 180 : 0 }
-    , transitionDuration = 250
-    ;
-
-  multibar
-    .stacked(false)
-    ;
-  xAxis
-    .orient('bottom')
-    .tickPadding(7)
-    .highlightZero(true)
-    .showMaxMin(false)
-    .tickFormat(function(d) { return d })
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    .tickFormat(d3.format(',.1f'))
-    ;
-
-  controls.updateState(false);
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(multibar.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(multibar.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart) };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-      //------------------------------------------------------------
-      // Display noData message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = multibar.xScale();
-      y = multibar.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-multiBarWithLegend').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-multiBarWithLegend').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-barsWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-controlsWrap');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width(availableWidth - controlWidth());
-
-        if (multibar.barColor())
-          data.forEach(function(series,i) {
-            series.color = d3.rgb('#ccc').darker(i * 1.5).toString();
-          })
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + controlWidth() + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Controls
-
-      if (showControls) {
-        var controlsData = [
-          { key: 'Grouped', disabled: multibar.stacked() },
-          { key: 'Stacked', disabled: !multibar.stacked() }
-        ];
-
-        controls.width(controlWidth()).color(['#444', '#444', '#444']);
-        g.select('.nv-controlsWrap')
-            .datum(controlsData)
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-            .call(controls);
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      multibar
-        .disabled(data.map(function(series) { return series.disabled }))
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled }))
-
-
-      var barsWrap = g.select('.nv-barsWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      barsWrap.transition().call(multibar);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-          xAxis
-            .scale(x)
-            .ticks( availableWidth / 100 )
-            .tickSize(-availableHeight, 0);
-
-          g.select('.nv-x.nv-axis')
-              .attr('transform', 'translate(0,' + y.range()[0] + ')');
-          g.select('.nv-x.nv-axis').transition()
-              .call(xAxis);
-
-          var xTicks = g.select('.nv-x.nv-axis > g').selectAll('g');
-
-          xTicks
-              .selectAll('line, text')
-              .style('opacity', 1)
-
-          if (staggerLabels) {
-              var getTranslate = function(x,y) {
-                  return "translate(" + x + "," + y + ")";
-              };
-
-              var staggerUp = 5, staggerDown = 17;  //pixels to stagger by
-              // Issue #140
-              xTicks
-                .selectAll("text")
-                .attr('transform', function(d,i,j) { 
-                    return  getTranslate(0, (j % 2 == 0 ? staggerUp : staggerDown));
-                  });
-
-              var totalInBetweenTicks = d3.selectAll(".nv-x.nv-axis .nv-wrap g g text")[0].length;
-              g.selectAll(".nv-x.nv-axis .nv-axisMaxMin text")
-                .attr("transform", function(d,i) {
-                    return getTranslate(0, (i === 0 || totalInBetweenTicks % 2 !== 0) ? staggerDown : staggerUp);
-                });
-          }
-
-          if (reduceXTicks)
-            xTicks
-              .filter(function(d,i) {
-                  return i % Math.ceil(data[0].values.length / (availableWidth / 100)) !== 0;
-                })
-              .selectAll('text, line')
-              .style('opacity', 0);
-
-          if(rotateLabels)
-            xTicks
-              .selectAll('.tick text')
-              .attr('transform', 'rotate(' + rotateLabels + ' 0,0)')
-              .style('text-anchor', rotateLabels > 0 ? 'start' : 'end');
-          
-          g.select('.nv-x.nv-axis').selectAll('g.nv-axisMaxMin text')
-              .style('opacity', 1);
-      }
-
-
-      if (showYAxis) {      
-          yAxis
-            .scale(y)
-            .ticks( availableHeight / 36 )
-            .tickSize( -availableWidth, 0);
-
-          g.select('.nv-y.nv-axis').transition()
-              .call(yAxis);
-      }
-
-
-      //------------------------------------------------------------
-
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        state = newState;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      controls.dispatch.on('legendClick', function(d,i) {
-        if (!d.disabled) return;
-        controlsData = controlsData.map(function(s) {
-          s.disabled = true;
-          return s;
-        });
-        d.disabled = false;
-
-        switch (d.key) {
-          case 'Grouped':
-            multibar.stacked(false);
-            break;
-          case 'Stacked':
-            multibar.stacked(true);
-            break;
-        }
-
-        state.stacked = multibar.stacked();
-        dispatch.stateChange(state);
-
-        chart.update();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode)
-      });
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        if (typeof e.stacked !== 'undefined') {
-          multibar.stacked(e.stacked);
-          state.stacked = e.stacked;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  multibar.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  multibar.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.multibar = multibar;
-  chart.legend = legend;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-
-  d3.rebind(chart, multibar, 'x', 'y', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'clipEdge',
-   'id', 'stacked', 'delay', 'barColor','groupSpacing');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showControls = function(_) {
-    if (!arguments.length) return showControls;
-    showControls = _;
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.reduceXTicks= function(_) {
-    if (!arguments.length) return reduceXTicks;
-    reduceXTicks = _;
-    return chart;
-  };
-
-  chart.rotateLabels = function(_) {
-    if (!arguments.length) return rotateLabels;
-    rotateLabels = _;
-    return chart;
-  }
-
-  chart.staggerLabels = function(_) {
-    if (!arguments.length) return staggerLabels;
-    staggerLabels = _;
-    return chart;
-  };
-
-  chart.tooltip = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-  
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.multiBarHorizontal = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , id = Math.floor(Math.random() * 10000) //Create semi-unique ID in case user doesn't select one
-    , x = d3.scale.ordinal()
-    , y = d3.scale.linear()
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , forceY = [0] // 0 is forced by default.. this makes sense for the majority of bar graphs... user can always do chart.forceY([]) to remove
-    , color = nv.utils.defaultColor()
-    , barColor = null // adding the ability to set the color for each rather than the whole group
-    , disabled // used in conjunction with barColor to communicate from multiBarHorizontalChart what series are disabled
-    , stacked = false
-    , showValues = false
-    , valuePadding = 60
-    , valueFormat = d3.format(',.2f')
-    , delay = 1200
-    , xDomain
-    , yDomain
-    , xRange
-    , yRange
-    , dispatch = d3.dispatch('chartClick', 'elementClick', 'elementDblClick', 'elementMouseover', 'elementMouseout')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0 //used to store previous scales
-      ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-
-      if (stacked)
-        data = d3.layout.stack()
-                 .offset('zero')
-                 .values(function(d){ return d.values })
-                 .y(getY)
-                 (data);
-
-
-      //add series index to each data point for reference
-      data = data.map(function(series, i) {
-        series.values = series.values.map(function(point) {
-          point.series = i;
-          return point;
-        });
-        return series;
-      });
-
-
-
-      //------------------------------------------------------------
-      // HACK for negative value stacking
-      if (stacked)
-        data[0].values.map(function(d,i) {
-          var posBase = 0, negBase = 0;
-          data.map(function(d) {
-            var f = d.values[i]
-            f.size = Math.abs(f.y);
-            if (f.y<0)  {
-              f.y1 = negBase - f.size;
-              negBase = negBase - f.size;
-            } else
-            { 
-              f.y1 = posBase;
-              posBase = posBase + f.size;
-            }
-          });
-        });
-
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      // remap and flatten the data for use in calculating the scales' domains
-      var seriesData = (xDomain && yDomain) ? [] : // if we know xDomain and yDomain, no need to calculate
-            data.map(function(d) {
-              return d.values.map(function(d,i) {
-                return { x: getX(d,i), y: getY(d,i), y0: d.y0, y1: d.y1 }
-              })
-            });
-
-      x   .domain(xDomain || d3.merge(seriesData).map(function(d) { return d.x }))
-          .rangeBands(xRange || [0, availableHeight], .1);
-
-      //y   .domain(yDomain || d3.extent(d3.merge(seriesData).map(function(d) { return d.y + (stacked ? d.y0 : 0) }).concat(forceY)))
-      y   .domain(yDomain || d3.extent(d3.merge(seriesData).map(function(d) { return stacked ? (d.y > 0 ? d.y1 + d.y : d.y1 ) : d.y }).concat(forceY)))
-
-      if (showValues && !stacked)
-        y.range(yRange || [(y.domain()[0] < 0 ? valuePadding : 0), availableWidth - (y.domain()[1] > 0 ? valuePadding : 0) ]);
-      else
-        y.range(yRange || [0, availableWidth]);
-
-      x0 = x0 || x;
-      y0 = y0 || d3.scale.linear().domain(y.domain()).range([y(0),y(0)]);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = d3.select(this).selectAll('g.nv-wrap.nv-multibarHorizontal').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-multibarHorizontal');
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-groups');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-
-      var groups = wrap.select('.nv-groups').selectAll('.nv-group')
-          .data(function(d) { return d }, function(d,i) { return i });
-      groups.enter().append('g')
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6);
-      groups.exit().transition()
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6)
-          .remove();
-      groups
-          .attr('class', function(d,i) { return 'nv-group nv-series-' + i })
-          .classed('hover', function(d) { return d.hover })
-          .style('fill', function(d,i){ return color(d, i) })
-          .style('stroke', function(d,i){ return color(d, i) });
-      groups.transition()
-          .style('stroke-opacity', 1)
-          .style('fill-opacity', .75);
-
-
-      var bars = groups.selectAll('g.nv-bar')
-          .data(function(d) { return d.values });
-
-      bars.exit().remove();
-
-
-      var barsEnter = bars.enter().append('g')
-          .attr('transform', function(d,i,j) {
-              return 'translate(' + y0(stacked ? d.y0 : 0) + ',' + (stacked ? 0 : (j * x.rangeBand() / data.length ) + x(getX(d,i))) + ')'
-          });
-
-      barsEnter.append('rect')
-          .attr('width', 0)
-          .attr('height', x.rangeBand() / (stacked ? 1 : data.length) )
-
-      bars
-          .on('mouseover', function(d,i) { //TODO: figure out why j works above, but not here
-            d3.select(this).classed('hover', true);
-            dispatch.elementMouseover({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [ y(getY(d,i) + (stacked ? d.y0 : 0)), x(getX(d,i)) + (x.rangeBand() * (stacked ? data.length / 2 : d.series + .5) / data.length) ],
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-          })
-          .on('mouseout', function(d,i) {
-            d3.select(this).classed('hover', false);
-            dispatch.elementMouseout({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-          })
-          .on('click', function(d,i) {
-            dispatch.elementClick({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (stacked ? data.length / 2 : d.series + .5) / data.length), y(getY(d,i) + (stacked ? d.y0 : 0))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-            d3.event.stopPropagation();
-          })
-          .on('dblclick', function(d,i) {
-            dispatch.elementDblClick({
-              value: getY(d,i),
-              point: d,
-              series: data[d.series],
-              pos: [x(getX(d,i)) + (x.rangeBand() * (stacked ? data.length / 2 : d.series + .5) / data.length), y(getY(d,i) + (stacked ? d.y0 : 0))],  // TODO: Figure out why the value appears to be shifted
-              pointIndex: i,
-              seriesIndex: d.series,
-              e: d3.event
-            });
-            d3.event.stopPropagation();
-          });
-
-
-      barsEnter.append('text');
-
-      if (showValues && !stacked) {
-        bars.select('text')
-            .attr('text-anchor', function(d,i) { return getY(d,i) < 0 ? 'end' : 'start' })
-            .attr('y', x.rangeBand() / (data.length * 2))
-            .attr('dy', '.32em')
-            .text(function(d,i) { return valueFormat(getY(d,i)) })
-        bars.transition()
-          .select('text')
-            .attr('x', function(d,i) { return getY(d,i) < 0 ? -4 : y(getY(d,i)) - y(0) + 4 })
-      } else {
-        bars.selectAll('text').text('');
-      }
-
-      bars
-          .attr('class', function(d,i) { return getY(d,i) < 0 ? 'nv-bar negative' : 'nv-bar positive'})
-
-      if (barColor) {
-        if (!disabled) disabled = data.map(function() { return true });
-        bars
-          .style('fill', function(d,i,j) { return d3.rgb(barColor(d,i)).darker(  disabled.map(function(d,i) { return i }).filter(function(d,i){ return !disabled[i]  })[j]   ).toString(); })
-          .style('stroke', function(d,i,j) { return d3.rgb(barColor(d,i)).darker(  disabled.map(function(d,i) { return i }).filter(function(d,i){ return !disabled[i]  })[j]   ).toString(); });
-      }
-
-      if (stacked)
-        bars.transition()
-            .attr('transform', function(d,i) {
-              return 'translate(' + y(d.y1) + ',' + x(getX(d,i)) + ')'
-            })
-          .select('rect')
-            .attr('width', function(d,i) {
-              return Math.abs(y(getY(d,i) + d.y0) - y(d.y0))
-            })
-            .attr('height', x.rangeBand() );
-      else
-        bars.transition()
-            .attr('transform', function(d,i) {
-              //TODO: stacked must be all positive or all negative, not both?
-              return 'translate(' + 
-              (getY(d,i) < 0 ? y(getY(d,i)) : y(0))
-              + ',' +
-              (d.series * x.rangeBand() / data.length
-              +
-              x(getX(d,i)) )
-              + ')'
-            })
-          .select('rect')
-            .attr('height', x.rangeBand() / data.length )
-            .attr('width', function(d,i) {
-              return Math.max(Math.abs(y(getY(d,i)) - y(0)),1)
-            });
-
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.forceY = function(_) {
-    if (!arguments.length) return forceY;
-    forceY = _;
-    return chart;
-  };
-
-  chart.stacked = function(_) {
-    if (!arguments.length) return stacked;
-    stacked = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.barColor = function(_) {
-    if (!arguments.length) return barColor;
-    barColor = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.disabled = function(_) {
-    if (!arguments.length) return disabled;
-    disabled = _;
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.delay = function(_) {
-    if (!arguments.length) return delay;
-    delay = _;
-    return chart;
-  };
-
-  chart.showValues = function(_) {
-    if (!arguments.length) return showValues;
-    showValues = _;
-    return chart;
-  };
-
-  chart.valueFormat= function(_) {
-    if (!arguments.length) return valueFormat;
-    valueFormat = _;
-    return chart;
-  };
-
-  chart.valuePadding = function(_) {
-    if (!arguments.length) return valuePadding;
-    valuePadding = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.multiBarHorizontalChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var multibar = nv.models.multiBarHorizontal()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , legend = nv.models.legend().height(30)
-    , controls = nv.models.legend().height(30)
-    ;
-
-  var margin = {top: 30, right: 20, bottom: 50, left: 60}
-    , width = null
-    , height = null
-    , color = nv.utils.defaultColor()
-    , showControls = true
-    , showLegend = true
-    , stacked = false
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + ' - ' + x + '</h3>' +
-               '<p>' +  y + '</p>'
-      }
-    , x //can be accessed via chart.xScale()
-    , y //can be accessed via chart.yScale()
-    , state = { stacked: stacked }
-    , defaultState = null
-    , noData = 'No Data Available.'
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , controlWidth = function() { return showControls ? 180 : 0 }
-    , transitionDuration = 250
-    ;
-
-  multibar
-    .stacked(stacked)
-    ;
-  xAxis
-    .orient('left')
-    .tickPadding(5)
-    .highlightZero(false)
-    .showMaxMin(false)
-    .tickFormat(function(d) { return d })
-    ;
-  yAxis
-    .orient('bottom')
-    .tickFormat(d3.format(',.1f'))
-    ;
-
-  controls.updateState(false);
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(multibar.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(multibar.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'e' : 'w', null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart) };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = multibar.xScale();
-      y = multibar.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-multiBarHorizontalChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-multiBarHorizontalChart').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-barsWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-controlsWrap');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width(availableWidth - controlWidth());
-
-        if (multibar.barColor())
-          data.forEach(function(series,i) {
-            series.color = d3.rgb('#ccc').darker(i * 1.5).toString();
-          })
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + controlWidth() + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Controls
-
-      if (showControls) {
-        var controlsData = [
-          { key: 'Grouped', disabled: multibar.stacked() },
-          { key: 'Stacked', disabled: !multibar.stacked() }
-        ];
-
-        controls.width(controlWidth()).color(['#444', '#444', '#444']);
-        g.select('.nv-controlsWrap')
-            .datum(controlsData)
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-            .call(controls);
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      multibar
-        .disabled(data.map(function(series) { return series.disabled }))
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color(d, i);
-        }).filter(function(d,i) { return !data[i].disabled }))
-
-
-      var barsWrap = g.select('.nv-barsWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-
-      barsWrap.transition().call(multibar);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      xAxis
-        .scale(x)
-        .ticks( availableHeight / 24 )
-        .tickSize(-availableWidth, 0);
-
-      g.select('.nv-x.nv-axis').transition()
-          .call(xAxis);
-
-      var xTicks = g.select('.nv-x.nv-axis').selectAll('g');
-
-      xTicks
-          .selectAll('line, text')
-          .style('opacity', 1)
-
-
-      yAxis
-        .scale(y)
-        .ticks( availableWidth / 100 )
-        .tickSize( -availableHeight, 0);
-
-      g.select('.nv-y.nv-axis')
-          .attr('transform', 'translate(0,' + availableHeight + ')');
-      g.select('.nv-y.nv-axis').transition()
-          .call(yAxis);
-
-      //------------------------------------------------------------
-
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        state = newState;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      controls.dispatch.on('legendClick', function(d,i) {
-        if (!d.disabled) return;
-        controlsData = controlsData.map(function(s) {
-          s.disabled = true;
-          return s;
-        });
-        d.disabled = false;
-
-        switch (d.key) {
-          case 'Grouped':
-            multibar.stacked(false);
-            break;
-          case 'Stacked':
-            multibar.stacked(true);
-            break;
-        }
-
-        state.stacked = multibar.stacked();
-        dispatch.stateChange(state);
-
-        chart.update();
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        if (typeof e.stacked !== 'undefined') {
-          multibar.stacked(e.stacked);
-          state.stacked = e.stacked;
-        }
-
-        selection.call(chart);
-      });
-      //============================================================
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  multibar.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  multibar.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.multibar = multibar;
-  chart.legend = legend;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-
-  d3.rebind(chart, multibar, 'x', 'y', 'xDomain', 'yDomain', 'xRange', 'yRange', 'forceX', 'forceY', 'clipEdge', 'id', 'delay', 'showValues', 'valueFormat', 'stacked', 'barColor');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    return chart;
-  };
-
-  chart.showControls = function(_) {
-    if (!arguments.length) return showControls;
-    showControls = _;
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.tooltip = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-  //============================================================
-
-
-  return chart;
-}
-nv.models.multiChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 30, right: 20, bottom: 50, left: 60},
-      color = d3.scale.category20().range(),
-      width = null, 
-      height = null,
-      showLegend = true,
-      tooltips = true,
-      tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' at ' + x + '</p>'
-      },
-      x,
-      y,
-      yDomain1,
-      yDomain2
-      ; //can be accessed via chart.lines.[x/y]Scale()
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x = d3.scale.linear(),
-      yScale1 = d3.scale.linear(),
-      yScale2 = d3.scale.linear(),
-
-      lines1 = nv.models.line().yScale(yScale1),
-      lines2 = nv.models.line().yScale(yScale2),
-
-      bars1 = nv.models.multiBar().stacked(false).yScale(yScale1),
-      bars2 = nv.models.multiBar().stacked(false).yScale(yScale2),
-
-      stack1 = nv.models.stackedArea().yScale(yScale1),
-      stack2 = nv.models.stackedArea().yScale(yScale2),
-
-      xAxis = nv.models.axis().scale(x).orient('bottom').tickPadding(5),
-      yAxis1 = nv.models.axis().scale(yScale1).orient('left'),
-      yAxis2 = nv.models.axis().scale(yScale2).orient('right'),
-
-      legend = nv.models.legend().height(30),
-      dispatch = d3.dispatch('tooltipShow', 'tooltipHide');
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(lines1.x()(e.point, e.pointIndex)),
-        y = ((e.series.yAxis == 2) ? yAxis2 : yAxis1).tickFormat()(lines1.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, undefined, undefined, offsetElement.offsetParent);
-  };
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      chart.update = function() { container.transition().call(chart); };
-      chart.container = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      var dataLines1 = data.filter(function(d) {return !d.disabled && d.type == 'line' && d.yAxis == 1})
-      var dataLines2 = data.filter(function(d) {return !d.disabled && d.type == 'line' && d.yAxis == 2})
-      var dataBars1 = data.filter(function(d) {return !d.disabled && d.type == 'bar' && d.yAxis == 1})
-      var dataBars2 = data.filter(function(d) {return !d.disabled && d.type == 'bar' && d.yAxis == 2})
-      var dataStack1 = data.filter(function(d) {return !d.disabled && d.type == 'area' && d.yAxis == 1})
-      var dataStack2 = data.filter(function(d) {return !d.disabled && d.type == 'area' && d.yAxis == 2})
-
-      var series1 = data.filter(function(d) {return !d.disabled && d.yAxis == 1})
-            .map(function(d) {
-              return d.values.map(function(d,i) {
-                return { x: d.x, y: d.y }
-              })
-            })
-
-      var series2 = data.filter(function(d) {return !d.disabled && d.yAxis == 2})
-            .map(function(d) {
-              return d.values.map(function(d,i) {
-                return { x: d.x, y: d.y }
-              })
-            })
-
-      x   .domain(d3.extent(d3.merge(series1.concat(series2)), function(d) { return d.x } ))
-          .range([0, availableWidth]);
-
-      var wrap = container.selectAll('g.wrap.multiChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'wrap nvd3 multiChart').append('g');
-
-      gEnter.append('g').attr('class', 'x axis');
-      gEnter.append('g').attr('class', 'y1 axis');
-      gEnter.append('g').attr('class', 'y2 axis');
-      gEnter.append('g').attr('class', 'lines1Wrap');
-      gEnter.append('g').attr('class', 'lines2Wrap');
-      gEnter.append('g').attr('class', 'bars1Wrap');
-      gEnter.append('g').attr('class', 'bars2Wrap');
-      gEnter.append('g').attr('class', 'stack1Wrap');
-      gEnter.append('g').attr('class', 'stack2Wrap');
-      gEnter.append('g').attr('class', 'legendWrap');
-
-      var g = wrap.select('g');
-
-      if (showLegend) {
-        legend.width( availableWidth / 2 );
-
-        g.select('.legendWrap')
-            .datum(data.map(function(series) { 
-              series.originalKey = series.originalKey === undefined ? series.key : series.originalKey;
-              series.key = series.originalKey + (series.yAxis == 1 ? '' : ' (right axis)');
-              return series;
-            }))
-          .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        g.select('.legendWrap')
-            .attr('transform', 'translate(' + ( availableWidth / 2 ) + ',' + (-margin.top) +')');
-      }
-
-
-      lines1
-        .width(availableWidth)
-        .height(availableHeight)
-        .interpolate("monotone")
-        .color(data.map(function(d,i) {
-          return d.color || color[i % color.length];
-        }).filter(function(d,i) { return !data[i].disabled && data[i].yAxis == 1 && data[i].type == 'line'}));
-
-      lines2
-        .width(availableWidth)
-        .height(availableHeight)
-        .interpolate("monotone")
-        .color(data.map(function(d,i) {
-          return d.color || color[i % color.length];
-        }).filter(function(d,i) { return !data[i].disabled && data[i].yAxis == 2 && data[i].type == 'line'}));
-
-      bars1
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color[i % color.length];
-        }).filter(function(d,i) { return !data[i].disabled && data[i].yAxis == 1 && data[i].type == 'bar'}));
-
-      bars2
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color[i % color.length];
-        }).filter(function(d,i) { return !data[i].disabled && data[i].yAxis == 2 && data[i].type == 'bar'}));
-
-      stack1
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color[i % color.length];
-        }).filter(function(d,i) { return !data[i].disabled && data[i].yAxis == 1 && data[i].type == 'area'}));
-
-      stack2
-        .width(availableWidth)
-        .height(availableHeight)
-        .color(data.map(function(d,i) {
-          return d.color || color[i % color.length];
-        }).filter(function(d,i) { return !data[i].disabled && data[i].yAxis == 2 && data[i].type == 'area'}));
-
-      g.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-
-      var lines1Wrap = g.select('.lines1Wrap')
-          .datum(dataLines1)
-      var bars1Wrap = g.select('.bars1Wrap')
-          .datum(dataBars1)
-      var stack1Wrap = g.select('.stack1Wrap')
-          .datum(dataStack1)
-
-      var lines2Wrap = g.select('.lines2Wrap')
-          .datum(dataLines2)
-      var bars2Wrap = g.select('.bars2Wrap')
-          .datum(dataBars2)
-      var stack2Wrap = g.select('.stack2Wrap')
-          .datum(dataStack2)
-
-      var extraValue1 = dataStack1.length ? dataStack1.map(function(a){return a.values}).reduce(function(a,b){
-        return a.map(function(aVal,i){return {x: aVal.x, y: aVal.y + b[i].y}})
-      }).concat([{x:0, y:0}]) : []
-      var extraValue2 = dataStack2.length ? dataStack2.map(function(a){return a.values}).reduce(function(a,b){
-        return a.map(function(aVal,i){return {x: aVal.x, y: aVal.y + b[i].y}})
-      }).concat([{x:0, y:0}]) : []
-
-      yScale1 .domain(yDomain1 || d3.extent(d3.merge(series1).concat(extraValue1), function(d) { return d.y } ))
-              .range([0, availableHeight])
-
-      yScale2 .domain(yDomain2 || d3.extent(d3.merge(series2).concat(extraValue2), function(d) { return d.y } ))
-              .range([0, availableHeight])
-
-      lines1.yDomain(yScale1.domain())
-      bars1.yDomain(yScale1.domain())
-      stack1.yDomain(yScale1.domain())
-
-      lines2.yDomain(yScale2.domain())
-      bars2.yDomain(yScale2.domain())
-      stack2.yDomain(yScale2.domain())
-
-      if(dataStack1.length){d3.transition(stack1Wrap).call(stack1);}
-      if(dataStack2.length){d3.transition(stack2Wrap).call(stack2);}
-
-      if(dataBars1.length){d3.transition(bars1Wrap).call(bars1);}
-      if(dataBars2.length){d3.transition(bars2Wrap).call(bars2);}
-
-      if(dataLines1.length){d3.transition(lines1Wrap).call(lines1);}
-      if(dataLines2.length){d3.transition(lines2Wrap).call(lines2);}
-      
-
-
-      xAxis
-        .ticks( availableWidth / 100 )
-        .tickSize(-availableHeight, 0);
-
-      g.select('.x.axis')
-          .attr('transform', 'translate(0,' + availableHeight + ')');
-      d3.transition(g.select('.x.axis'))
-          .call(xAxis);
-
-      yAxis1
-        .ticks( availableHeight / 36 )
-        .tickSize( -availableWidth, 0);
-
-
-      d3.transition(g.select('.y1.axis'))
-          .call(yAxis1);
-
-      yAxis2
-        .ticks( availableHeight / 36 )
-        .tickSize( -availableWidth, 0);
-
-      d3.transition(g.select('.y2.axis'))
-          .call(yAxis2);
-
-      g.select('.y2.axis')
-          .style('opacity', series2.length ? 1 : 0)
-          .attr('transform', 'translate(' + x.range()[1] + ',0)');
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        chart.update();
-      });
-     
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  lines1.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines1.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  lines2.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines2.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  bars1.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  bars1.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  bars2.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  bars2.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  stack1.dispatch.on('tooltipShow', function(e) {
-    //disable tooltips when value ~= 0
-    //// TODO: consider removing points from voronoi that have 0 value instead of this hack
-    if (!Math.round(stack1.y()(e.point) * 100)) {  // 100 will not be good for very small numbers... will have to think about making this valu dynamic, based on data range
-      setTimeout(function() { d3.selectAll('.point.hover').classed('hover', false) }, 0);
-      return false;
-    }
-
-    e.pos = [e.pos[0] + margin.left, e.pos[1] + margin.top],
-    dispatch.tooltipShow(e);
-  });
-
-  stack1.dispatch.on('tooltipHide', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  stack2.dispatch.on('tooltipShow', function(e) {
-    //disable tooltips when value ~= 0
-    //// TODO: consider removing points from voronoi that have 0 value instead of this hack
-    if (!Math.round(stack2.y()(e.point) * 100)) {  // 100 will not be good for very small numbers... will have to think about making this valu dynamic, based on data range
-      setTimeout(function() { d3.selectAll('.point.hover').classed('hover', false) }, 0);
-      return false;
-    }
-
-    e.pos = [e.pos[0] + margin.left, e.pos[1] + margin.top],
-    dispatch.tooltipShow(e);
-  });
-
-  stack2.dispatch.on('tooltipHide', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-    lines1.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines1.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  lines2.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  lines2.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-
-
-  //============================================================
-  // Global getters and setters
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-  chart.lines1 = lines1;
-  chart.lines2 = lines2;
-  chart.bars1 = bars1;
-  chart.bars2 = bars2;
-  chart.stack1 = stack1;
-  chart.stack2 = stack2;
-  chart.xAxis = xAxis;
-  chart.yAxis1 = yAxis1;
-  chart.yAxis2 = yAxis2;
-  chart.options = nv.utils.optionsFunc.bind(chart);
-
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    lines1.x(_);
-    bars1.x(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    lines1.y(_);
-    bars1.y(_);
-    return chart;
-  };
-
-  chart.yDomain1 = function(_) {
-    if (!arguments.length) return yDomain1;
-    yDomain1 = _;
-    return chart;
-  };
-
-  chart.yDomain2 = function(_) {
-    if (!arguments.length) return yDomain2;
-    yDomain2 = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin = _;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = _;
-    legend.color(_);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  return chart;
-}
-
-
-nv.models.ohlcBar = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , id = Math.floor(Math.random() * 10000) //Create semi-unique ID in case user doesn't select one
-    , x = d3.scale.linear()
-    , y = d3.scale.linear()
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , getOpen = function(d) { return d.open }
-    , getClose = function(d) { return d.close }
-    , getHigh = function(d) { return d.high }
-    , getLow = function(d) { return d.low }
-    , forceX = []
-    , forceY = []
-    , padData     = false // If true, adds half a data points width to front and back, for lining up a line chart with a bar chart
-    , clipEdge = true
-    , color = nv.utils.defaultColor()
-    , xDomain
-    , yDomain
-    , xRange
-    , yRange
-    , dispatch = d3.dispatch('chartClick', 'elementClick', 'elementDblClick', 'elementMouseover', 'elementMouseout')
-    ;
-
-  //============================================================
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  //TODO: store old scales for transitions
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x   .domain(xDomain || d3.extent(data[0].values.map(getX).concat(forceX) ));
-
-      if (padData)
-        x.range(xRange || [availableWidth * .5 / data[0].values.length, availableWidth * (data[0].values.length - .5)  / data[0].values.length ]);
-      else
-        x.range(xRange || [0, availableWidth]);
-
-      y   .domain(yDomain || [
-            d3.min(data[0].values.map(getLow).concat(forceY)),
-            d3.max(data[0].values.map(getHigh).concat(forceY))
-          ])
-          .range(yRange || [availableHeight, 0]);
-
-      // If scale's domain don't have a range, slightly adjust to make one... so a chart can show a single data point
-      if (x.domain()[0] === x.domain()[1] || y.domain()[0] === y.domain()[1]) singlePoint = true;
-      if (x.domain()[0] === x.domain()[1])
-        x.domain()[0] ?
-            x.domain([x.domain()[0] - x.domain()[0] * 0.01, x.domain()[1] + x.domain()[1] * 0.01])
-          : x.domain([-1,1]);
-
-      if (y.domain()[0] === y.domain()[1])
-        y.domain()[0] ?
-            y.domain([y.domain()[0] + y.domain()[0] * 0.01, y.domain()[1] - y.domain()[1] * 0.01])
-          : y.domain([-1,1]);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = d3.select(this).selectAll('g.nv-wrap.nv-ohlcBar').data([data[0].values]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-ohlcBar');
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-ticks');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      container
-          .on('click', function(d,i) {
-            dispatch.chartClick({
-                data: d,
-                index: i,
-                pos: d3.event,
-                id: id
-            });
-          });
-
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-chart-clip-path-' + id)
-        .append('rect');
-
-      wrap.select('#nv-chart-clip-path-' + id + ' rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      g   .attr('clip-path', clipEdge ? 'url(#nv-chart-clip-path-' + id + ')' : '');
-
-
-
-      var ticks = wrap.select('.nv-ticks').selectAll('.nv-tick')
-          .data(function(d) { return d });
-
-      ticks.exit().remove();
-
-
-      var ticksEnter = ticks.enter().append('path')
-          .attr('class', function(d,i,j) { return (getOpen(d,i) > getClose(d,i) ? 'nv-tick negative' : 'nv-tick positive') + ' nv-tick-' + j + '-' + i })
-          .attr('d', function(d,i) {
-            var w = (availableWidth / data[0].values.length) * .9;
-            return 'm0,0l0,'
-                 + (y(getOpen(d,i))
-                 - y(getHigh(d,i)))
-                 + 'l'
-                 + (-w/2)
-                 + ',0l'
-                 + (w/2)
-                 + ',0l0,'
-                 + (y(getLow(d,i)) - y(getOpen(d,i)))
-                 + 'l0,'
-                 + (y(getClose(d,i))
-                 - y(getLow(d,i)))
-                 + 'l'
-                 + (w/2)
-                 + ',0l'
-                 + (-w/2)
-                 + ',0z';
-          })
-          .attr('transform', function(d,i) { return 'translate(' + x(getX(d,i)) + ',' + y(getHigh(d,i)) + ')'; })
-          //.attr('fill', function(d,i) { return color[0]; })
-          //.attr('stroke', function(d,i) { return color[0]; })
-          //.attr('x', 0 )
-          //.attr('y', function(d,i) {  return y(Math.max(0, getY(d,i))) })
-          //.attr('height', function(d,i) { return Math.abs(y(getY(d,i)) - y(0)) })
-          .on('mouseover', function(d,i) {
-            d3.select(this).classed('hover', true);
-            dispatch.elementMouseover({
-                point: d,
-                series: data[0],
-                pos: [x(getX(d,i)), y(getY(d,i))],  // TODO: Figure out why the value appears to be shifted
-                pointIndex: i,
-                seriesIndex: 0,
-                e: d3.event
-            });
-
-          })
-          .on('mouseout', function(d,i) {
-                d3.select(this).classed('hover', false);
-                dispatch.elementMouseout({
-                    point: d,
-                    series: data[0],
-                    pointIndex: i,
-                    seriesIndex: 0,
-                    e: d3.event
-                });
-          })
-          .on('click', function(d,i) {
-                dispatch.elementClick({
-                    //label: d[label],
-                    value: getY(d,i),
-                    data: d,
-                    index: i,
-                    pos: [x(getX(d,i)), y(getY(d,i))],
-                    e: d3.event,
-                    id: id
-                });
-              d3.event.stopPropagation();
-          })
-          .on('dblclick', function(d,i) {
-              dispatch.elementDblClick({
-                  //label: d[label],
-                  value: getY(d,i),
-                  data: d,
-                  index: i,
-                  pos: [x(getX(d,i)), y(getY(d,i))],
-                  e: d3.event,
-                  id: id
-              });
-              d3.event.stopPropagation();
-          });
-
-      ticks
-          .attr('class', function(d,i,j) { return (getOpen(d,i) > getClose(d,i) ? 'nv-tick negative' : 'nv-tick positive') + ' nv-tick-' + j + '-' + i })
-      d3.transition(ticks)
-          .attr('transform', function(d,i) { return 'translate(' + x(getX(d,i)) + ',' + y(getHigh(d,i)) + ')'; })
-          .attr('d', function(d,i) {
-            var w = (availableWidth / data[0].values.length) * .9;
-            return 'm0,0l0,'
-                 + (y(getOpen(d,i))
-                 - y(getHigh(d,i)))
-                 + 'l'
-                 + (-w/2)
-                 + ',0l'
-                 + (w/2)
-                 + ',0l0,'
-                 + (y(getLow(d,i))
-                 - y(getOpen(d,i)))
-                 + 'l0,'
-                 + (y(getClose(d,i))
-                 - y(getLow(d,i)))
-                 + 'l'
-                 + (w/2)
-                 + ',0l'
-                 + (-w/2)
-                 + ',0z';
-          })
-          //.attr('width', (availableWidth / data[0].values.length) * .9 )
-
-
-      //d3.transition(ticks)
-          //.attr('y', function(d,i) {  return y(Math.max(0, getY(d,i))) })
-          //.attr('height', function(d,i) { return Math.abs(y(getY(d,i)) - y(0)) });
-          //.order();  // not sure if this makes any sense for this model
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = _;
-    return chart;
-  };
-
-  chart.open = function(_) {
-    if (!arguments.length) return getOpen;
-    getOpen = _;
-    return chart;
-  };
-
-  chart.close = function(_) {
-    if (!arguments.length) return getClose;
-    getClose = _;
-    return chart;
-  };
-
-  chart.high = function(_) {
-    if (!arguments.length) return getHigh;
-    getHigh = _;
-    return chart;
-  };
-
-  chart.low = function(_) {
-    if (!arguments.length) return getLow;
-    getLow = _;
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.forceX = function(_) {
-    if (!arguments.length) return forceX;
-    forceX = _;
-    return chart;
-  };
-
-  chart.forceY = function(_) {
-    if (!arguments.length) return forceY;
-    forceY = _;
-    return chart;
-  };
-
-  chart.padData = function(_) {
-    if (!arguments.length) return padData;
-    padData = _;
-    return chart;
-  };
-
-  chart.clipEdge = function(_) {
-    if (!arguments.length) return clipEdge;
-    clipEdge = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-nv.models.pie = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 500
-    , height = 500
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , getDescription = function(d) { return d.description }
-    , id = Math.floor(Math.random() * 10000) //Create semi-unique ID in case user doesn't select one
-    , color = nv.utils.defaultColor()
-    , valueFormat = d3.format(',.2f')
-    , showLabels = true
-    , pieLabelsOutside = true
-    , donutLabelsOutside = false
-    , labelType = "key"
-    , labelThreshold = .02 //if slice percentage is under this, don't show label
-    , donut = false
-    , labelSunbeamLayout = false
-    , startAngle = false
-    , endAngle = false
-    , donutRatio = 0.5
-    , dispatch = d3.dispatch('chartClick', 'elementClick', 'elementDblClick', 'elementMouseover', 'elementMouseout')
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          radius = Math.min(availableWidth, availableHeight) / 2,
-          arcRadius = radius-(radius / 5),
-          container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      //var wrap = container.selectAll('.nv-wrap.nv-pie').data([data]);
-      var wrap = container.selectAll('.nv-wrap.nv-pie').data(data);
-      var wrapEnter = wrap.enter().append('g').attr('class','nvd3 nv-wrap nv-pie nv-chart-' + id);
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-pie');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-      g.select('.nv-pie').attr('transform', 'translate(' + availableWidth / 2 + ',' + availableHeight / 2 + ')');
-
-      //------------------------------------------------------------
-
-
-      container
-          .on('click', function(d,i) {
-              dispatch.chartClick({
-                  data: d,
-                  index: i,
-                  pos: d3.event,
-                  id: id
-              });
-          });
-
-
-      var arc = d3.svg.arc()
-                  .outerRadius(arcRadius);
-
-      if (startAngle) arc.startAngle(startAngle)
-      if (endAngle) arc.endAngle(endAngle);
-      if (donut) arc.innerRadius(radius * donutRatio);
-
-      // Setup the Pie chart and choose the data element
-      var pie = d3.layout.pie()
-          .sort(null)
-          .value(function(d) { return d.disabled ? 0 : getY(d) });
-
-      var slices = wrap.select('.nv-pie').selectAll('.nv-slice')
-          .data(pie);
-
-      slices.exit().remove();
-
-      var ae = slices.enter().append('g')
-              .attr('class', 'nv-slice')
-              .on('mouseover', function(d,i){
-                d3.select(this).classed('hover', true);
-                dispatch.elementMouseover({
-                    label: getX(d.data),
-                    value: getY(d.data),
-                    point: d.data,
-                    pointIndex: i,
-                    pos: [d3.event.pageX, d3.event.pageY],
-                    id: id
-                });
-              })
-              .on('mouseout', function(d,i){
-                d3.select(this).classed('hover', false);
-                dispatch.elementMouseout({
-                    label: getX(d.data),
-                    value: getY(d.data),
-                    point: d.data,
-                    index: i,
-                    id: id
-                });
-              })
-              .on('click', function(d,i) {
-                dispatch.elementClick({
-                    label: getX(d.data),
-                    value: getY(d.data),
-                    point: d.data,
-                    index: i,
-                    pos: d3.event,
-                    id: id
-                });
-                d3.event.stopPropagation();
-              })
-              .on('dblclick', function(d,i) {
-                dispatch.elementDblClick({
-                    label: getX(d.data),
-                    value: getY(d.data),
-                    point: d.data,
-                    index: i,
-                    pos: d3.event,
-                    id: id
-                });
-                d3.event.stopPropagation();
-              });
-
-        slices
-            .attr('fill', function(d,i) { return color(d, i); })
-            .attr('stroke', function(d,i) { return color(d, i); });
-
-        var paths = ae.append('path')
-            .each(function(d) { this._current = d; });
-            //.attr('d', arc);
-
-        d3.transition(slices.select('path'))
-            .attr('d', arc)
-            .attrTween('d', arcTween);
-
-        if (showLabels) {
-          // This does the normal label
-          var labelsArc = d3.svg.arc().innerRadius(0);
-          
-          if (pieLabelsOutside){ labelsArc = arc; }
-
-          if (donutLabelsOutside) { labelsArc = d3.svg.arc().outerRadius(arc.outerRadius()); }
-
-          ae.append("g").classed("nv-label", true)
-            .each(function(d, i) {
-              var group = d3.select(this);
-
-              group
-                .attr('transform', function(d) {
-                     if (labelSunbeamLayout) {
-                       d.outerRadius = arcRadius + 10; // Set Outer Coordinate
-                       d.innerRadius = arcRadius + 15; // Set Inner Coordinate
-                       var rotateAngle = (d.startAngle + d.endAngle) / 2 * (180 / Math.PI);
-                       if ((d.startAngle+d.endAngle)/2 < Math.PI) {
-                       	 rotateAngle -= 90;
-                       } else {
-                       	 rotateAngle += 90;
-                       }
-                       return 'translate(' + labelsArc.centroid(d) + ') rotate(' + rotateAngle + ')';
-                     } else {
-                       d.outerRadius = radius + 10; // Set Outer Coordinate
-                       d.innerRadius = radius + 15; // Set Inner Coordinate
-                       return 'translate(' + labelsArc.centroid(d) + ')'
-                     }
-                });
-
-              group.append('rect')
-                  .style('stroke', '#fff')
-                  .style('fill', '#fff')
-                  .attr("rx", 3)
-                  .attr("ry", 3);
-
-              group.append('text')
-                  .style('text-anchor', labelSunbeamLayout ? ((d.startAngle + d.endAngle) / 2 < Math.PI ? 'start' : 'end') : 'middle') //center the text on it's origin or begin/end if orthogonal aligned
-                  .style('fill', '#000')
-
-
-          });
-
-          slices.select(".nv-label").transition()
-            .attr('transform', function(d) {
-            	if (labelSunbeamLayout) {
-                  d.outerRadius = arcRadius + 10; // Set Outer Coordinate
-                  d.innerRadius = arcRadius + 15; // Set Inner Coordinate
-                  var rotateAngle = (d.startAngle + d.endAngle) / 2 * (180 / Math.PI);
-                  if ((d.startAngle+d.endAngle)/2 < Math.PI) {
-                    rotateAngle -= 90;
-                  } else {
-                    rotateAngle += 90;
-                  }
-                  return 'translate(' + labelsArc.centroid(d) + ') rotate(' + rotateAngle + ')';
-                } else {
-                  d.outerRadius = radius + 10; // Set Outer Coordinate
-                  d.innerRadius = radius + 15; // Set Inner Coordinate
-                  return 'translate(' + labelsArc.centroid(d) + ')'
-                }
-            });
-
-          slices.each(function(d, i) {
-            var slice = d3.select(this);
-
-            slice
-              .select(".nv-label text")
-                .style('text-anchor', labelSunbeamLayout ? ((d.startAngle + d.endAngle) / 2 < Math.PI ? 'start' : 'end') : 'middle') //center the text on it's origin or begin/end if orthogonal aligned
-                .text(function(d, i) {
-                  var percent = (d.endAngle - d.startAngle) / (2 * Math.PI);
-                  var labelTypes = {
-                    "key" : getX(d.data),
-                    "value": getY(d.data),
-                    "percent": d3.format('%')(percent)
-                  };
-                  return (d.value && percent > labelThreshold) ? labelTypes[labelType] : '';
-                });
-
-            var textBox = slice.select('text').node().getBBox();
-            slice.select(".nv-label rect")
-              .attr("width", textBox.width + 10)
-              .attr("height", textBox.height + 10)
-              .attr("transform", function() {
-                return "translate(" + [textBox.x - 5, textBox.y - 5] + ")";
-              });
-          });
-        }
-
-
-        // Computes the angle of an arc, converting from radians to degrees.
-        function angle(d) {
-          var a = (d.startAngle + d.endAngle) * 90 / Math.PI - 90;
-          return a > 90 ? a - 180 : a;
-        }
-
-        function arcTween(a) {
-          a.endAngle = isNaN(a.endAngle) ? 0 : a.endAngle;
-          a.startAngle = isNaN(a.startAngle) ? 0 : a.startAngle;
-          if (!donut) a.innerRadius = 0;
-          var i = d3.interpolate(this._current, a);
-          this._current = i(0);
-          return function(t) {
-            return arc(i(t));
-          };
-        }
-
-        function tweenPie(b) {
-          b.innerRadius = 0;
-          var i = d3.interpolate({startAngle: 0, endAngle: 0}, b);
-          return function(t) {
-              return arc(i(t));
-          };
-        }
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-  chart.options = nv.utils.optionsFunc.bind(chart);
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.values = function(_) {
-    nv.log("pie.values() is no longer supported.");
-    return chart;
-  };
-
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = _;
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = d3.functor(_);
-    return chart;
-  };
-  
-  chart.description = function(_) {
-    if (!arguments.length) return getDescription;
-    getDescription = _;
-    return chart;
-  };
-
-  chart.showLabels = function(_) {
-    if (!arguments.length) return showLabels;
-    showLabels = _;
-    return chart;
-  };
-  
-  chart.labelSunbeamLayout = function(_) {
-    if (!arguments.length) return labelSunbeamLayout;
-    labelSunbeamLayout = _;
-    return chart;
-  };
-
-  chart.donutLabelsOutside = function(_) {
-    if (!arguments.length) return donutLabelsOutside;
-    donutLabelsOutside = _;
-    return chart;
-  };
-  
-  chart.pieLabelsOutside = function(_) {
-    if (!arguments.length) return pieLabelsOutside;
-    pieLabelsOutside = _;
-    return chart;
-  };
-
-  chart.labelType = function(_) {
-    if (!arguments.length) return labelType;
-    labelType = _;
-    labelType = labelType || "key";
-    return chart;
-  };
-
-  chart.donut = function(_) {
-    if (!arguments.length) return donut;
-    donut = _;
-    return chart;
-  };
-  
-  chart.donutRatio = function(_) {
-    if (!arguments.length) return donutRatio;
-    donutRatio = _;
-    return chart;
-  };
-
-  chart.startAngle = function(_) {
-    if (!arguments.length) return startAngle;
-    startAngle = _;
-    return chart;
-  };
-
-  chart.endAngle = function(_) {
-    if (!arguments.length) return endAngle;
-    endAngle = _;
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.valueFormat = function(_) {
-    if (!arguments.length) return valueFormat;
-    valueFormat = _;
-    return chart;
-  };
-
-  chart.labelThreshold = function(_) {
-    if (!arguments.length) return labelThreshold;
-    labelThreshold = _;
-    return chart;
-  };
-  //============================================================
-
-
-  return chart;
-}
-nv.models.pieChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var pie = nv.models.pie()
-    , legend = nv.models.legend()
-    ;
-
-  var margin = {top: 30, right: 20, bottom: 20, left: 20}
-    , width = null
-    , height = null
-    , showLegend = true
-    , color = nv.utils.defaultColor()
-    , tooltips = true
-    , tooltip = function(key, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + '</p>'
-      }
-    , state = {}
-    , defaultState = null
-    , noData = "No Data Available."
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var tooltipLabel = pie.description()(e.point) || pie.x()(e.point)
-    var left = e.pos[0] + ( (offsetElement && offsetElement.offsetLeft) || 0 ),
-        top = e.pos[1] + ( (offsetElement && offsetElement.offsetTop) || 0),
-        y = pie.valueFormat()(pie.y()(e.point)),
-        content = tooltip(tooltipLabel, y, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().call(chart); };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-pieChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-pieChart').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-pieWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend
-          .width( availableWidth )
-          .key(pie.x());
-
-        wrap.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        wrap.select('.nv-legendWrap')
-            .attr('transform', 'translate(0,' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      pie
-        .width(availableWidth)
-        .height(availableHeight);
-
-
-      var pieWrap = g.select('.nv-pieWrap')
-          .datum([data]);
-
-      d3.transition(pieWrap).call(pie);
-
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      legend.dispatch.on('stateChange', function(newState) {
-        state = newState;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      pie.dispatch.on('elementMouseout.tooltip', function(e) {
-        dispatch.tooltipHide(e);
-      });
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-
-    });
-
-    return chart;
-  }
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  pie.dispatch.on('elementMouseover.tooltip', function(e) {
-    e.pos = [e.pos[0] +  margin.left, e.pos[1] + margin.top];
-    dispatch.tooltipShow(e);
-  });
-
-  dispatch.on('tooltipShow', function(e) {
-    if (tooltips) showTooltip(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.legend = legend;
-  chart.dispatch = dispatch;
-  chart.pie = pie;
-
-  d3.rebind(chart, pie, 'valueFormat', 'values', 'x', 'y', 'description', 'id', 'showLabels', 'donutLabelsOutside', 'pieLabelsOutside', 'labelType', 'donut', 'donutRatio', 'labelThreshold');
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    pie.color(color);
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.scatter = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin       = {top: 0, right: 0, bottom: 0, left: 0}
-    , width        = 960
-    , height       = 500
-    , color        = nv.utils.defaultColor() // chooses color
-    , id           = Math.floor(Math.random() * 100000) //Create semi-unique ID incase user doesn't select one
-    , x            = d3.scale.linear()
-    , y            = d3.scale.linear()
-    , z            = d3.scale.linear() //linear because d3.svg.shape.size is treated as area
-    , getX         = function(d) { return d.x } // accessor to get the x value
-    , getY         = function(d) { return d.y } // accessor to get the y value
-    , getSize      = function(d) { return d.size || 1} // accessor to get the point size
-    , getShape     = function(d) { return d.shape || 'circle' } // accessor to get point shape
-    , onlyCircles  = true // Set to false to use shapes
-    , forceX       = [] // List of numbers to Force into the X scale (ie. 0, or a max / min, etc.)
-    , forceY       = [] // List of numbers to Force into the Y scale
-    , forceSize    = [] // List of numbers to Force into the Size scale
-    , interactive  = true // If true, plots a voronoi overlay for advanced point intersection
-    , pointKey     = null
-    , pointActive  = function(d) { return !d.notActive } // any points that return false will be filtered out
-    , padData      = false // If true, adds half a data points width to front and back, for lining up a line chart with a bar chart
-    , padDataOuter = .1 //outerPadding to imitate ordinal scale outer padding
-    , clipEdge     = false // if true, masks points within x and y scale
-    , clipVoronoi  = true // if true, masks each point with a circle... can turn off to slightly increase performance
-    , clipRadius   = function() { return 25 } // function to get the radius for voronoi point clips
-    , xDomain      = null // Override x domain (skips the calculation from data)
-    , yDomain      = null // Override y domain
-    , xRange       = null // Override x range
-    , yRange       = null // Override y range
-    , sizeDomain   = null // Override point size domain
-    , sizeRange    = null
-    , singlePoint  = false
-    , dispatch     = d3.dispatch('elementClick', 'elementMouseover', 'elementMouseout')
-    , useVoronoi   = true
-    ;
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0, z0 // used to store previous scales
-    , timeoutID
-    , needsUpdate = false // Flag for when the points are visually updating, but the interactive layer is behind, to disable tooltips
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-      //add series index to each data point for reference
-      data = data.map(function(series, i) {
-        series.values = series.values.map(function(point) {
-          point.series = i;
-          return point;
-        });
-        return series;
-      });
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      // remap and flatten the data for use in calculating the scales' domains
-      var seriesData = (xDomain && yDomain && sizeDomain) ? [] : // if we know xDomain and yDomain and sizeDomain, no need to calculate.... if Size is constant remember to set sizeDomain to speed up performance
-            d3.merge(
-              data.map(function(d) {
-                return d.values.map(function(d,i) {
-                  return { x: getX(d,i), y: getY(d,i), size: getSize(d,i) }
-                })
-              })
-            );
-
-      x   .domain(xDomain || d3.extent(seriesData.map(function(d) { return d.x; }).concat(forceX)))
-
-      if (padData && data[0])
-        x.range(xRange || [(availableWidth * padDataOuter +  availableWidth) / (2 *data[0].values.length), availableWidth - availableWidth * (1 + padDataOuter) / (2 * data[0].values.length)  ]);
-        //x.range([availableWidth * .5 / data[0].values.length, availableWidth * (data[0].values.length - .5)  / data[0].values.length ]);
-      else
-        x.range(xRange || [0, availableWidth]);
-
-      y   .domain(yDomain || d3.extent(seriesData.map(function(d) { return d.y }).concat(forceY)))
-          .range(yRange || [availableHeight, 0]);
-
-      z   .domain(sizeDomain || d3.extent(seriesData.map(function(d) { return d.size }).concat(forceSize)))
-          .range(sizeRange || [16, 256]);
-
-      // If scale's domain don't have a range, slightly adjust to make one... so a chart can show a single data point
-      if (x.domain()[0] === x.domain()[1] || y.domain()[0] === y.domain()[1]) singlePoint = true;
-      if (x.domain()[0] === x.domain()[1])
-        x.domain()[0] ?
-            x.domain([x.domain()[0] - x.domain()[0] * 0.01, x.domain()[1] + x.domain()[1] * 0.01])
-          : x.domain([-1,1]);
-
-      if (y.domain()[0] === y.domain()[1])
-        y.domain()[0] ?
-            y.domain([y.domain()[0] - y.domain()[0] * 0.01, y.domain()[1] + y.domain()[1] * 0.01])
-          : y.domain([-1,1]);
-
-      if ( isNaN(x.domain()[0])) {
-          x.domain([-1,1]);
-      }
-
-      if ( isNaN(y.domain()[0])) {
-          y.domain([-1,1]);
-      }
-
-
-      x0 = x0 || x;
-      y0 = y0 || y;
-      z0 = z0 || z;
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-scatter').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-scatter nv-chart-' + id + (singlePoint ? ' nv-single-point' : ''));
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-groups');
-      gEnter.append('g').attr('class', 'nv-point-paths');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-edge-clip-' + id)
-        .append('rect');
-
-      wrap.select('#nv-edge-clip-' + id + ' rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      g   .attr('clip-path', clipEdge ? 'url(#nv-edge-clip-' + id + ')' : '');
-
-
-      function updateInteractiveLayer() {
-
-        if (!interactive) return false;
-
-        var eventElements;
-
-        var vertices = d3.merge(data.map(function(group, groupIndex) {
-            return group.values
-              .map(function(point, pointIndex) {
-                // *Adding noise to make duplicates very unlikely
-                // *Injecting series and point index for reference
-                /* *Adding a 'jitter' to the points, because there's an issue in d3.geom.voronoi.
-                */
-                var pX = getX(point,pointIndex);
-                var pY = getY(point,pointIndex);
-
-                return [x(pX)+ Math.random() * 1e-7, 
-                        y(pY)+ Math.random() * 1e-7, 
-                        groupIndex, 
-                        pointIndex, point]; //temp hack to add noise untill I think of a better way so there are no duplicates
-              })
-              .filter(function(pointArray, pointIndex) {
-                return pointActive(pointArray[4], pointIndex); // Issue #237.. move filter to after map, so pointIndex is correct!
-              })
-          })
-        );
-
-
-
-        //inject series and point index for reference into voronoi
-        if (useVoronoi === true) {
-
-          if (clipVoronoi) {
-            var pointClipsEnter = wrap.select('defs').selectAll('.nv-point-clips')
-                .data([id])
-              .enter();
-
-            pointClipsEnter.append('clipPath')
-                  .attr('class', 'nv-point-clips')
-                  .attr('id', 'nv-points-clip-' + id);
-
-            var pointClips = wrap.select('#nv-points-clip-' + id).selectAll('circle')
-                .data(vertices);
-            pointClips.enter().append('circle')
-                .attr('r', clipRadius);
-            pointClips.exit().remove();
-            pointClips
-                .attr('cx', function(d) { return d[0] })
-                .attr('cy', function(d) { return d[1] });
-
-            wrap.select('.nv-point-paths')
-                .attr('clip-path', 'url(#nv-points-clip-' + id + ')');
-          }
-
-
-          if(vertices.length) {
-            // Issue #283 - Adding 2 dummy points to the voronoi b/c voronoi requires min 3 points to work
-            vertices.push([x.range()[0] - 20, y.range()[0] - 20, null, null]);
-            vertices.push([x.range()[1] + 20, y.range()[1] + 20, null, null]);
-            vertices.push([x.range()[0] - 20, y.range()[0] + 20, null, null]);
-            vertices.push([x.range()[1] + 20, y.range()[1] - 20, null, null]);
-          }
-
-          var bounds = d3.geom.polygon([
-              [-10,-10],
-              [-10,height + 10],
-              [width + 10,height + 10],
-              [width + 10,-10]
-          ]);
-
-          var voronoi = d3.geom.voronoi(vertices).map(function(d, i) {
-              return {
-                'data': bounds.clip(d),
-                'series': vertices[i][2],
-                'point': vertices[i][3]
-              }
-            });
-
-
-          var pointPaths = wrap.select('.nv-point-paths').selectAll('path')
-              .data(voronoi);
-          pointPaths.enter().append('path')
-              .attr('class', function(d,i) { return 'nv-path-'+i; });
-          pointPaths.exit().remove();
-          pointPaths
-              .attr('d', function(d) {
-                if (d.data.length === 0) 
-                    return 'M 0 0'
-                else 
-                    return 'M' + d.data.join('L') + 'Z'; 
-              });
-
-          var mouseEventCallback = function(d,mDispatch) {
-                if (needsUpdate) return 0;
-                var series = data[d.series];
-                if (typeof series === 'undefined') return;
-                
-                var point  = series.values[d.point];
-
-                mDispatch({
-                  point: point,
-                  series: series,
-                  pos: [x(getX(point, d.point)) + margin.left, y(getY(point, d.point)) + margin.top],
-                  seriesIndex: d.series,
-                  pointIndex: d.point
-                });
-          };
-
-          pointPaths
-              .on('click', function(d) {
-                mouseEventCallback(d, dispatch.elementClick);
-              })
-              .on('mouseover', function(d) {
-                mouseEventCallback(d, dispatch.elementMouseover);
-              })
-              .on('mouseout', function(d, i) {
-                mouseEventCallback(d, dispatch.elementMouseout);
-              });
-
-
-        } else {
-          /*
-          // bring data in form needed for click handlers
-          var dataWithPoints = vertices.map(function(d, i) {
-              return {
-                'data': d,
-                'series': vertices[i][2],
-                'point': vertices[i][3]
-              }
-            });
-           */
-
-          // add event handlers to points instead voronoi paths
-          wrap.select('.nv-groups').selectAll('.nv-group')
-            .selectAll('.nv-point')
-              //.data(dataWithPoints)
-              //.style('pointer-events', 'auto') // recativate events, disabled by css
-              .on('click', function(d,i) { 
-                //nv.log('test', d, i);
-                if (needsUpdate || !data[d.series]) return 0; //check if this is a dummy point
-                var series = data[d.series],
-                    point  = series.values[i];
-
-                dispatch.elementClick({
-                  point: point,
-                  series: series,
-                  pos: [x(getX(point, i)) + margin.left, y(getY(point, i)) + margin.top],
-                  seriesIndex: d.series,
-                  pointIndex: i
-                });
-              })
-              .on('mouseover', function(d,i) {
-                if (needsUpdate || !data[d.series]) return 0; //check if this is a dummy point
-                var series = data[d.series],
-                    point  = series.values[i];
-
-                dispatch.elementMouseover({
-                  point: point,
-                  series: series,
-                  pos: [x(getX(point, i)) + margin.left, y(getY(point, i)) + margin.top],
-                  seriesIndex: d.series,
-                  pointIndex: i
-                });
-              })
-              .on('mouseout', function(d,i) {
-                if (needsUpdate || !data[d.series]) return 0; //check if this is a dummy point
-                var series = data[d.series],
-                    point  = series.values[i];
-
-                dispatch.elementMouseout({
-                  point: point,
-                  series: series,
-                  seriesIndex: d.series,
-                  pointIndex: i
-                });
-              });
-          }
-
-          needsUpdate = false;
-      }
-
-      needsUpdate = true;
-
-      var groups = wrap.select('.nv-groups').selectAll('.nv-group')
-          .data(function(d) { return d }, function(d) { return d.key });
-      groups.enter().append('g')
-          .style('stroke-opacity', 1e-6)
-          .style('fill-opacity', 1e-6);
-      groups.exit()
-          .remove();
-      groups
-          .attr('class', function(d,i) { return 'nv-group nv-series-' + i })
-          .classed('hover', function(d) { return d.hover });
-      groups
-          .transition()
-          .style('fill', function(d,i) { return color(d, i) })
-          .style('stroke', function(d,i) { return color(d, i) })
-          .style('stroke-opacity', 1)
-          .style('fill-opacity', .5);
-
-
-      if (onlyCircles) {
-
-        var points = groups.selectAll('circle.nv-point')
-            .data(function(d) { return d.values }, pointKey);
-        points.enter().append('circle')
-            .style('fill', function (d,i) { return d.color })
-            .style('stroke', function (d,i) { return d.color })
-            .attr('cx', function(d,i) { return nv.utils.NaNtoZero(x0(getX(d,i))) })
-            .attr('cy', function(d,i) { return nv.utils.NaNtoZero(y0(getY(d,i))) })
-            .attr('r', function(d,i) { return Math.sqrt(z(getSize(d,i))/Math.PI) });
-        points.exit().remove();
-        groups.exit().selectAll('path.nv-point').transition()
-            .attr('cx', function(d,i) { return nv.utils.NaNtoZero(x(getX(d,i))) })
-            .attr('cy', function(d,i) { return nv.utils.NaNtoZero(y(getY(d,i))) })
-            .remove();
-        points.each(function(d,i) {
-          d3.select(this)
-            .classed('nv-point', true)
-            .classed('nv-point-' + i, true)
-            .classed('hover',false)
-            ;
-        });
-        points.transition()
-            .attr('cx', function(d,i) { return nv.utils.NaNtoZero(x(getX(d,i))) })
-            .attr('cy', function(d,i) { return nv.utils.NaNtoZero(y(getY(d,i))) })
-            .attr('r', function(d,i) { return Math.sqrt(z(getSize(d,i))/Math.PI) });
-
-      } else {
-
-        var points = groups.selectAll('path.nv-point')
-            .data(function(d) { return d.values });
-        points.enter().append('path')
-            .style('fill', function (d,i) { return d.color })
-            .style('stroke', function (d,i) { return d.color })
-            .attr('transform', function(d,i) {
-              return 'translate(' + x0(getX(d,i)) + ',' + y0(getY(d,i)) + ')'
-            })
-            .attr('d',
-              d3.svg.symbol()
-                .type(getShape)
-                .size(function(d,i) { return z(getSize(d,i)) })
-            );
-        points.exit().remove();
-        groups.exit().selectAll('path.nv-point')
-            .transition()
-            .attr('transform', function(d,i) {
-              return 'translate(' + x(getX(d,i)) + ',' + y(getY(d,i)) + ')'
-            })
-            .remove();
-        points.each(function(d,i) {
-          d3.select(this)
-            .classed('nv-point', true)
-            .classed('nv-point-' + i, true)
-            .classed('hover',false)
-            ;
-        });
-        points.transition()
-            .attr('transform', function(d,i) {
-              //nv.log(d,i,getX(d,i), x(getX(d,i)));
-              return 'translate(' + x(getX(d,i)) + ',' + y(getY(d,i)) + ')'
-            })
-            .attr('d',
-              d3.svg.symbol()
-                .type(getShape)
-                .size(function(d,i) { return z(getSize(d,i)) })
-            );
-      }
-
-
-      // Delay updating the invisible interactive layer for smoother animation
-      clearTimeout(timeoutID); // stop repeat calls to updateInteractiveLayer
-      timeoutID = setTimeout(updateInteractiveLayer, 300);
-      //updateInteractiveLayer();
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-      z0 = z.copy();
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-  chart.clearHighlights = function() {
-      //Remove the 'hover' class from all highlighted points.
-      d3.selectAll(".nv-chart-" + id + " .nv-point.hover").classed("hover",false);
-  };
-
-  chart.highlightPoint = function(seriesIndex,pointIndex,isHoverOver) {
-      d3.select(".nv-chart-" + id + " .nv-series-" + seriesIndex + " .nv-point-" + pointIndex)
-          .classed("hover",isHoverOver); 
-  };
-
-
-  dispatch.on('elementMouseover.point', function(d) {
-     if (interactive) chart.highlightPoint(d.seriesIndex,d.pointIndex,true);
-  });
-
-  dispatch.on('elementMouseout.point', function(d) {
-     if (interactive) chart.highlightPoint(d.seriesIndex,d.pointIndex,false);
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = d3.functor(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = d3.functor(_);
-    return chart;
-  };
-
-  chart.size = function(_) {
-    if (!arguments.length) return getSize;
-    getSize = d3.functor(_);
-    return chart;
-  };
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.zScale = function(_) {
-    if (!arguments.length) return z;
-    z = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.sizeDomain = function(_) {
-    if (!arguments.length) return sizeDomain;
-    sizeDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.sizeRange = function(_) {
-    if (!arguments.length) return sizeRange;
-    sizeRange = _;
-    return chart;
-  };
-
-  chart.forceX = function(_) {
-    if (!arguments.length) return forceX;
-    forceX = _;
-    return chart;
-  };
-
-  chart.forceY = function(_) {
-    if (!arguments.length) return forceY;
-    forceY = _;
-    return chart;
-  };
-
-  chart.forceSize = function(_) {
-    if (!arguments.length) return forceSize;
-    forceSize = _;
-    return chart;
-  };
-
-  chart.interactive = function(_) {
-    if (!arguments.length) return interactive;
-    interactive = _;
-    return chart;
-  };
-
-  chart.pointKey = function(_) {
-    if (!arguments.length) return pointKey;
-    pointKey = _;
-    return chart;
-  };
-
-  chart.pointActive = function(_) {
-    if (!arguments.length) return pointActive;
-    pointActive = _;
-    return chart;
-  };
-
-  chart.padData = function(_) {
-    if (!arguments.length) return padData;
-    padData = _;
-    return chart;
-  };
-
-  chart.padDataOuter = function(_) {
-    if (!arguments.length) return padDataOuter;
-    padDataOuter = _;
-    return chart;
-  };
-
-  chart.clipEdge = function(_) {
-    if (!arguments.length) return clipEdge;
-    clipEdge = _;
-    return chart;
-  };
-
-  chart.clipVoronoi= function(_) {
-    if (!arguments.length) return clipVoronoi;
-    clipVoronoi = _;
-    return chart;
-  };
-
-  chart.useVoronoi= function(_) {
-    if (!arguments.length) return useVoronoi;
-    useVoronoi = _;
-    if (useVoronoi === false) {
-        clipVoronoi = false;
-    }
-    return chart;
-  };
-
-  chart.clipRadius = function(_) {
-    if (!arguments.length) return clipRadius;
-    clipRadius = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.shape = function(_) {
-    if (!arguments.length) return getShape;
-    getShape = _;
-    return chart;
-  };
-
-  chart.onlyCircles = function(_) {
-    if (!arguments.length) return onlyCircles;
-    onlyCircles = _;
-    return chart;
-  };
-
-  chart.id = function(_) {
-    if (!arguments.length) return id;
-    id = _;
-    return chart;
-  };
-
-  chart.singlePoint = function(_) {
-    if (!arguments.length) return singlePoint;
-    singlePoint = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-nv.models.scatterChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var scatter      = nv.models.scatter()
-    , xAxis        = nv.models.axis()
-    , yAxis        = nv.models.axis()
-    , legend       = nv.models.legend()
-    , controls     = nv.models.legend()
-    , distX        = nv.models.distribution()
-    , distY        = nv.models.distribution()
-    ;
-
-  var margin       = {top: 30, right: 20, bottom: 50, left: 75}
-    , width        = null
-    , height       = null
-    , color        = nv.utils.defaultColor()
-    , x            = d3.fisheye ? d3.fisheye.scale(d3.scale.linear).distortion(0) : scatter.xScale()
-    , y            = d3.fisheye ? d3.fisheye.scale(d3.scale.linear).distortion(0) : scatter.yScale()
-    , xPadding     = 0
-    , yPadding     = 0
-    , showDistX    = false
-    , showDistY    = false
-    , showLegend   = true
-    , showXAxis    = true
-    , showYAxis    = true
-    , rightAlignYAxis = false
-    , showControls = !!d3.fisheye
-    , fisheye      = 0
-    , pauseFisheye = false
-    , tooltips     = true
-    , tooltipX     = function(key, x, y) { return '<strong>' + x + '</strong>' }
-    , tooltipY     = function(key, x, y) { return '<strong>' + y + '</strong>' }
-    , tooltip      = null
-    , state = {}
-    , defaultState = null
-    , dispatch     = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , noData       = "No Data Available."
-    , transitionDuration = 250
-    ;
-
-  scatter
-    .xScale(x)
-    .yScale(y)
-    ;
-  xAxis
-    .orient('bottom')
-    .tickPadding(10)
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    .tickPadding(10)
-    ;
-  distX
-    .axis('x')
-    ;
-  distY
-    .axis('y')
-    ;
-
-  controls.updateState(false);
-
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0;
-
-  var showTooltip = function(e, offsetElement) {
-    //TODO: make tooltip style an option between single or dual on axes (maybe on all charts with axes?)
-
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        leftX = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        topX = y.range()[0] + margin.top + ( offsetElement.offsetTop || 0),
-        leftY = x.range()[0] + margin.left + ( offsetElement.offsetLeft || 0 ),
-        topY = e.pos[1] + ( offsetElement.offsetTop || 0),
-        xVal = xAxis.tickFormat()(scatter.x()(e.point, e.pointIndex)),
-        yVal = yAxis.tickFormat()(scatter.y()(e.point, e.pointIndex));
-
-      if( tooltipX != null )
-          nv.tooltip.show([leftX, topX], tooltipX(e.series.key, xVal, yVal, e, chart), 'n', 1, offsetElement, 'x-nvtooltip');
-      if( tooltipY != null )
-          nv.tooltip.show([leftY, topY], tooltipY(e.series.key, xVal, yVal, e, chart), 'e', 1, offsetElement, 'y-nvtooltip');
-      if( tooltip != null )
-          nv.tooltip.show([left, top], tooltip(e.series.key, xVal, yVal, e, chart), e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  var controlsData = [
-    { key: 'Magnify', disabled: true }
-  ];
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart); };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display noData message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x0 = x0 || x;
-      y0 = y0 || y;
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-scatterChart').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-scatterChart nv-chart-' + scatter.id());
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      // background for pointer events
-      gEnter.append('rect').attr('class', 'nvd3 nv-background');
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-scatterWrap');
-      gEnter.append('g').attr('class', 'nv-distWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-controlsWrap');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        var legendWidth = (showControls) ? availableWidth / 2 : availableWidth;
-        legend.width(legendWidth);
-
-        wrap.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        wrap.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + (availableWidth - legendWidth) + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Controls
-
-      if (showControls) {
-        controls.width(180).color(['#444']);
-        g.select('.nv-controlsWrap')
-            .datum(controlsData)
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-            .call(controls);
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      scatter
-          .width(availableWidth)
-          .height(availableHeight)
-          .color(data.map(function(d,i) {
-            return d.color || color(d, i);
-          }).filter(function(d,i) { return !data[i].disabled }));
-
-      if (xPadding !== 0)
-        scatter.xDomain(null);
-
-      if (yPadding !== 0)
-        scatter.yDomain(null);
-
-      wrap.select('.nv-scatterWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-          .call(scatter);
-
-      //Adjust for x and y padding
-      if (xPadding !== 0) {
-        var xRange = x.domain()[1] - x.domain()[0];
-        scatter.xDomain([x.domain()[0] - (xPadding * xRange), x.domain()[1] + (xPadding * xRange)]);
-      }
-
-      if (yPadding !== 0) {
-        var yRange = y.domain()[1] - y.domain()[0];
-        scatter.yDomain([y.domain()[0] - (yPadding * yRange), y.domain()[1] + (yPadding * yRange)]);
-      }
-
-      //Only need to update the scatter again if x/yPadding changed the domain.
-      if (yPadding !== 0 || xPadding !== 0) {
-        wrap.select('.nv-scatterWrap')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(scatter);
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-      if (showXAxis) {
-        xAxis
-            .scale(x)
-            .ticks( xAxis.ticks() && xAxis.ticks().length ? xAxis.ticks() : availableWidth / 100 )
-            .tickSize( -availableHeight , 0);
-
-        g.select('.nv-x.nv-axis')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')')
-            .call(xAxis);
-
-      }
-
-      if (showYAxis) {
-        yAxis
-            .scale(y)
-            .ticks( yAxis.ticks() && yAxis.ticks().length ? yAxis.ticks() : availableHeight / 36 )
-            .tickSize( -availableWidth, 0);
-
-        g.select('.nv-y.nv-axis')
-            .call(yAxis);
-      }
-
-
-      if (showDistX) {
-        distX
-            .getData(scatter.x())
-            .scale(x)
-            .width(availableWidth)
-            .color(data.map(function(d,i) {
-              return d.color || color(d, i);
-            }).filter(function(d,i) { return !data[i].disabled }));
-        gEnter.select('.nv-distWrap').append('g')
-            .attr('class', 'nv-distributionX');
-        g.select('.nv-distributionX')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distX);
-      }
-
-      if (showDistY) {
-        distY
-            .getData(scatter.y())
-            .scale(y)
-            .width(availableHeight)
-            .color(data.map(function(d,i) {
-              return d.color || color(d, i);
-            }).filter(function(d,i) { return !data[i].disabled }));
-        gEnter.select('.nv-distWrap').append('g')
-            .attr('class', 'nv-distributionY');
-        g.select('.nv-distributionY')
-            .attr('transform', 
-              'translate(' + (rightAlignYAxis ? availableWidth : -distY.size() ) + ',0)')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distY);
-      }
-
-      //------------------------------------------------------------
-
-
-
-
-      if (d3.fisheye) {
-        g.select('.nv-background')
-            .attr('width', availableWidth)
-            .attr('height', availableHeight);
-
-        g.select('.nv-background').on('mousemove', updateFisheye);
-        g.select('.nv-background').on('click', function() { pauseFisheye = !pauseFisheye;});
-        scatter.dispatch.on('elementClick.freezeFisheye', function() {
-          pauseFisheye = !pauseFisheye;
-        });
-      }
-
-
-      function updateFisheye() {
-        if (pauseFisheye) {
-          g.select('.nv-point-paths').style('pointer-events', 'all');
-          return false;
-        }
-
-        g.select('.nv-point-paths').style('pointer-events', 'none' );
-
-        var mouse = d3.mouse(this);
-        x.distortion(fisheye).focus(mouse[0]);
-        y.distortion(fisheye).focus(mouse[1]);
-
-        g.select('.nv-scatterWrap')
-            .call(scatter);
-
-        if (showXAxis)
-          g.select('.nv-x.nv-axis').call(xAxis);
-        
-        if (showYAxis)
-          g.select('.nv-y.nv-axis').call(yAxis);
-        
-        g.select('.nv-distributionX')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distX);
-        g.select('.nv-distributionY')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distY);
-      }
-
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      controls.dispatch.on('legendClick', function(d,i) {
-        d.disabled = !d.disabled;
-
-        fisheye = d.disabled ? 0 : 2.5;
-        g.select('.nv-background') .style('pointer-events', d.disabled ? 'none' : 'all');
-        g.select('.nv-point-paths').style('pointer-events', d.disabled ? 'all' : 'none' );
-
-        if (d.disabled) {
-          x.distortion(fisheye).focus(0);
-          y.distortion(fisheye).focus(0);
-
-          g.select('.nv-scatterWrap').call(scatter);
-          g.select('.nv-x.nv-axis').call(xAxis);
-          g.select('.nv-y.nv-axis').call(yAxis);
-        } else {
-          pauseFisheye = false;
-        }
-
-        chart.update();
-      });
-
-      legend.dispatch.on('stateChange', function(newState) {
-        state.disabled = newState.disabled;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      scatter.dispatch.on('elementMouseover.tooltip', function(e) {
-        d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-distx-' + e.pointIndex)
-            .attr('y1', function(d,i) { return e.pos[1] - availableHeight;});
-        d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-disty-' + e.pointIndex)
-            .attr('x2', e.pos[0] + distX.size());
-
-        e.pos = [e.pos[0] + margin.left, e.pos[1] + margin.top];
-        dispatch.tooltipShow(e);
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  scatter.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-
-    d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-distx-' + e.pointIndex)
-        .attr('y1', 0);
-    d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-disty-' + e.pointIndex)
-        .attr('x2', distY.size());
-  });
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.scatter = scatter;
-  chart.legend = legend;
-  chart.controls = controls;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-  chart.distX = distX;
-  chart.distY = distY;
-
-  d3.rebind(chart, scatter, 'id', 'interactive', 'pointActive', 'x', 'y', 'shape', 'size', 'xScale', 'yScale', 'zScale', 'xDomain', 'yDomain', 'xRange', 'yRange', 'sizeDomain', 'sizeRange', 'forceX', 'forceY', 'forceSize', 'clipVoronoi', 'clipRadius', 'useVoronoi');
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    distX.color(color);
-    distY.color(color);
-    return chart;
-  };
-
-  chart.showDistX = function(_) {
-    if (!arguments.length) return showDistX;
-    showDistX = _;
-    return chart;
-  };
-
-  chart.showDistY = function(_) {
-    if (!arguments.length) return showDistY;
-    showDistY = _;
-    return chart;
-  };
-
-  chart.showControls = function(_) {
-    if (!arguments.length) return showControls;
-    showControls = _;
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-
-  chart.fisheye = function(_) {
-    if (!arguments.length) return fisheye;
-    fisheye = _;
-    return chart;
-  };
-
-  chart.xPadding = function(_) {
-    if (!arguments.length) return xPadding;
-    xPadding = _;
-    return chart;
-  };
-
-  chart.yPadding = function(_) {
-    if (!arguments.length) return yPadding;
-    yPadding = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.tooltipXContent = function(_) {
-    if (!arguments.length) return tooltipX;
-    tooltipX = _;
-    return chart;
-  };
-
-  chart.tooltipYContent = function(_) {
-    if (!arguments.length) return tooltipY;
-    tooltipY = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-  
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.scatterPlusLineChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var scatter      = nv.models.scatter()
-    , xAxis        = nv.models.axis()
-    , yAxis        = nv.models.axis()
-    , legend       = nv.models.legend()
-    , controls     = nv.models.legend()
-    , distX        = nv.models.distribution()
-    , distY        = nv.models.distribution()
-    ;
-
-  var margin       = {top: 30, right: 20, bottom: 50, left: 75}
-    , width        = null
-    , height       = null
-    , color        = nv.utils.defaultColor()
-    , x            = d3.fisheye ? d3.fisheye.scale(d3.scale.linear).distortion(0) : scatter.xScale()
-    , y            = d3.fisheye ? d3.fisheye.scale(d3.scale.linear).distortion(0) : scatter.yScale()
-    , showDistX    = false
-    , showDistY    = false
-    , showLegend   = true
-    , showXAxis    = true
-    , showYAxis    = true
-    , rightAlignYAxis = false
-    , showControls = !!d3.fisheye
-    , fisheye      = 0
-    , pauseFisheye = false
-    , tooltips     = true
-    , tooltipX     = function(key, x, y) { return '<strong>' + x + '</strong>' }
-    , tooltipY     = function(key, x, y) { return '<strong>' + y + '</strong>' }
-    , tooltip      = function(key, x, y, date) { return '<h3>' + key + '</h3>' 
-                                                      + '<p>' + date + '</p>' }
-    , state = {}
-    , defaultState = null
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , noData       = "No Data Available."
-    , transitionDuration = 250
-    ;
-
-  scatter
-    .xScale(x)
-    .yScale(y)
-    ;
-  xAxis
-    .orient('bottom')
-    .tickPadding(10)
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    .tickPadding(10)
-    ;
-  distX
-    .axis('x')
-    ;
-  distY
-    .axis('y')
-    ;
-  
-  controls.updateState(false);
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var x0, y0;
-
-  var showTooltip = function(e, offsetElement) {
-    //TODO: make tooltip style an option between single or dual on axes (maybe on all charts with axes?)
-
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        leftX = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        topX = y.range()[0] + margin.top + ( offsetElement.offsetTop || 0),
-        leftY = x.range()[0] + margin.left + ( offsetElement.offsetLeft || 0 ),
-        topY = e.pos[1] + ( offsetElement.offsetTop || 0),
-        xVal = xAxis.tickFormat()(scatter.x()(e.point, e.pointIndex)),
-        yVal = yAxis.tickFormat()(scatter.y()(e.point, e.pointIndex));
-
-      if( tooltipX != null )
-          nv.tooltip.show([leftX, topX], tooltipX(e.series.key, xVal, yVal, e, chart), 'n', 1, offsetElement, 'x-nvtooltip');
-      if( tooltipY != null )
-          nv.tooltip.show([leftY, topY], tooltipY(e.series.key, xVal, yVal, e, chart), 'e', 1, offsetElement, 'y-nvtooltip');
-      if( tooltip != null )
-          nv.tooltip.show([left, top], tooltip(e.series.key, xVal, yVal, e.point.tooltip, e, chart), e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  var controlsData = [
-    { key: 'Magnify', disabled: true }
-  ];
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart); };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display noData message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = scatter.xScale();
-      y = scatter.yScale();
-
-      x0 = x0 || x;
-      y0 = y0 || y;
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-scatterChart').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-scatterChart nv-chart-' + scatter.id());
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g')
-
-      // background for pointer events
-      gEnter.append('rect').attr('class', 'nvd3 nv-background').style("pointer-events","none");
-
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-scatterWrap');
-      gEnter.append('g').attr('class', 'nv-regressionLinesWrap');
-      gEnter.append('g').attr('class', 'nv-distWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-controlsWrap');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        legend.width( availableWidth / 2 );
-
-        wrap.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        wrap.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + (availableWidth / 2) + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Controls
-
-      if (showControls) {
-        controls.width(180).color(['#444']);
-        g.select('.nv-controlsWrap')
-            .datum(controlsData)
-            .attr('transform', 'translate(0,' + (-margin.top) +')')
-            .call(controls);
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      scatter
-          .width(availableWidth)
-          .height(availableHeight)
-          .color(data.map(function(d,i) {
-            return d.color || color(d, i);
-          }).filter(function(d,i) { return !data[i].disabled }))
-
-      wrap.select('.nv-scatterWrap')
-          .datum(data.filter(function(d) { return !d.disabled }))
-          .call(scatter);
-
-      wrap.select('.nv-regressionLinesWrap')
-          .attr('clip-path', 'url(#nv-edge-clip-' + scatter.id() + ')');
-
-      var regWrap = wrap.select('.nv-regressionLinesWrap').selectAll('.nv-regLines')
-                      .data(function(d) {return d });
-      
-      regWrap.enter().append('g').attr('class', 'nv-regLines');
-
-      var regLine = regWrap.selectAll('.nv-regLine').data(function(d){return [d]});
-      var regLineEnter = regLine.enter()
-                       .append('line').attr('class', 'nv-regLine')
-                       .style('stroke-opacity', 0);
-
-      regLine
-          .transition()
-          .attr('x1', x.range()[0])
-          .attr('x2', x.range()[1])
-          .attr('y1', function(d,i) {return y(x.domain()[0] * d.slope + d.intercept) })
-          .attr('y2', function(d,i) { return y(x.domain()[1] * d.slope + d.intercept) })
-          .style('stroke', function(d,i,j) { return color(d,j) })
-          .style('stroke-opacity', function(d,i) {
-            return (d.disabled || typeof d.slope === 'undefined' || typeof d.intercept === 'undefined') ? 0 : 1 
-          });
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-        xAxis
-            .scale(x)
-            .ticks( xAxis.ticks() ? xAxis.ticks() : availableWidth / 100 )
-            .tickSize( -availableHeight , 0);
-
-        g.select('.nv-x.nv-axis')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')')
-            .call(xAxis);
-      }
-
-      if (showYAxis) {
-        yAxis
-            .scale(y)
-            .ticks( yAxis.ticks() ? yAxis.ticks() : availableHeight / 36 )
-            .tickSize( -availableWidth, 0);
-
-        g.select('.nv-y.nv-axis')
-            .call(yAxis);
-      }
-
-
-      if (showDistX) {
-        distX
-            .getData(scatter.x())
-            .scale(x)
-            .width(availableWidth)
-            .color(data.map(function(d,i) {
-              return d.color || color(d, i);
-            }).filter(function(d,i) { return !data[i].disabled }));
-        gEnter.select('.nv-distWrap').append('g')
-            .attr('class', 'nv-distributionX');
-        g.select('.nv-distributionX')
-            .attr('transform', 'translate(0,' + y.range()[0] + ')')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distX);
-      }
-
-      if (showDistY) {
-        distY
-            .getData(scatter.y())
-            .scale(y)
-            .width(availableHeight)
-            .color(data.map(function(d,i) {
-              return d.color || color(d, i);
-            }).filter(function(d,i) { return !data[i].disabled }));
-        gEnter.select('.nv-distWrap').append('g')
-            .attr('class', 'nv-distributionY');
-        g.select('.nv-distributionY')
-            .attr('transform', 'translate(' + (rightAlignYAxis ? availableWidth : -distY.size() ) + ',0)')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distY);
-      }
-
-      //------------------------------------------------------------
-
-
-
-
-      if (d3.fisheye) {
-        g.select('.nv-background')
-            .attr('width', availableWidth)
-            .attr('height', availableHeight)
-            ;
-
-        g.select('.nv-background').on('mousemove', updateFisheye);
-        g.select('.nv-background').on('click', function() { pauseFisheye = !pauseFisheye;});
-        scatter.dispatch.on('elementClick.freezeFisheye', function() {
-          pauseFisheye = !pauseFisheye;
-        });
-      }
-
-
-      function updateFisheye() {
-        if (pauseFisheye) {
-          g.select('.nv-point-paths').style('pointer-events', 'all');
-          return false;
-        }
-
-        g.select('.nv-point-paths').style('pointer-events', 'none' );
-
-        var mouse = d3.mouse(this);
-        x.distortion(fisheye).focus(mouse[0]);
-        y.distortion(fisheye).focus(mouse[1]);
-
-        g.select('.nv-scatterWrap')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(scatter);
-
-        if (showXAxis)
-          g.select('.nv-x.nv-axis').call(xAxis);
-
-        if (showYAxis)
-          g.select('.nv-y.nv-axis').call(yAxis);
-        
-        g.select('.nv-distributionX')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distX);
-        g.select('.nv-distributionY')
-            .datum(data.filter(function(d) { return !d.disabled }))
-            .call(distY);
-      }
-
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      controls.dispatch.on('legendClick', function(d,i) {
-        d.disabled = !d.disabled;
-
-        fisheye = d.disabled ? 0 : 2.5;
-        g.select('.nv-background') .style('pointer-events', d.disabled ? 'none' : 'all');
-        g.select('.nv-point-paths').style('pointer-events', d.disabled ? 'all' : 'none' );
-
-        if (d.disabled) {
-          x.distortion(fisheye).focus(0);
-          y.distortion(fisheye).focus(0);
-
-          g.select('.nv-scatterWrap').call(scatter);
-          g.select('.nv-x.nv-axis').call(xAxis);
-          g.select('.nv-y.nv-axis').call(yAxis);
-        } else {
-          pauseFisheye = false;
-        }
-
-        chart.update();
-      });
-
-      legend.dispatch.on('stateChange', function(newState) { 
-        state = newState;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-
-      scatter.dispatch.on('elementMouseover.tooltip', function(e) {
-        d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-distx-' + e.pointIndex)
-            .attr('y1', e.pos[1] - availableHeight);
-        d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-disty-' + e.pointIndex)
-            .attr('x2', e.pos[0] + distX.size());
-
-        e.pos = [e.pos[0] + margin.left, e.pos[1] + margin.top];
-        dispatch.tooltipShow(e);
-      });
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        chart.update();
-      });
-
-      //============================================================
-
-
-      //store old scales for use in transitions on update
-      x0 = x.copy();
-      y0 = y.copy();
-
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  scatter.dispatch.on('elementMouseout.tooltip', function(e) {
-    dispatch.tooltipHide(e);
-
-    d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-distx-' + e.pointIndex)
-        .attr('y1', 0);
-    d3.select('.nv-chart-' + scatter.id() + ' .nv-series-' + e.seriesIndex + ' .nv-disty-' + e.pointIndex)
-        .attr('x2', distY.size());
-  });
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.scatter = scatter;
-  chart.legend = legend;
-  chart.controls = controls;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-  chart.distX = distX;
-  chart.distY = distY;
-
-  d3.rebind(chart, scatter, 'id', 'interactive', 'pointActive', 'x', 'y', 'shape', 'size', 'xScale', 'yScale', 'zScale', 'xDomain', 'yDomain', 'xRange', 'yRange', 'sizeDomain', 'sizeRange', 'forceX', 'forceY', 'forceSize', 'clipVoronoi', 'clipRadius', 'useVoronoi');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    distX.color(color);
-    distY.color(color);
-    return chart;
-  };
-
-  chart.showDistX = function(_) {
-    if (!arguments.length) return showDistX;
-    showDistX = _;
-    return chart;
-  };
-
-  chart.showDistY = function(_) {
-    if (!arguments.length) return showDistY;
-    showDistY = _;
-    return chart;
-  };
-
-  chart.showControls = function(_) {
-    if (!arguments.length) return showControls;
-    showControls = _;
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.fisheye = function(_) {
-    if (!arguments.length) return fisheye;
-    fisheye = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.tooltipXContent = function(_) {
-    if (!arguments.length) return tooltipX;
-    tooltipX = _;
-    return chart;
-  };
-
-  chart.tooltipYContent = function(_) {
-    if (!arguments.length) return tooltipY;
-    tooltipY = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.sparkline = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 2, right: 0, bottom: 2, left: 0}
-    , width = 400
-    , height = 32
-    , animate = true
-    , x = d3.scale.linear()
-    , y = d3.scale.linear()
-    , getX = function(d) { return d.x }
-    , getY = function(d) { return d.y }
-    , color = nv.utils.getColor(['#000'])
-    , xDomain
-    , yDomain
-    , xRange
-    , yRange
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x   .domain(xDomain || d3.extent(data, getX ))
-          .range(xRange || [0, availableWidth]);
-
-      y   .domain(yDomain || d3.extent(data, getY ))
-          .range(yRange || [availableHeight, 0]);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-sparkline').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-sparkline');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')')
-
-      //------------------------------------------------------------
-
-
-      var paths = wrap.selectAll('path')
-          .data(function(d) { return [d] });
-      paths.enter().append('path');
-      paths.exit().remove();
-      paths
-          .style('stroke', function(d,i) { return d.color || color(d, i) })
-          .attr('d', d3.svg.line()
-            .x(function(d,i) { return x(getX(d,i)) })
-            .y(function(d,i) { return y(getY(d,i)) })
-          );
-
-
-      // TODO: Add CURRENT data point (Need Min, Mac, Current / Most recent)
-      var points = wrap.selectAll('circle.nv-point')
-          .data(function(data) {
-              var yValues = data.map(function(d, i) { return getY(d,i); });
-              function pointIndex(index) {
-                  if (index != -1) {
-	              var result = data[index];
-                      result.pointIndex = index;
-                      return result;
-                  } else {
-                      return null;
-                  }
-              }
-              var maxPoint = pointIndex(yValues.lastIndexOf(y.domain()[1])),
-                  minPoint = pointIndex(yValues.indexOf(y.domain()[0])),
-                  currentPoint = pointIndex(yValues.length - 1);
-              return [minPoint, maxPoint, currentPoint].filter(function (d) {return d != null;});
-          });
-      points.enter().append('circle');
-      points.exit().remove();
-      points
-          .attr('cx', function(d,i) { return x(getX(d,d.pointIndex)) })
-          .attr('cy', function(d,i) { return y(getY(d,d.pointIndex)) })
-          .attr('r', 2)
-          .attr('class', function(d,i) {
-            return getX(d, d.pointIndex) == x.domain()[1] ? 'nv-point nv-currentValue' :
-                   getY(d, d.pointIndex) == y.domain()[0] ? 'nv-point nv-minValue' : 'nv-point nv-maxValue'
-          });
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = d3.functor(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = d3.functor(_);
-    return chart;
-  };
-
-  chart.xScale = function(_) {
-    if (!arguments.length) return x;
-    x = _;
-    return chart;
-  };
-
-  chart.yScale = function(_) {
-    if (!arguments.length) return y;
-    y = _;
-    return chart;
-  };
-
-  chart.xDomain = function(_) {
-    if (!arguments.length) return xDomain;
-    xDomain = _;
-    return chart;
-  };
-
-  chart.yDomain = function(_) {
-    if (!arguments.length) return yDomain;
-    yDomain = _;
-    return chart;
-  };
-
-  chart.xRange = function(_) {
-    if (!arguments.length) return xRange;
-    xRange = _;
-    return chart;
-  };
-
-  chart.yRange = function(_) {
-    if (!arguments.length) return yRange;
-    yRange = _;
-    return chart;
-  };
-
-  chart.animate = function(_) {
-    if (!arguments.length) return animate;
-    animate = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.sparklinePlus = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var sparkline = nv.models.sparkline();
-
-  var margin = {top: 15, right: 100, bottom: 10, left: 50}
-    , width = null
-    , height = null
-    , x
-    , y
-    , index = []
-    , paused = false
-    , xTickFormat = d3.format(',r')
-    , yTickFormat = d3.format(',.2f')
-    , showValue = true
-    , alignValue = true
-    , rightAlignValue = false
-    , noData = "No Data Available."
-    ;
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this);
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      
-
-      chart.update = function() { chart(selection) };
-      chart.container = this;
-
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      var currentValue = sparkline.y()(data[data.length-1], data.length-1);
-
-      //------------------------------------------------------------
-
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = sparkline.xScale();
-      y = sparkline.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-sparklineplus').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-sparklineplus');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-sparklineWrap');
-      gEnter.append('g').attr('class', 'nv-valueWrap');
-      gEnter.append('g').attr('class', 'nv-hoverArea');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      var sparklineWrap = g.select('.nv-sparklineWrap');
-
-      sparkline
-        .width(availableWidth)
-        .height(availableHeight);
-
-      sparklineWrap
-          .call(sparkline);
-
-      //------------------------------------------------------------
-
-
-      var valueWrap = g.select('.nv-valueWrap');
-      
-      var value = valueWrap.selectAll('.nv-currentValue')
-          .data([currentValue]);
-
-      value.enter().append('text').attr('class', 'nv-currentValue')
-          .attr('dx', rightAlignValue ? -8 : 8)
-          .attr('dy', '.9em')
-          .style('text-anchor', rightAlignValue ? 'end' : 'start');
-
-      value
-          .attr('x', availableWidth + (rightAlignValue ? margin.right : 0))
-          .attr('y', alignValue ? function(d) { return y(d) } : 0)
-          .style('fill', sparkline.color()(data[data.length-1], data.length-1))
-          .text(yTickFormat(currentValue));
-
-
-
-      gEnter.select('.nv-hoverArea').append('rect')
-          .on('mousemove', sparklineHover)
-          .on('click', function() { paused = !paused })
-          .on('mouseout', function() { index = []; updateValueLine(); });
-          //.on('mouseout', function() { index = null; updateValueLine(); });
-
-      g.select('.nv-hoverArea rect')
-          .attr('transform', function(d) { return 'translate(' + -margin.left + ',' + -margin.top + ')' })
-          .attr('width', availableWidth + margin.left + margin.right)
-          .attr('height', availableHeight + margin.top);
-
-
-
-      function updateValueLine() { //index is currently global (within the chart), may or may not keep it that way
-        if (paused) return;
-
-        var hoverValue = g.selectAll('.nv-hoverValue').data(index)
-
-        var hoverEnter = hoverValue.enter()
-          .append('g').attr('class', 'nv-hoverValue')
-            .style('stroke-opacity', 0)
-            .style('fill-opacity', 0);
-
-        hoverValue.exit()
-          .transition().duration(250)
-            .style('stroke-opacity', 0)
-            .style('fill-opacity', 0)
-            .remove();
-
-        hoverValue
-            .attr('transform', function(d) { return 'translate(' + x(sparkline.x()(data[d],d)) + ',0)' })
-          .transition().duration(250)
-            .style('stroke-opacity', 1)
-            .style('fill-opacity', 1);
-
-        if (!index.length) return;
-
-        hoverEnter.append('line')
-            .attr('x1', 0)
-            .attr('y1', -margin.top)
-            .attr('x2', 0)
-            .attr('y2', availableHeight);
-
-
-        hoverEnter.append('text').attr('class', 'nv-xValue')
-            .attr('x', -6)
-            .attr('y', -margin.top)
-            .attr('text-anchor', 'end')
-            .attr('dy', '.9em')
-
-
-        g.select('.nv-hoverValue .nv-xValue')
-            .text(xTickFormat(sparkline.x()(data[index[0]], index[0])));
-
-        hoverEnter.append('text').attr('class', 'nv-yValue')
-            .attr('x', 6)
-            .attr('y', -margin.top)
-            .attr('text-anchor', 'start')
-            .attr('dy', '.9em')
-
-        g.select('.nv-hoverValue .nv-yValue')
-            .text(yTickFormat(sparkline.y()(data[index[0]], index[0])));
-
-      }
-
-
-      function sparklineHover() {
-        if (paused) return;
-
-        var pos = d3.mouse(this)[0] - margin.left;
-
-        function getClosestIndex(data, x) {
-          var distance = Math.abs(sparkline.x()(data[0], 0) - x);
-          var closestIndex = 0;
-          for (var i = 0; i < data.length; i++){
-            if (Math.abs(sparkline.x()(data[i], i) - x) < distance) {
-              distance = Math.abs(sparkline.x()(data[i], i) - x);
-              closestIndex = i;
-            }
-          }
-          return closestIndex;
-        }
-
-        index = [getClosestIndex(data, Math.round(x.invert(pos)))];
-
-        updateValueLine();
-      }
-
-    });
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.sparkline = sparkline;
-
-  d3.rebind(chart, sparkline, 'x', 'y', 'xScale', 'yScale', 'color');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.xTickFormat = function(_) {
-    if (!arguments.length) return xTickFormat;
-    xTickFormat = _;
-    return chart;
-  };
-
-  chart.yTickFormat = function(_) {
-    if (!arguments.length) return yTickFormat;
-    yTickFormat = _;
-    return chart;
-  };
-
-  chart.showValue = function(_) {
-    if (!arguments.length) return showValue;
-    showValue = _;
-    return chart;
-  };
-
-  chart.alignValue = function(_) {
-    if (!arguments.length) return alignValue;
-    alignValue = _;
-    return chart;
-  };
-
-  chart.rightAlignValue = function(_) {
-    if (!arguments.length) return rightAlignValue;
-    rightAlignValue = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.stackedArea = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var margin = {top: 0, right: 0, bottom: 0, left: 0}
-    , width = 960
-    , height = 500
-    , color = nv.utils.defaultColor() // a function that computes the color
-    , id = Math.floor(Math.random() * 100000) //Create semi-unique ID incase user doesn't selet one
-    , getX = function(d) { return d.x } // accessor to get the x value from a data point
-    , getY = function(d) { return d.y } // accessor to get the y value from a data point
-    , style = 'stack'
-    , offset = 'zero'
-    , order = 'default'
-    , interpolate = 'linear'  // controls the line interpolation
-    , clipEdge = false // if true, masks lines within x and y scale
-    , x //can be accessed via chart.xScale()
-    , y //can be accessed via chart.yScale()
-    , scatter = nv.models.scatter()
-    , dispatch =  d3.dispatch('tooltipShow', 'tooltipHide', 'areaClick', 'areaMouseover', 'areaMouseout')
-    ;
-
-  scatter
-    .size(2.2) // default size
-    .sizeDomain([2.2,2.2]) // all the same size by default
-    ;
-
-  /************************************
-   * offset:
-   *   'wiggle' (stream)
-   *   'zero' (stacked)
-   *   'expand' (normalize to 100%)
-   *   'silhouette' (simple centered)
-   *
-   * order:
-   *   'inside-out' (stream)
-   *   'default' (input order)
-   ************************************/
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var availableWidth = width - margin.left - margin.right,
-          availableHeight = height - margin.top - margin.bottom,
-          container = d3.select(this);
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = scatter.xScale();
-      y = scatter.yScale();
-
-      //------------------------------------------------------------
-
-
-      // Injecting point index into each point because d3.layout.stack().out does not give index
-      data = data.map(function(aseries, i) {
-               aseries.seriesIndex = i;
-               aseries.values = aseries.values.map(function(d, j) {
-                 d.index = j;
-                 d.seriesIndex = i;
-                 return d;
-               })
-               return aseries;
-             });
-
-      var dataFiltered = data.filter(function(series) {
-            return !series.disabled;
-      });
-
-      data = d3.layout.stack()
-               .order(order)
-               .offset(offset)
-               .values(function(d) { return d.values })  //TODO: make values customizeable in EVERY model in this fashion
-               .x(getX)
-               .y(getY)
-               .out(function(d, y0, y) {
-                    var yHeight = (getY(d) === 0) ? 0 : y;
-                    d.display = {
-                      y: yHeight,
-                     y0: y0
-                    };
-                })
-              (dataFiltered);
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-stackedarea').data([data]);
-      var wrapEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-stackedarea');
-      var defsEnter = wrapEnter.append('defs');
-      var gEnter = wrapEnter.append('g');
-      var g = wrap.select('g');
-
-      gEnter.append('g').attr('class', 'nv-areaWrap');
-      gEnter.append('g').attr('class', 'nv-scatterWrap');
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      //------------------------------------------------------------
-
-
-      scatter
-        .width(availableWidth)
-        .height(availableHeight)
-        .x(getX)
-        .y(function(d) { return d.display.y + d.display.y0 })
-        .forceY([0])
-        .color(data.map(function(d,i) {
-          return d.color || color(d, d.seriesIndex);
-        }));
-
-
-      var scatterWrap = g.select('.nv-scatterWrap')
-          .datum(data);
-
-      scatterWrap.call(scatter);
-
-      defsEnter.append('clipPath')
-          .attr('id', 'nv-edge-clip-' + id)
-        .append('rect');
-
-      wrap.select('#nv-edge-clip-' + id + ' rect')
-          .attr('width', availableWidth)
-          .attr('height', availableHeight);
-
-      g   .attr('clip-path', clipEdge ? 'url(#nv-edge-clip-' + id + ')' : '');
-
-      var area = d3.svg.area()
-          .x(function(d,i)  { return x(getX(d,i)) })
-          .y0(function(d) { 
-              return y(d.display.y0) 
-          })
-          .y1(function(d) { 
-              return y(d.display.y + d.display.y0) 
-          })
-          .interpolate(interpolate);
-
-      var zeroArea = d3.svg.area()
-          .x(function(d,i)  { return x(getX(d,i)) })
-          .y0(function(d) { return y(d.display.y0) })
-          .y1(function(d) { return y(d.display.y0) });
-
-
-      var path = g.select('.nv-areaWrap').selectAll('path.nv-area')
-          .data(function(d) { return d });
-
-      path.enter().append('path').attr('class', function(d,i) { return 'nv-area nv-area-' + i })
-          .attr('d', function(d,i){
-            return zeroArea(d.values, d.seriesIndex);
-          })
-          .on('mouseover', function(d,i) {
-            d3.select(this).classed('hover', true);
-            dispatch.areaMouseover({
-              point: d,
-              series: d.key,
-              pos: [d3.event.pageX, d3.event.pageY],
-              seriesIndex: i
-            });
-          })
-          .on('mouseout', function(d,i) {
-            d3.select(this).classed('hover', false);
-            dispatch.areaMouseout({
-              point: d,
-              series: d.key,
-              pos: [d3.event.pageX, d3.event.pageY],
-              seriesIndex: i
-            });
-          })
-          .on('click', function(d,i) {
-            d3.select(this).classed('hover', false);
-            dispatch.areaClick({
-              point: d,
-              series: d.key,
-              pos: [d3.event.pageX, d3.event.pageY],
-              seriesIndex: i
-            });
-          })
-      path.exit().transition()
-          .attr('d', function(d,i) { return zeroArea(d.values,i) })
-          .remove();
-      path
-          .style('fill', function(d,i){ 
-            return d.color || color(d, d.seriesIndex) 
-          })
-          .style('stroke', function(d,i){ return d.color || color(d, d.seriesIndex) });
-      path.transition()
-          .attr('d', function(d,i) { 
-            return area(d.values,i) 
-          });
-
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      scatter.dispatch.on('elementMouseover.area', function(e) {
-        g.select('.nv-chart-' + id + ' .nv-area-' + e.seriesIndex).classed('hover', true);
-      });
-      scatter.dispatch.on('elementMouseout.area', function(e) {
-        g.select('.nv-chart-' + id + ' .nv-area-' + e.seriesIndex).classed('hover', false);
-      });
-
-      //============================================================
-
-    });
-
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  scatter.dispatch.on('elementClick.area', function(e) {
-    dispatch.areaClick(e);
-  })
-  scatter.dispatch.on('elementMouseover.tooltip', function(e) {
-        e.pos = [e.pos[0] + margin.left, e.pos[1] + margin.top],
-        dispatch.tooltipShow(e);
-  });
-  scatter.dispatch.on('elementMouseout.tooltip', function(e) {
-        dispatch.tooltipHide(e);
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Global getters and setters
-  //------------------------------------------------------------
-
-  chart.dispatch = dispatch;
-  chart.scatter = scatter;
-
-  d3.rebind(chart, scatter, 'interactive', 'size', 'xScale', 'yScale', 'zScale', 'xDomain', 'yDomain', 'xRange', 'yRange', 
-    'sizeDomain', 'forceX', 'forceY', 'forceSize', 'clipVoronoi', 'useVoronoi','clipRadius','highlightPoint','clearHighlights');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.x = function(_) {
-    if (!arguments.length) return getX;
-    getX = d3.functor(_);
-    return chart;
-  };
-
-  chart.y = function(_) {
-    if (!arguments.length) return getY;
-    getY = d3.functor(_);
-    return chart;
-  }
-
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.clipEdge = function(_) {
-    if (!arguments.length) return clipEdge;
-    clipEdge = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    return chart;
-  };
-
-  chart.offset = function(_) {
-    if (!arguments.length) return offset;
-    offset = _;
-    return chart;
-  };
-
-  chart.order = function(_) {
-    if (!arguments.length) return order;
-    order = _;
-    return chart;
-  };
-
-  //shortcut for offset + order
-  chart.style = function(_) {
-    if (!arguments.length) return style;
-    style = _;
-
-    switch (style) {
-      case 'stack':
-        chart.offset('zero');
-        chart.order('default');
-        break;
-      case 'stream':
-        chart.offset('wiggle');
-        chart.order('inside-out');
-        break;
-      case 'stream-center':
-          chart.offset('silhouette');
-          chart.order('inside-out');
-          break;
-      case 'expand':
-        chart.offset('expand');
-        chart.order('default');
-        break;
-    }
-
-    return chart;
-  };
-
-  chart.interpolate = function(_) {
-	    if (!arguments.length) return interpolate;
-	    interpolate = _;
-	    return chart;
-  };
-  //============================================================
-
-
-  return chart;
-}
-
-nv.models.stackedAreaChart = function() {
-  "use strict";
-  //============================================================
-  // Public Variables with Default Settings
-  //------------------------------------------------------------
-
-  var stacked = nv.models.stackedArea()
-    , xAxis = nv.models.axis()
-    , yAxis = nv.models.axis()
-    , legend = nv.models.legend()
-    , controls = nv.models.legend()
-    , interactiveLayer = nv.interactiveGuideline()
-    ;
-
-  var margin = {top: 30, right: 25, bottom: 50, left: 60}
-    , width = null
-    , height = null
-    , color = nv.utils.defaultColor() // a function that takes in d, i and returns color
-    , showControls = true
-    , showLegend = true
-    , showXAxis = true
-    , showYAxis = true
-    , rightAlignYAxis = false
-    , useInteractiveGuideline = false
-    , tooltips = true
-    , tooltip = function(key, x, y, e, graph) {
-        return '<h3>' + key + '</h3>' +
-               '<p>' +  y + ' on ' + x + '</p>'
-      }
-    , x //can be accessed via chart.xScale()
-    , y //can be accessed via chart.yScale()
-    , yAxisTickFormat = d3.format(',.2f')
-    , state = { style: stacked.style() }
-    , defaultState = null
-    , noData = 'No Data Available.'
-    , dispatch = d3.dispatch('tooltipShow', 'tooltipHide', 'stateChange', 'changeState')
-    , controlWidth = 250
-    , cData = ['Stacked','Stream','Expanded']
-    , transitionDuration = 250
-    ;
-
-  xAxis
-    .orient('bottom')
-    .tickPadding(7)
-    ;
-  yAxis
-    .orient((rightAlignYAxis) ? 'right' : 'left')
-    ;
-
-  controls.updateState(false);
-  //============================================================
-
-
-  //============================================================
-  // Private Variables
-  //------------------------------------------------------------
-
-  var showTooltip = function(e, offsetElement) {
-    var left = e.pos[0] + ( offsetElement.offsetLeft || 0 ),
-        top = e.pos[1] + ( offsetElement.offsetTop || 0),
-        x = xAxis.tickFormat()(stacked.x()(e.point, e.pointIndex)),
-        y = yAxis.tickFormat()(stacked.y()(e.point, e.pointIndex)),
-        content = tooltip(e.series.key, x, y, e, chart);
-
-    nv.tooltip.show([left, top], content, e.value < 0 ? 'n' : 's', null, offsetElement);
-  };
-
-  //============================================================
-
-
-  function chart(selection) {
-    selection.each(function(data) {
-      var container = d3.select(this),
-          that = this;
-
-      var availableWidth = (width  || parseInt(container.style('width')) || 960)
-                             - margin.left - margin.right,
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-
-      chart.update = function() { container.transition().duration(transitionDuration).call(chart); };
-      chart.container = this;
-
-      //set state.disabled
-      state.disabled = data.map(function(d) { return !!d.disabled });
-
-      if (!defaultState) {
-        var key;
-        defaultState = {};
-        for (key in state) {
-          if (state[key] instanceof Array)
-            defaultState[key] = state[key].slice(0);
-          else
-            defaultState[key] = state[key];
-        }
-      }
-
-      //------------------------------------------------------------
-      // Display No Data message if there's nothing to show.
-
-      if (!data || !data.length || !data.filter(function(d) { return d.values.length }).length) {
-        var noDataText = container.selectAll('.nv-noData').data([noData]);
-
-        noDataText.enter().append('text')
-          .attr('class', 'nvd3 nv-noData')
-          .attr('dy', '-.7em')
-          .style('text-anchor', 'middle');
-
-        noDataText
-          .attr('x', margin.left + availableWidth / 2)
-          .attr('y', margin.top + availableHeight / 2)
-          .text(function(d) { return d });
-
-        return chart;
-      } else {
-        container.selectAll('.nv-noData').remove();
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Scales
-
-      x = stacked.xScale();
-      y = stacked.yScale();
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup containers and skeleton of chart
-
-      var wrap = container.selectAll('g.nv-wrap.nv-stackedAreaChart').data([data]);
-      var gEnter = wrap.enter().append('g').attr('class', 'nvd3 nv-wrap nv-stackedAreaChart').append('g');
-      var g = wrap.select('g');
-
-      gEnter.append("rect").style("opacity",0);
-      gEnter.append('g').attr('class', 'nv-x nv-axis');
-      gEnter.append('g').attr('class', 'nv-y nv-axis');
-      gEnter.append('g').attr('class', 'nv-stackedWrap');
-      gEnter.append('g').attr('class', 'nv-legendWrap');
-      gEnter.append('g').attr('class', 'nv-controlsWrap');
-      gEnter.append('g').attr('class', 'nv-interactive');
-
-      g.select("rect").attr("width",availableWidth).attr("height",availableHeight);
-      //------------------------------------------------------------
-      // Legend
-
-      if (showLegend) {
-        var legendWidth = (showControls) ? availableWidth - controlWidth : availableWidth;
-        legend
-          .width(legendWidth);
-
-        g.select('.nv-legendWrap')
-            .datum(data)
-            .call(legend);
-
-        if ( margin.top != legend.height()) {
-          margin.top = legend.height();
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-        g.select('.nv-legendWrap')
-            .attr('transform', 'translate(' + (availableWidth-legendWidth) + ',' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Controls
-
-      if (showControls) {
-        var controlsData = [
-          { key: 'Stacked', disabled: stacked.offset() != 'zero' },
-          { key: 'Stream', disabled: stacked.offset() != 'wiggle' },
-          { key: 'Expanded', disabled: stacked.offset() != 'expand' }
-        ];
-
-        controlWidth = (cData.length/3) * 260;
-
-        controlsData = controlsData.filter(function(d) {
-          return cData.indexOf(d.key) > -1;
-        })
-
-        controls
-          .width( controlWidth )
-          .color(['#444', '#444', '#444']);
-
-        g.select('.nv-controlsWrap')
-            .datum(controlsData)
-            .call(controls);
-
-
-        if ( margin.top != Math.max(controls.height(), legend.height()) ) {
-          margin.top = Math.max(controls.height(), legend.height());
-          availableHeight = (height || parseInt(container.style('height')) || 400)
-                             - margin.top - margin.bottom;
-        }
-
-
-        g.select('.nv-controlsWrap')
-            .attr('transform', 'translate(0,' + (-margin.top) +')');
-      }
-
-      //------------------------------------------------------------
-
-
-      wrap.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
-      if (rightAlignYAxis) {
-          g.select(".nv-y.nv-axis")
-              .attr("transform", "translate(" + availableWidth + ",0)");
-      }
-
-      //------------------------------------------------------------
-      // Main Chart Component(s)
-
-      //------------------------------------------------------------
-      //Set up interactive layer
-      if (useInteractiveGuideline) {
-        interactiveLayer
-           .width(availableWidth)
-           .height(availableHeight)
-           .margin({left: margin.left, top: margin.top})
-           .svgContainer(container)
-           .xScale(x);
-        wrap.select(".nv-interactive").call(interactiveLayer);
-      }
-      
-      stacked
-        .width(availableWidth)
-        .height(availableHeight)
-
-      var stackedWrap = g.select('.nv-stackedWrap')
-          .datum(data);
-
-      stackedWrap.transition().call(stacked);
-
-      //------------------------------------------------------------
-
-
-      //------------------------------------------------------------
-      // Setup Axes
-
-      if (showXAxis) {
-        xAxis
-          .scale(x)
-          .ticks( availableWidth / 100 )
-          .tickSize( -availableHeight, 0);
-
-        g.select('.nv-x.nv-axis')
-            .attr('transform', 'translate(0,' + availableHeight + ')');
- 
-        g.select('.nv-x.nv-axis')
-          .transition().duration(0)
-            .call(xAxis);
-      }
-
-      if (showYAxis) {
-        yAxis
-          .scale(y)
-          .ticks(stacked.offset() == 'wiggle' ? 0 : availableHeight / 36)
-          .tickSize(-availableWidth, 0)
-          .setTickFormat(stacked.offset() == 'expand' ? d3.format('%') : yAxisTickFormat);
-
-        g.select('.nv-y.nv-axis')
-          .transition().duration(0)
-            .call(yAxis);
-      }
-
-      //------------------------------------------------------------
-
-
-      //============================================================
-      // Event Handling/Dispatching (in chart's scope)
-      //------------------------------------------------------------
-
-      stacked.dispatch.on('areaClick.toggle', function(e) {
-        if (data.filter(function(d) { return !d.disabled }).length === 1)
-          data = data.map(function(d) {
-            d.disabled = false;
-            return d
-          });
-        else
-          data = data.map(function(d,i) {
-            d.disabled = (i != e.seriesIndex);
-            return d
-          });
-
-        state.disabled = data.map(function(d) { return !!d.disabled });
-        dispatch.stateChange(state);
-
-        chart.update();
-      });
-
-      legend.dispatch.on('stateChange', function(newState) {
-        state.disabled = newState.disabled;
-        dispatch.stateChange(state);
-        chart.update();
-      });
-
-      controls.dispatch.on('legendClick', function(d,i) {
-        if (!d.disabled) return;
-
-        controlsData = controlsData.map(function(s) {
-          s.disabled = true;
-          return s;
-        });
-        d.disabled = false;
-
-        switch (d.key) {
-          case 'Stacked':
-            stacked.style('stack');
-            break;
-          case 'Stream':
-            stacked.style('stream');
-            break;
-          case 'Expanded':
-            stacked.style('expand');
-            break;
-        }
-
-        state.style = stacked.style();
-        dispatch.stateChange(state);
-
-        chart.update();
-      });
-
-
-      interactiveLayer.dispatch.on('elementMousemove', function(e) {
-          stacked.clearHighlights();
-          var singlePoint, pointIndex, pointXLocation, allData = [];
-          data
-          .filter(function(series, i) { 
-            series.seriesIndex = i;
-            return !series.disabled; 
-          })
-          .forEach(function(series,i) {
-              pointIndex = nv.interactiveBisect(series.values, e.pointXValue, chart.x());
-              stacked.highlightPoint(i, pointIndex, true);
-              var point = series.values[pointIndex];
-              if (typeof point === 'undefined') return;
-              if (typeof singlePoint === 'undefined') singlePoint = point;
-              if (typeof pointXLocation === 'undefined') pointXLocation = chart.xScale()(chart.x()(point,pointIndex));
-              allData.push({
-                  key: series.key,
-                  value: chart.y()(point, pointIndex),
-                  color: color(series,series.seriesIndex),
-                  stackedValue: point.display
-              });
-          });
-
-          allData.reverse();
-
-          //Highlight the tooltip entry based on which stack the mouse is closest to.
-          if (allData.length > 2) {
-            var yValue = chart.yScale().invert(e.mouseY);
-            var yDistMax = Infinity, indexToHighlight = null;
-            allData.forEach(function(series,i) {
-               if ( yValue >= series.stackedValue.y0 && yValue <= (series.stackedValue.y0 + series.stackedValue.y))
-               {
-                  indexToHighlight = i;
-                  return;
-               }
-            });
-            if (indexToHighlight != null)
-               allData[indexToHighlight].highlight = true;
-          }
-
-          var xValue = xAxis.tickFormat()(chart.x()(singlePoint,pointIndex));
-          interactiveLayer.tooltip
-                  .position({left: pointXLocation + margin.left, top: e.mouseY + margin.top})
-                  .chartContainer(that.parentNode)
-                  .enabled(tooltips)
-                  .valueFormatter(function(d,i) {
-                     return yAxis.tickFormat()(d);
-                  })
-                  .data(
-                      {
-                        value: xValue,
-                        series: allData
-                      }
-                  )();
-
-          interactiveLayer.renderGuideLine(pointXLocation);
-
-      });
-
-      interactiveLayer.dispatch.on("elementMouseout",function(e) {
-          dispatch.tooltipHide();
-          stacked.clearHighlights();
-      });
-
-
-      dispatch.on('tooltipShow', function(e) {
-        if (tooltips) showTooltip(e, that.parentNode);
-      });
-
-      // Update chart from a state object passed to event handler
-      dispatch.on('changeState', function(e) {
-
-        if (typeof e.disabled !== 'undefined') {
-          data.forEach(function(series,i) {
-            series.disabled = e.disabled[i];
-          });
-
-          state.disabled = e.disabled;
-        }
-
-        if (typeof e.style !== 'undefined') {
-          stacked.style(e.style);
-        }
-
-        chart.update();
-      });
-
-    });
-
-
-    return chart;
-  }
-
-
-  //============================================================
-  // Event Handling/Dispatching (out of chart's scope)
-  //------------------------------------------------------------
-
-  stacked.dispatch.on('tooltipShow', function(e) {
-    //disable tooltips when value ~= 0
-    //// TODO: consider removing points from voronoi that have 0 value instead of this hack
-    /*
-    if (!Math.round(stacked.y()(e.point) * 100)) {  // 100 will not be good for very small numbers... will have to think about making this valu dynamic, based on data range
-      setTimeout(function() { d3.selectAll('.point.hover').classed('hover', false) }, 0);
-      return false;
-    }
-   */
-
-    e.pos = [e.pos[0] + margin.left, e.pos[1] + margin.top],
-    dispatch.tooltipShow(e);
-  });
-
-  stacked.dispatch.on('tooltipHide', function(e) {
-    dispatch.tooltipHide(e);
-  });
-
-  dispatch.on('tooltipHide', function() {
-    if (tooltips) nv.tooltip.cleanup();
-  });
-
-  //============================================================
-
-
-  //============================================================
-  // Expose Public Variables
-  //------------------------------------------------------------
-
-  // expose chart's sub-components
-  chart.dispatch = dispatch;
-  chart.stacked = stacked;
-  chart.legend = legend;
-  chart.controls = controls;
-  chart.xAxis = xAxis;
-  chart.yAxis = yAxis;
-  chart.interactiveLayer = interactiveLayer;
-
-  d3.rebind(chart, stacked, 'x', 'y', 'size', 'xScale', 'yScale', 'xDomain', 'yDomain', 'xRange', 'yRange', 'sizeDomain', 'interactive', 'useVoronoi', 'offset', 'order', 'style', 'clipEdge', 'forceX', 'forceY', 'forceSize', 'interpolate');
-
-  chart.options = nv.utils.optionsFunc.bind(chart);
-  
-  chart.margin = function(_) {
-    if (!arguments.length) return margin;
-    margin.top    = typeof _.top    != 'undefined' ? _.top    : margin.top;
-    margin.right  = typeof _.right  != 'undefined' ? _.right  : margin.right;
-    margin.bottom = typeof _.bottom != 'undefined' ? _.bottom : margin.bottom;
-    margin.left   = typeof _.left   != 'undefined' ? _.left   : margin.left;
-    return chart;
-  };
-
-  chart.width = function(_) {
-    if (!arguments.length) return width;
-    width = _;
-    return chart;
-  };
-
-  chart.height = function(_) {
-    if (!arguments.length) return height;
-    height = _;
-    return chart;
-  };
-
-  chart.color = function(_) {
-    if (!arguments.length) return color;
-    color = nv.utils.getColor(_);
-    legend.color(color);
-    stacked.color(color);
-    return chart;
-  };
-
-  chart.showControls = function(_) {
-    if (!arguments.length) return showControls;
-    showControls = _;
-    return chart;
-  };
-
-  chart.showLegend = function(_) {
-    if (!arguments.length) return showLegend;
-    showLegend = _;
-    return chart;
-  };
-
-  chart.showXAxis = function(_) {
-    if (!arguments.length) return showXAxis;
-    showXAxis = _;
-    return chart;
-  };
-
-  chart.showYAxis = function(_) {
-    if (!arguments.length) return showYAxis;
-    showYAxis = _;
-    return chart;
-  };
-
-  chart.rightAlignYAxis = function(_) {
-    if(!arguments.length) return rightAlignYAxis;
-    rightAlignYAxis = _;
-    yAxis.orient( (_) ? 'right' : 'left');
-    return chart;
-  };
-
-  chart.useInteractiveGuideline = function(_) {
-    if(!arguments.length) return useInteractiveGuideline;
-    useInteractiveGuideline = _;
-    if (_ === true) {
-       chart.interactive(false);
-       chart.useVoronoi(false);
-    }
-    return chart;
-  };
-
-  chart.tooltip = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.tooltips = function(_) {
-    if (!arguments.length) return tooltips;
-    tooltips = _;
-    return chart;
-  };
-
-  chart.tooltipContent = function(_) {
-    if (!arguments.length) return tooltip;
-    tooltip = _;
-    return chart;
-  };
-
-  chart.state = function(_) {
-    if (!arguments.length) return state;
-    state = _;
-    return chart;
-  };
-
-  chart.defaultState = function(_) {
-    if (!arguments.length) return defaultState;
-    defaultState = _;
-    return chart;
-  };
-
-  chart.noData = function(_) {
-    if (!arguments.length) return noData;
-    noData = _;
-    return chart;
-  };
-
-  chart.transitionDuration = function(_) {
-    if (!arguments.length) return transitionDuration;
-    transitionDuration = _;
-    return chart;
-  };
-
-  chart.controlsData = function(_) {
-    if (!arguments.length) return cData;
-    cData = _;
-    return chart;
-  };
-
-  yAxis.setTickFormat = yAxis.tickFormat;
-
-  yAxis.tickFormat = function(_) {
-    if (!arguments.length) return yAxisTickFormat;
-    yAxisTickFormat = _;
-    return yAxis;
-  };
-
-
-  //============================================================
-
-  return chart;
-}
-})();
\ No newline at end of file
diff --git a/airflow/www_rbac/static_config.py b/airflow/www_rbac/static_config.py
new file mode 100644
index 0000000000000..7b5560c193d57
--- /dev/null
+++ b/airflow/www_rbac/static_config.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import print_function
+
+import os
+import json
+from typing import Dict
+
+from flask import url_for
+
+manifest = dict()  # type: Dict[str, str]
+
+
+def configure_manifest_files(app):
+    """
+    Loads the manifest file and register the `url_for_asset_` template tag.
+    :param app:
+    :return:
+    """
+
+    def parse_manifest_json():
+        # noinspection PyBroadException
+        try:
+            global manifest
+            manifest_file = os.path.join(os.path.dirname(__file__),
+                                         'static/dist/manifest.json')
+            with open(manifest_file, 'r') as f:
+                manifest.update(json.load(f))
+
+                for k in manifest.keys():
+                    manifest[k] = os.path.join("dist", manifest[k])
+        except Exception:
+            print("Please make sure to build the frontend in "
+                  "static/ directory and restart the server")
+            pass
+
+    def get_asset_url(filename):
+        if app.debug:
+            parse_manifest_json()
+        return url_for('static', filename=manifest.get(filename, ''))
+
+    parse_manifest_json()
+
+    @app.context_processor
+    def get_url_for_asset():
+        """
+        Template tag to return the asset URL.
+        WebPack renders the assets after minification and modification
+        under the static/dist folder.
+        This template tag reads the asset name in manifest.json and returns
+        the appropriate file.
+        """
+        return dict(url_for_asset=get_asset_url)
diff --git a/airflow/www_rbac/templates/airflow/chart.html b/airflow/www_rbac/templates/airflow/chart.html
index 983a8dc055ab1..82ce7a1096ce7 100644
--- a/airflow/www_rbac/templates/airflow/chart.html
+++ b/airflow/www_rbac/templates/airflow/chart.html
@@ -21,15 +21,14 @@
 {% block head_css %}
 {{ super() }}
 <link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='tree.css') }}">
+    href="{{ url_for('static', filename='css/tree.css') }}">
 <link rel="stylesheet" type="text/css"
     href="{{ url_for('appbuilder.static',filename='datepicker/bootstrap-datepicker.css')}}" >
+<link href="{{ url_for_asset('dataTables.bootstrap.min.css') }}" rel="stylesheet" type="text/css" >
 <link rel="stylesheet" type="text/css"
-      href="{{ url_for("static", filename="dataTables.bootstrap.css") }}">
-<link rel="stylesheet" type="text/css"
-      href="{{ url_for("static", filename="nv.d3.css") }}">
-<script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-<script src="{{ url_for('static', filename='nv.d3.js') }}"></script>
+      href="{{ url_for_asset('nv.d3.min.css') }}">
+<script src="{{ url_for_asset('d3.min.js') }}"></script>
+<script src="{{ url_for_asset('nv.d3.min.js') }}"></script>
 {% endblock %}
 
 {% block content %}
diff --git a/airflow/www_rbac/templates/airflow/circles.html b/airflow/www_rbac/templates/airflow/circles.html
index 027b4ff4e464e..3362540a84f2f 100644
--- a/airflow/www_rbac/templates/airflow/circles.html
+++ b/airflow/www_rbac/templates/airflow/circles.html
@@ -1,13 +1,13 @@
-{# 
+{#
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
-  
+
     http://www.apache.org/licenses/LICENSE-2.0
-  
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,7 +26,7 @@ <h1>Airflow 404 = lots of circles</h1>
         <svg></svg>
 
     </div>
-    <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
+    <script src="{{ url_for_asset('d3.min.js') }}"></script>
     <script>
     var height =700;
     var width = document.getElementById("div_svg").offsetWidth;
diff --git a/airflow/www_rbac/templates/airflow/config.html b/airflow/www_rbac/templates/airflow/config.html
index 501ef66ec1873..a68c5a7aef49a 100644
--- a/airflow/www_rbac/templates/airflow/config.html
+++ b/airflow/www_rbac/templates/airflow/config.html
@@ -33,10 +33,6 @@ <h2>{{ title }}</h2>
         <h5>{{ subtitle }}</h5>
     {% endif %}
 
-    {% if code %}
-        <pre>{{ code }}</pre>
-    {% endif %}
-
     {% if code_html %}
         {{ code_html|safe }}
     {% endif %}
diff --git a/airflow/www_rbac/templates/airflow/confirm.html b/airflow/www_rbac/templates/airflow/confirm.html
index 3d280356547b3..5b5253d9ebb17 100644
--- a/airflow/www_rbac/templates/airflow/confirm.html
+++ b/airflow/www_rbac/templates/airflow/confirm.html
@@ -26,10 +26,15 @@ <h2>Wait a minute.</h2>
         <pre><code>{{ details }}</code></pre>
     {% endif %}
     </div>
-    <button class="btn btn-primary" onclick="window.location += '&confirmed=true'">
-        OK!
-    </button>
-    <button class="btn" onclick="window.history.back()">bail.</button>
+    <form method="POST">
+      <input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
+      <input name="confirmed" type="hidden" value="true">
+      {% for name,val in request.form.items() if name != "csrf_token" %}
+        <input type="hidden" name="{{ name }}" value="{{ val }}">
+      {% endfor %}
+      <input class="btn btn-primary" type="submit" value="OK!">
+      <button class="btn" onclick="window.history.back(); return false">bail.</button>
+    </form>
     <br>
     <br>
 {% endblock %}
diff --git a/airflow/www_rbac/templates/airflow/conn_create.html b/airflow/www_rbac/templates/airflow/conn_create.html
index 5e6326cab2566..02bade6c4e803 100644
--- a/airflow/www_rbac/templates/airflow/conn_create.html
+++ b/airflow/www_rbac/templates/airflow/conn_create.html
@@ -1,13 +1,13 @@
-{# 
+{#
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
-  
+
     http://www.apache.org/licenses/LICENSE-2.0
-  
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,5 +19,5 @@
 
 {% block tail %}
   {{ super() }}
-    <script src="{{ url_for('static', filename='connection_form.js') }}"></script>
+    <script src="{{ url_for_asset('connectionForm.js') }}"></script>
 {% endblock %}
diff --git a/airflow/www_rbac/templates/airflow/conn_edit.html b/airflow/www_rbac/templates/airflow/conn_edit.html
index 2ff44be88dba2..92f9331aa4b8e 100644
--- a/airflow/www_rbac/templates/airflow/conn_edit.html
+++ b/airflow/www_rbac/templates/airflow/conn_edit.html
@@ -1,13 +1,13 @@
-{# 
+{#
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
-  
+
     http://www.apache.org/licenses/LICENSE-2.0
-  
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,5 +19,5 @@
 
 {% block tail %}
   {{ super() }}
-    <script src="{{ url_for('static', filename='connection_form.js') }}"></script>
+    <script src="{{ url_for_asset(filename='connectionForm.js') }}"></script>
 {% endblock %}
diff --git a/airflow/www_rbac/templates/airflow/dag.html b/airflow/www_rbac/templates/airflow/dag.html
index eb449085023b9..2d4f14337f70c 100644
--- a/airflow/www_rbac/templates/airflow/dag.html
+++ b/airflow/www_rbac/templates/airflow/dag.html
@@ -21,8 +21,7 @@
 
 {% block head_css %}
   {{ super() }}
-  <link href="{{ url_for('static', filename='main.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='bootstrap-toggle.min.css') }}" rel="stylesheet" type="text/css">
+  <link href="{{ url_for_asset('bootstrap-toggle.min.css') }}" rel="stylesheet" type="text/css">
 {% endblock %}
 
 {% block content %}
@@ -46,59 +45,75 @@ <h4 class="pull-right">
   </div>
   <div class="clearfix"></div>
   <div>
+    {% set base_date_arg = request.args.get('base_date') %}
+    {% set num_runs_arg = request.args.get('num_runs') %}
     <ul class="nav nav-pills">
       {% if dag.parent_dag %}
-          <li class="never_active"><a href="{{ url_for('Airflow.' + dag.default_view, dag_id=dag.parent_dag.dag_id) }}">
+          <li class="never_active"><a href="{{ url_for('Airflow.' + dag.get_default_view(), dag_id=dag.parent_dag.dag_id) }}">
               <span class="glyphicon glyphicon-arrow-left" aria-hidden="true"></span>
               Back to {{ dag.parent_dag.dag_id }}</a>
           </li>
       {% endif %}
-      <li><a href="{{ url_for("Airflow.graph", dag_id=dag.dag_id, root=root) }}">
+      <li><a href="{{ url_for('Airflow.graph', dag_id=dag.dag_id, root=root, num_runs=num_runs_arg, base_date=base_date_arg) }}">
           <span class="glyphicon glyphicon-certificate" aria-hidden="true"></span>
         Graph View</a></li>
-      <li><a href="{{ url_for("Airflow.tree", dag_id=dag.dag_id, num_runs=num_runs, root=root) }}">
+      <li><a href="{{ url_for('Airflow.tree', dag_id=dag.dag_id, num_runs=num_runs_arg, root=root, base_date=base_date_arg) }}">
           <span class="glyphicon glyphicon-tree-deciduous" aria-hidden="true"></span>
         Tree View
       </a></li>
-      <li><a href="{{ url_for("Airflow.duration", dag_id=dag.dag_id, days=30, root=root) }}">
+      <li><a href="{{ url_for('Airflow.duration', dag_id=dag.dag_id, days=30, root=root, num_runs=num_runs_arg, base_date=base_date_arg) }}">
           <span class="glyphicon glyphicon-stats" aria-hidden="true"></span>
           Task Duration
       </a></li>
-      <li><a href="{{ url_for("Airflow.tries", dag_id=dag.dag_id, days=30, root=root) }}">
+      <li><a href="{{ url_for('Airflow.tries', dag_id=dag.dag_id, days=30, root=root, num_runs=num_runs_arg, base_date=base_date_arg) }}">
           <span class="glyphicon glyphicon-duplicate" aria-hidden="true"></span>
           Task Tries
       </a></li>
       <li>
-        <a href="{{ url_for("Airflow.landing_times", dag_id=dag.dag_id, days=30, root=root) }}">
+        <a href="{{ url_for('Airflow.landing_times', dag_id=dag.dag_id, days=30, root=root, num_runs=num_runs_arg, base_date=base_date_arg) }}">
           <span class="glyphicon glyphicon-plane" aria-hidden="true"></span>
           Landing Times
         </a>
       </li>
       <li>
-        <a href="{{ url_for("Airflow.gantt", dag_id=dag.dag_id, root=root) }}">
+        <a href="{{ url_for('Airflow.gantt', dag_id=dag.dag_id, root=root, num_runs=num_runs_arg, base_date=base_date_arg) }}">
           <span class="glyphicon glyphicon-align-left" aria-hidden="true"></span>
           <i class="icon-align-left"></i>
           Gantt
         </a>
       </li>
       <li>
-      <a href="{{ url_for("Airflow.dag_details", dag_id=dag.dag_id) }}">
+      <a href="{{ url_for('Airflow.dag_details', dag_id=dag.dag_id) }}">
           <span class="glyphicon glyphicon-list" aria-hidden="true"></span>
           Details
         </a>
       </li>
       <li>
-        <a href="{{ url_for("Airflow.code", dag_id=dag.dag_id, root=root) }}">
+        <a href="{{ url_for('Airflow.code', dag_id=dag.dag_id, root=root) }}">
           <span class="glyphicon glyphicon-file" aria-hidden="true"></span>
           Code
         </a>
       </li>
       <li>
-        <a href="{{ url_for("Airflow.refresh", dag_id=dag.dag_id) }}" title="Refresh">
+        <a href="{{ url_for('Airflow.trigger', dag_id=dag.dag_id, origin=url_for('Airflow.tree', dag_id=dag.dag_id)) }}"
+          onclick="return confirmTriggerDag(this, '{{ dag.safe_dag_id }}')">
+          <span class="glyphicon glyphicon-play-circle" aria-hidden="true"></span>
+          Trigger DAG
+        </a>
+      </li>
+      <li>
+        <a href="{{ url_for('Airflow.refresh', dag_id=dag.dag_id) }}" title="Refresh" onclick="postAsForm(this.href); return false">
           <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
           Refresh
         </a>
       </li>
+      <li>
+        <a href="{{ url_for('Airflow.delete', dag_id=dag.dag_id, root=root) }}"
+        onclick="return confirmDeleteDag(this, '{{ dag.safe_dag_id }}')">
+        <span class="glyphicon glyphicon-remove-circle" style="color:red" aria-hidden="true"></span>
+        Delete
+        </a>
+      </li>
     </ul>
   </div>
   <hr>
@@ -138,74 +153,122 @@ <h4 class="modal-title" id="myModalLabel">
             View Log
           </button>
           <hr/>
-          <button id="btn_run" type="button" class="btn btn-primary"
-            title="Runs a single task instance">
-            Run
-          </button>
-          <span class="btn-group">
-            <button id="btn_ignore_all_deps"
-              type="button" class="btn" data-toggle="button"
-              title="Ignores all non-critical dependencies, including task state and task_deps"
-              >Ignore All Deps</button>
-          </span>
-          <span class="btn-group">
-            <button id="btn_ignore_ti_state"
-              type="button" class="btn" data-toggle="button"
-              title="Ignore previous success/failure"
-              >Ignore Task State</button>
-          </span>
-          <button id="btn_ignore_task_deps"
-            type="button" class="btn" data-toggle="button"
-            title="Disregard the task-specific dependencies, e.g. status of upstream task instances and depends_on_past"
-            >Ignore Task Deps</button>
-          </span>
+          <div>
+            <label style="display:inline"> Download Log (by attempts): </label>
+            <ul class="nav nav-pills" role="tablist" id="try_index" style="display:inline">
+            </ul>
+          </div>
           <hr/>
-          <button id="btn_clear" type="button" class="btn btn-primary"
-              title="Clearing deletes the previous state of the task instance, allowing it to get re-triggered by the scheduler or a backfill command"
-              >Clear
-          </button>
-          <span class="btn-group">
-            <button id="btn_past"
-              type="button" class="btn" data-toggle="button"
-              title="Also include past task instances when clearing this one"
-              >Past</button>
-            <button id="btn_future"
-              type="button" class="btn" data-toggle="button"
-              title="Also include future task instances when clearing this one"
-              >Future</button>
-            <button id="btn_upstream"
-              type="button" class="btn" data-toggle="button"
-              title="Also include upstream dependencies"
-              >Upstream</button>
-            <button id="btn_downstream"
-              type="button" class="btn active" data-toggle="button"
-              title="Also include downstream dependencies"
-              >
-              Downstream
+          <form method="POST">
+            <input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
+            <input name="dag_id" type="hidden">
+            <input name="task_id" type="hidden">
+            <input name="execution_date" type="hidden">
+            <input name="origin" type="hidden" value="{{ request.base_url }}">
+            <button id="btn_run" type="button" class="btn btn-primary"
+              title="Runs a single task instance"
+              data-action="{{ url_for('Airflow.run') }}">
+              Run
             </button>
-            <button id="btn_recursive"
-              type="button" class="btn active" data-toggle="button">
-               Recursive
+            <input type="hidden" name="ignore_all_deps">
+            <input type="hidden" name="ignore_ti_state">
+            <input type="hidden" name="ignore_task_deps">
+            <span class="btn-group" data-toggle="buttons">
+              <label
+                class="btn"
+                title="Ignores all non-critical dependencies, including task state and task_deps">
+                <input type="checkbox" value="true" name="ignore_all_deps" autocomplete="off">
+                Ignore All Deps</label>
+              <label class="btn"
+                title="Ignore previous success/failure">
+                <input type="checkbox" value="true" name="ignore_ti_state" autocomplete="off">
+                Ignore Task State
+              </label>
+              <label class="btn"
+                title="Disregard the task-specific dependencies, e.g. status of upstream task instances and depends_on_past">
+                <input type="checkbox" value="true" name="ignore_task_deps" autocomplete="off">
+                Ignore Task Deps
+              </label>
+            </span>
+            <hr/>
+            <button id="btn_clear" type="button" class="btn btn-primary"
+                title="Clearing deletes the previous state of the task instance, allowing it to get re-triggered by the scheduler or a backfill command"
+                data-action="{{ url_for('Airflow.clear') }}"
+                >Clear
             </button>
-          </span>
-          <hr/>
-          <button id="btn_success" type="button" class="btn btn-primary">
-            Mark Success
-          </button>
-          <span class="btn-group">
-            <button id="btn_success_past"
-              type="button" class="btn" data-toggle="button">Past</button>
-            <button id="btn_success_future"
-              type="button" class="btn" data-toggle="button">
-              Future
+            <span class="btn-group" data-toggle="buttons">
+              <label class="btn"
+                title="Also include past task instances when clearing this one">
+                <input type="checkbox" value="true" name="past" autocomplete="off">
+                Past
+              </label>
+              <label class="btn"
+                title="Also include future task instances when clearing this one">
+                <input type="checkbox" value="true" name="future" autocomplete="off">
+                Future
+              </label>
+              <label class="btn"
+                title="Also include upstream dependencies">
+                <input type="checkbox" value="true" name="upstream" autocomplete="off">
+                Upstream
+              </label>
+              <label class="btn active"
+                title="Also include downstream dependencies">
+                <input type="checkbox" value="true" name="downstream" checked autocomplete="off">
+                Downstream
+              </label>
+              <label class="btn active">
+                <input type="checkbox" value="true" name="recursive" checked autocomplete="off">
+                 Recursive
+              </label>
+            </span>
+            <hr/>
+            <button id="btn_failed" type="button" class="btn btn-primary"
+                data-action="{{ url_for('Airflow.failed') }}">
+              Mark Failed
             </button>
-            <button id="btn_success_upstream"
-              type="button" class="btn" data-toggle="button">Upstream</button>
-            <button id="btn_success_downstream"
-              type="button" class="btn" data-toggle="button">
-              Downstream
+            <span class="btn-group" data-toggle="buttons">
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_past" autocomplete="off">
+                Past
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_future" autocomplete="off">
+                Future
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_upstream" autocomplete="off">
+                Upstream
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="failed_downstream" autocomplete="off">
+                Downstream
+              </label>
+            </span>
+            <hr/>
+            <button id="btn_success" type="button" class="btn btn-primary"
+                data-action="{{ url_for('Airflow.success') }}">
+              Mark Success
             </button>
-          </span>
+            <span class="btn-group" data-toggle="buttons">
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_past" autocomplete="off">
+                Past
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_future" autocomplete="off">
+                Future
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_upstream" autocomplete="off">
+                Upstream
+              </label>
+              <label class="btn">
+                <input type="checkbox" value="true" name="success_downstream" autocomplete="off">
+                Downstream
+              </label>
+            </span>
+          </form>
         </div>
         <div class="modal-footer">
           <button type="button" class="btn btn-default" data-dismiss="modal">
@@ -227,15 +290,24 @@ <h4 class="modal-title" id="dagModalLabel">
           </h4>
         </div>
         <div class="modal-body">
-          <button id="btn_edit_dagrun" type="button" class="btn btn-primary">
-            Edit
-          </button>
-          <button id="btn_dagrun_clear" type="button" class="btn btn-primary">
-            Clear
-          </button>
-          <button id="btn_dagrun_success" type="button" class="btn btn-primary">
-            Mark Success
-          </button>
+          <form method="POST">
+            <button id="btn_edit_dagrun" type="button" class="btn btn-primary">
+              Edit
+            </button>
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+            <input name="dag_id" type="hidden">
+            <input name="execution_date" type="hidden">
+            <input name="origin" type="hidden" value="{{ request.base_url }}">
+            <button id="btn_dagrun_clear" type="button" class="btn btn-primary" data-action="{{ url_for('Airflow.dagrun_clear') }}">
+              Clear
+            </button>
+            <button id="btn_dagrun_failed" type="button" class="btn btn-primary" data-action="{{ url_for('Airflow.dagrun_failed') }}">
+              Mark Failed
+            </button>
+            <button id="btn_dagrun_success" type="button" class="btn btn-primary" data-action="{{ url_for('Airflow.dagrun_success') }}">
+              Mark Success
+            </button>
+          </form>
         </div>
         <div class="modal-footer">
           <button type="button" class="btn btn-default" data-dismiss="modal">
@@ -248,7 +320,7 @@ <h4 class="modal-title" id="dagModalLabel">
 {% endblock %}
 {% block tail %}
   {{ super() }}
-  <script src="{{ url_for('static', filename='bootstrap-toggle.min.js') }}"></script>
+  <script src="{{ url_for_asset('bootstrap-toggle.min.js') }}"></script>
   <script>
 function updateQueryStringParameter(uri, key, value) {
   var re = new RegExp("([?&])" + key + "=.*?(&|$)", "i");
@@ -269,10 +341,11 @@ <h4 class="modal-title" id="dagModalLabel">
 
     var id = '';
     var dag_id = '{{ dag.dag_id }}';
+    var dagTZ = '{{ dag.timezone.name }}'; // Being used in datetime-utils.js
     var task_id = '';
     var exection_date = '';
     var subdag_id = '';
-    function call_modal(t, d, sd) {
+    function call_modal(t, d, try_numbers, sd) {
       task_id = t;
       loc = String(window.location);
       $("#btn_filter").on("click", function(){
@@ -284,12 +357,35 @@ <h4 class="modal-title" id="dagModalLabel">
       $('#execution_date').html(d);
       $('#myModal').modal({});
       $("#myModal").css("margin-top","0px")
-        if (subdag_id===undefined)
-            $("#div_btn_subdag").hide();
-        else {
-            $("#div_btn_subdag").show();
-            subdag_id = "{{ dag.dag_id }}."+t;
+      if (subdag_id === undefined)
+          $("#div_btn_subdag").hide();
+      else {
+          $("#div_btn_subdag").show();
+          subdag_id = "{{ dag.dag_id }}."+t;
+      }
+
+      $("#try_index > li").remove();
+      var startIndex = (try_numbers > 2 ? 0 : 1)
+      for (var index = startIndex; index <  try_numbers; index++) {
+        var url = "{{ url_for('Airflow.get_logs_with_metadata') }}" +
+          "?dag_id=" + encodeURIComponent(dag_id) +
+          "&task_id=" + encodeURIComponent(task_id) +
+          "&execution_date=" + encodeURIComponent(execution_date) +
+          "&metadata=null" +
+          "&format=file";
+
+        var showLabel = index;
+        if (index != 0) {
+          url += "&try_number=" + index;
+        } else {
+          showLabel = 'All';
         }
+
+        $("#try_index").append(`<li role="presentation" style="display:inline">
+          <a href="${url}"> ${showLabel} </a>
+          </li>`
+        );
+      }
     }
 
     function call_modal_dag(dag) {
@@ -300,6 +396,24 @@ <h4 class="modal-title" id="dagModalLabel">
       $("#dagModal").css("margin-top","0px");
     }
 
+    function confirmTriggerDag(link, dag_id){
+      if (confirm("Are you sure you want to run '"+dag_id+"' now?")) {
+        postAsForm(link.href, {});
+      }
+      // Never follow the link
+      return false;
+    }
+
+    function confirmDeleteDag(link, dag_id){
+      if (confirm("Are you sure you want to delete '"+dag_id+"' now?\n\
+        This option will delete ALL metadata, DAG runs, etc.\n\
+        EXCEPT Log.\n\
+        This cannot be undone.")) {
+        postAsForm(link.href, {});
+      }
+      return false;
+    }
+
     $("#btn_rendered").click(function(){
       url = "{{ url_for('Airflow.rendered') }}" +
         "?task_id=" + encodeURIComponent(task_id) +
@@ -309,7 +423,7 @@ <h4 class="modal-title" id="dagModalLabel">
     });
 
     $("#btn_subdag").click(function(){
-      url = "{{ url_for( 'Airflow.' + dag.default_view ) }}" +
+      url = "{{ url_for( 'Airflow.' + dag.get_default_view() ) }}" +
         "?dag_id=" + encodeURIComponent(subdag_id) +
         "&execution_date=" + encodeURIComponent(execution_date);
       window.location = url;
@@ -339,79 +453,21 @@ <h4 class="modal-title" id="dagModalLabel">
       window.location = url;
     });
 
-    $("#btn_run").click(function(){
-      url = "{{ url_for('Airflow.run') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&ignore_all_deps=" + $('#btn_ignore_all_deps').hasClass('active') +
-        "&ignore_task_deps=" + $('#btn_ignore_task_deps').hasClass('active') +
-        "&ignore_ti_state=" + $('#btn_ignore_ti_state').hasClass('active') +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_clear").click(function(){
-      url = "{{ url_for('Airflow.clear') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&future=" + $('#btn_future').hasClass('active') +
-        "&past=" + $('#btn_past').hasClass('active') +
-        "&upstream=" + $('#btn_upstream').hasClass('active') +
-        "&downstream=" + $('#btn_downstream').hasClass('active') +
-        "&recursive=" + $('#btn_recursive').hasClass('active') +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_dagrun_clear").click(function(){
-      url = "{{ url_for('Airflow.dagrun_clear') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_success").click(function(){
-      url = "{{ url_for('Airflow.success') }}" +
-        "?task_id=" + encodeURIComponent(task_id) +
-        "&dag_id=" + encodeURIComponent(dag_id) +
-        "&upstream=" + $('#btn_success_upstream').hasClass('active') +
-        "&downstream=" + $('#btn_success_downstream').hasClass('active') +
-        "&future=" + $('#btn_success_future').hasClass('active') +
-        "&past=" + $('#btn_success_past').hasClass('active') +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-
-      window.location = url;
-    });
-
-    $('#btn_dagrun_success').click(function(){
-      url = "{{ url_for('Airflow.dagrun_success') }}" +
-        "?dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date) +
-        "&origin=" + encodeURIComponent(window.location);
-      window.location = url;
-    });
-
-    $("#btn_gantt").click(function(){
-      url = "{{ url_for('Airflow.gantt') }}" +
-        "?dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date);
-      window.location = url;
-    });
+    $('form button[data-action]').click(function (e) {
+      var form = $(this).closest('form').get(0);
+      form.dag_id.value = dag_id;
+      form.execution_date.value = execution_date;
+      form.origin.value = window.location;
+      if (form.task_id) {
+        form.task_id.value = task_id;
+      }
+      form.action = $(this).data("action");
+      form.submit();
+    })
 
-    $("#btn_graph").click(function(){
-      url = "{{ url_for('Airflow.graph') }}" +
-        "?dag_id=" + encodeURIComponent(dag_id) +
-        "&execution_date=" + encodeURIComponent(execution_date);
-      window.location = url;
-    });
 
     $("#pause_resume").change(function() {
-      var dag_id =  $(this).attr('dag_id');
+      var dag_id = $(this).attr('dag_id');
       if ($(this).prop('checked')) {
         is_paused = 'true'
       } else {
@@ -421,9 +477,8 @@ <h4 class="modal-title" id="dagModalLabel">
       $.post(url);
     });
 
-    $('#btn_edit_dagrun').click(function(){
-      window.location = "{{ url_for('DagModelView.show', pk=id) }}";
-
+    $('#btn_edit_dagrun').click(function() {
+      window.location = "{{ url_for('DagModelView.edit', pk=dag.dag_id) }}";
     });
 
   </script>
diff --git a/airflow/www_rbac/templates/airflow/dag_code.html b/airflow/www_rbac/templates/airflow/dag_code.html
index dfab08b0c06e8..cb667063525a9 100644
--- a/airflow/www_rbac/templates/airflow/dag_code.html
+++ b/airflow/www_rbac/templates/airflow/dag_code.html
@@ -21,25 +21,7 @@
 {% block content %}
     {{ super() }}
     <h4>{{ title }}</h4>
-    {% if html_code %}
-        {{ html_code|safe }}
-    {% endif %}
-    {% if code %}
-        <pre>{{ code }}</pre>
-    {% endif %}
-
-    {% if code_dict %}
-        {% for k, v in code_dict.items() %}
-            <h5>{{ k }}</h5>
-            <pre>{{ v }}</pre>
-        {% endfor %}
-    {% endif %}
-    {% if html_dict %}
-        {% for k, v in html_dict.items() %}
-            <h5>{{ k }}</h5>
-            {{ v|safe }}
-        {% endfor %}
-    {% endif %}
+    {{ html_code|safe }}
 {% endblock %}
 
 {% block tail %}
diff --git a/airflow/www_rbac/templates/airflow/dag_details.html b/airflow/www_rbac/templates/airflow/dag_details.html
index dfd28bc1f89ad..37f51b53064ca 100644
--- a/airflow/www_rbac/templates/airflow/dag_details.html
+++ b/airflow/www_rbac/templates/airflow/dag_details.html
@@ -29,7 +29,7 @@ <h2>{{ title }}</h2>
       <a
           class="btn"
           style="border: none; background-color:{{ State.color(state)}}; color: {{ State.color_fg(state) }};"
-          href="/taskinstance/list/?_flt_3_dag_id={{ dag.dag_id }}&_flt_3_state={{ state }}">
+          href="{{ url_for('TaskInstanceModelView.list', _flt_3_dag_id=dag.dag_id, _flt_3_state=state) }}">
         {{ state }} <span class="badge">{{ count }}</span>
       </a>
       {% endfor %}
@@ -42,7 +42,7 @@ <h2>{{ title }}</h2>
       </tr>
       <tr>
         <th>max_active_runs</td>
-        <td>{{ dag.active_runs | length }} / {{ dag.max_active_runs }}</td>
+        <td>{{ active_runs | length }} / {{ dag.max_active_runs }}</td>
       </tr>
       <tr>
         <th>concurrency</td>
diff --git a/airflow/www_rbac/templates/airflow/dags.html b/airflow/www_rbac/templates/airflow/dags.html
index a712e5a9ee4bc..10d5bcd55c9ed 100644
--- a/airflow/www_rbac/templates/airflow/dags.html
+++ b/airflow/www_rbac/templates/airflow/dags.html
@@ -21,14 +21,14 @@
 
 {% block head_css %}
 {{ super() }}
-<link href="{{ url_for("static", filename="dataTables.bootstrap.css") }}" rel="stylesheet" type="text/css" >
-<link href="{{ url_for("static", filename="bootstrap-toggle.min.css") }}" rel="stylesheet" type="text/css">
+<link href="{{ url_for_asset('dataTables.bootstrap.min.css') }}" rel="stylesheet" type="text/css" >
+<link href="{{ url_for_asset('bootstrap-toggle.min.css') }}" rel="stylesheet" type="text/css">
 {% endblock %}
 
 {% block content %}
   <h2>DAGs</h2>
 
-  <div id="main_content" style="display:none;">
+  <div id="main_content">
     <div class="row">
       <div class="col-sm-2">
       </div>
@@ -63,53 +63,37 @@ <h2>DAGs</h2>
             </tr>
         </thead>
         <tbody>
-        {% for dag_id in dag_ids_in_page %}
-            {% set dag = webserver_dags[dag_id] if dag_id in webserver_dags else None %}
+        {% for dag in dags %}
             <tr>
                 <!-- Column 1: Edit dag -->
                 <td class="text-center" style="width:10px;">
-                    {% if dag_id in orm_dags %}
-                    <a href="{{ url_for('DagModelView.show', pk=dag_id) }}" title="Info">
+                    <a href="{{ url_for('DagModelView.show', pk=dag.dag_id) }}" title="Info">
                         <span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
                     </a>
-                    {% endif %}
                 </td>
 
                 <!-- Column 2: Turn dag on/off -->
                 <td>
-                  {% if dag_id in orm_dags %}
-                    {% set disabled = 'disabled' if view_only else None %}
-                    <input id="toggle-{{ dag_id }}" {{ disabled }} dag_id="{{ dag_id }}" type="checkbox" {{ "checked" if not orm_dags[dag_id].is_paused else "" }} data-toggle="toggle" data-size="mini" method="post">
-                  {% endif %}
+                    <input id="toggle-{{ dag.dag_id }}" dag_id="{{ dag.dag_id }}" type="checkbox" {{ "checked" if not dag.is_paused else "" }} data-toggle="toggle" data-size="mini" method="post">
                 </td>
 
                 <!-- Column 3: Name -->
                 <td>
-                    {% if dag_id in webserver_dags %}
-                    <a href="{{ url_for('Airflow.'+dag.default_view, dag_id=dag.dag_id) }}" title="{{ dag.description }}">
-                        {{ dag_id }}
+                    <a href="{{ url_for('Airflow.'+ dag.get_default_view(), dag_id=dag.dag_id) }}" title="{{ dag.description }}">
+                        {{ dag.dag_id }}
                     </a>
-                    {% else %}
-                        {{ dag_id }}
-                        <span class="glyphicon glyphicon-info-sign" class="info" aria-hidden="true" title="This DAG isn't available in the web server's DagBag object. It shows up in this list because the scheduler marked it as active in the metadata database."></span>
-                    {% endif %}
-                    {% if dag_id not in orm_dags %}
-                        <span class="glyphicon glyphicon-info-sign" class="info" aria-hidden="true" title="This DAG seems to be existing only locally. The master scheduler doesn't seem to be aware of its existence."></span>
-                    {% endif %}
                 </td>
 
                 <!-- Column 4: Dag Schedule -->
                 <td>
-                    {% if dag_id in webserver_dags %}
                     <a class="label label-default schedule {{ dag.dag_id }}" href="{{ url_for('DagRunModelView.list') }}?_flt_3_dag_id={{ dag.dag_id }}">
                         {{ dag.schedule_interval }}
                     </a>
-                    {% endif %}
                 </td>
 
                 <!-- Column 5: Dag Owners -->
                 <td>
-                  {{ dag.owner if dag else orm_dags[dag_id].owners }}
+                  {{ dag.owners }}
                 </td>
 
                 <!-- Column 6: Recent Tasks -->
@@ -119,14 +103,12 @@ <h2>DAGs</h2>
 
                 <!-- Column 7: Last Run -->
                 <td class="text-nowrap latest_dag_run {{ dag.dag_id }}">
-                  {% if dag %}
-                    {% set last_run = dag.get_last_dagrun() %}
-                    {% if last_run and last_run.execution_date %}
-                      <a href="{{ url_for('Airflow.graph', dag_id=dag.dag_id, execution_date=last_run.execution_date) }}">
-                        {{ last_run.execution_date.strftime("%Y-%m-%d %H:%M") }}
-                      </a>
-                      <span aria-hidden="true" id="statuses_info" title="Start Date: {{ last_run.start_date.strftime("%Y-%m-%d %H:%M") }}" class="glyphicon glyphicon-info-sign"></span>
-                    {% endif %}
+                  {% set last_run = dag.get_last_dagrun(include_externally_triggered=True) %}
+                  {% if last_run and last_run.execution_date %}
+                    <a href="{{ url_for('Airflow.graph', dag_id=dag.dag_id, execution_date=last_run.execution_date) }}">
+                      {{ last_run.execution_date.strftime("%Y-%m-%d %H:%M") }}
+                    </a>
+                    <span aria-hidden="true" id="statuses_info" title="Start Date: {{ last_run.start_date.strftime("%Y-%m-%d %H:%M") }}" class="glyphicon glyphicon-info-sign"></span>
                   {% endif %}
                 </td>
 
@@ -140,8 +122,8 @@ <h2>DAGs</h2>
                 {% if dag %}
 
                 <!-- Trigger Dag -->
-                <a href="{{ url_for('Airflow.trigger', dag_id=dag.dag_id) }}"
-                   onclick="return confirmTriggerDag('{{ dag.safe_dag_id }}')">
+                <a href="{{ url_for('Airflow.trigger', dag_id=dag.dag_id) }}&origin={{ request.base_url }}"
+                   onclick="return confirmTriggerDag(this, '{{ dag.safe_dag_id }}')">
                     <span class="glyphicon glyphicon-play-circle" aria-hidden="true" data-original-title="Trigger Dag"></span>
                 </a>
 
@@ -187,10 +169,16 @@ <h2>DAGs</h2>
                 {% endif %}
 
                 <!-- Refresh -->
-                <a href="{{ url_for("Airflow.refresh", dag_id=dag_id) }}">
+                <a href="{{ url_for("Airflow.refresh", dag_id=dag.dag_id) }}" onclick="postAsForm(this.href); return false">
                   <span class="glyphicon glyphicon-refresh" aria-hidden="true" data-original-title="Refresh"></span>
                 </a>
 
+                <!-- Delete -->
+                <!-- Use dag_id instead of dag.dag_id, because the DAG might not exist in the webserver's DagBag -->
+                <a href="{{ url_for('Airflow.delete', dag_id=dag.dag_id) }}"
+                  onclick="return confirmDeleteDag(this, '{{ dag.dag_id }}')">
+                   <span class="glyphicon glyphicon-remove-circle" style="color:red" aria-hidden="true" data-original-title="Delete Dag"></span>
+                </a>
                 </td>
             </tr>
         {% endfor %}
@@ -219,14 +207,15 @@ <h2>DAGs</h2>
 
 {% block tail %}
   {{ super() }}
-  <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='bootstrap-toggle.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='bootstrap3-typeahead.min.js') }}"></script>
+  <script src="{{ url_for_asset('d3.min.js') }}"></script>
+  <script src="{{ url_for_asset('jquery.dataTables.min.js') }}"></script>
+  <script src="{{ url_for_asset('dataTables.bootstrap.min.js') }}"></script>
+  <script src="{{ url_for_asset('bootstrap-toggle.min.js') }}"></script>
+  <script src="{{ url_for_asset('bootstrap3-typeahead.min.js') }}"></script>
   <script>
 
-      const DAGS_INDEX = "{{ url_for('Airflow.index') }}"
-      const ENTER_KEY_CODE = 13
+      const DAGS_INDEX = "{{ url_for('Airflow.index') }}";
+      const ENTER_KEY_CODE = 13;
 
       $('#dag_query').on('keypress', function (e) {
         // check for key press on ENTER (key code 13) to trigger the search
@@ -242,8 +231,22 @@ <h2>DAGs</h2>
         window.location = DAGS_INDEX + "?page_size=" + p_size;
       });
 
-      function confirmTriggerDag(dag_id){
-          return confirm("Are you sure you want to run '"+dag_id+"' now?");
+      function confirmTriggerDag(link, dag_id){
+        if (confirm("Are you sure you want to run '"+dag_id+"' now?")) {
+          postAsForm(link.href, {});
+        }
+        // Never follow the link
+        return false;
+      }
+
+      function confirmDeleteDag(link, dag_id){
+        if (confirm("Are you sure you want to delete '"+dag_id+"' now?\n\
+          This option will delete ALL metadata, DAG runs, etc.\n\
+          EXCEPT Log.\n\
+          This cannot be undone.")) {
+          postAsForm(link.href, {});
+        }
+        return false;
       }
       all_dags = $("[id^=toggle]");
       $.each(all_dags, function(i,v) {
@@ -390,7 +393,7 @@ <h2>DAGs</h2>
             states = json[dag_id];
             g = d3.select('svg#task-run-' + dag_id)
               .attr('height', diameter + (stroke_width_hover * 2))
-              .attr('width', '240px')
+              .attr('width', '300px')
               .selectAll("g")
               .data(states)
               .enter()
diff --git a/airflow/www_rbac/templates/airflow/duration_chart.html b/airflow/www_rbac/templates/airflow/duration_chart.html
index 89ddb946d3dfc..e2eafd4ea89b1 100644
--- a/airflow/www_rbac/templates/airflow/duration_chart.html
+++ b/airflow/www_rbac/templates/airflow/duration_chart.html
@@ -21,15 +21,14 @@
 {% block head_css %}
 {{ super() }}
 <link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='tree.css') }}">
+    href="{{ url_for('static', filename='css/tree.css') }}">
 <link rel="stylesheet" type="text/css"
     href="{{url_for('appbuilder.static',filename='datepicker/bootstrap-datepicker.css')}}" >
+<link href="{{ url_for_asset('dataTables.bootstrap.min.css') }}" rel="stylesheet" type="text/css" >
 <link rel="stylesheet" type="text/css"
-      href="{{ url_for("static", filename="dataTables.bootstrap.css") }}">
-<link rel="stylesheet" type="text/css"
-      href="{{ url_for("static", filename="nv.d3.css") }}">
-<script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-<script src="{{ url_for('static', filename='nv.d3.js') }}"></script>
+      href="{{ url_for_asset('nv.d3.min.css') }}">
+<script src="{{ url_for_asset('d3.min.js') }}"></script>
+<script src="{{ url_for_asset('nv.d3.min.js') }}"></script>
 {% endblock %}
 
 {% block content %}
diff --git a/airflow/www_rbac/templates/airflow/gantt.html b/airflow/www_rbac/templates/airflow/gantt.html
index 3a567ee2eb4cc..7d8135a5da454 100644
--- a/airflow/www_rbac/templates/airflow/gantt.html
+++ b/airflow/www_rbac/templates/airflow/gantt.html
@@ -21,8 +21,8 @@
 {{ super() }}
 <link rel="stylesheet" type="text/css"
     href="{{url_for('appbuilder.static',filename='datepicker/bootstrap-datepicker.css')}}" >
-<link type="text/css" href="{{ url_for('static', filename='gantt.css') }}" rel="stylesheet" />
-<link type="text/css" href="{{ url_for('static', filename='tree.css') }}" rel="stylesheet" />
+<link type="text/css" href="{{ url_for('static', filename='css/gantt.css') }}" rel="stylesheet" />
+<link type="text/css" href="{{ url_for('static', filename='css/tree.css') }}" rel="stylesheet" />
 {% endblock %}
 
 {% block content %}
@@ -47,15 +47,15 @@
 {% block tail %}
 {{ super() }}
 <script src="{{ url_for('appbuilder.static',filename='datepicker/bootstrap-datepicker.js') }}"></script>
-<script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-<script src="{{ url_for('static', filename='d3.tip.v0.6.3.js') }}"></script>/
-<script src="{{ url_for('static', filename='gantt-chart-d3v2.js') }}"></script>
+<script src="{{ url_for_asset('d3.min.js') }}"></script>
+<script src="{{ url_for_asset('d3-tip.js') }}"></script>/
+<script src="{{ url_for_asset('ganttChartD3v2.js') }}"></script>
 <script>
   $( document ).ready(function() {
     var dag_id = '{{ dag.dag_id }}';
     var task_id = '';
     var exection_date = '';
-    data = {{ data |safe }};
+    data = {{ data |tojson|safe }};
     var gantt = d3.gantt()
       .taskTypes(data.taskNames)
       .taskStatus(data.taskStatus)
diff --git a/airflow/www_rbac/templates/airflow/graph.html b/airflow/www_rbac/templates/airflow/graph.html
index 35f79b6d71edb..123d9e42a9212 100644
--- a/airflow/www_rbac/templates/airflow/graph.html
+++ b/airflow/www_rbac/templates/airflow/graph.html
@@ -1,13 +1,13 @@
-{# 
+{#
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
-  
+
     http://www.apache.org/licenses/LICENSE-2.0
-  
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,126 +21,169 @@
 
 {% block head_css %}
 {{ super() }}
-<link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='dagre.css') }}">
-<link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='graph.css') }}">
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/graph.css') }}">
 {% endblock %}
 
 {% block content %}
 {{ super() }}
-    {% if dag.doc_md %}
-        <div class="rich_doc" style="margin-bottom: 15px;">{{ doc_md|safe }}</div>
-    {% endif %}
-    <div class="form-inline">
-        <form method="get" style="float:left;">
-            {{ state_token }}
-            Base date: {{ form.base_date(class_="form-control") }}
-            Number of runs: {{ form.num_runs(class_="form-control") }}
-            Run:
-            {{ form.execution_date(class_="form-control") | safe }}
-            Layout:
-            {{ form.arrange(class_="form-control") | safe }}
-            <input type="hidden" name="root" value="{{ root }}">
-            <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
-            <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
-            <input type="submit" value="Go" class="btn btn-default"
-            action="" method="get">
-        </form>
-            <div class="input-group" style="float: right;">
-              <input type="text" id="searchbox" class="form-control" placeholder="Search for..." onenter="null">
-            </div><!-- /input-group -->
-        <div style="clear: both;"/>
-    </div>
+{% if dag.doc_md %}
+<div class="rich_doc" style="margin-bottom: 15px;">{{ dag.doc_md|safe }}</div>
+{% endif %}
+<div class="form-inline">
+  <form method="get" style="float:left;">
+    {{ state_token }}
+    Base date: {{ form.base_date(class_="form-control") }}
+    Number of runs: {{ form.num_runs(class_="form-control") }}
+    Run:
+    {{ form.execution_date(class_="form-control") | safe }}
+    Layout:
+    {{ form.arrange(class_="form-control") | safe }}
+    <input type="hidden" name="root" value="{{ root }}">
+    <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
+    <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
+    <input type="submit" value="Go" class="btn btn-default">
+  </form>
+  <div class="input-group" style="float: right;">
+    <input type="text" id="searchbox" class="form-control" placeholder="Search for..." onenter="null">
+  </div>
+  <div style="clear: both;"></div>
+</div>
 <hr/>
 <div>
-    {% for op in operators %}
-    <div class="legend_item" style="border-width:1px;float:left;background:{{ op.ui_color }};color:{{ op.ui_fgcolor }};">
-        {{ op.__name__ }}
-    </div>
-    {% endfor %}
-
-    <div style"background-color: blue;">
-    <div class="legend_item state" style="border-color:white;">no status</div>
+  {% for op in operators %}
+  <div class="legend_item" style="border-width:1px;float:left;background:{{ op.ui_color }};color:{{ op.ui_fgcolor }};">
+      {{ op.__name__ }}
+  </div>
+  {% endfor %}
+
+  <div>
+    <div class="legend_item state" style="border-color:white;">no_status</div>
     <div class="legend_item state" style="border-color:grey;">queued</div>
-    <div class="legend_item state" style="border-color:gold;">retry</div>
+    <div class="legend_item state" style="border-color:gold;">up_for_retry</div>
+    <div class="legend_item state" style="border-color:turquoise;">up_for_reschedule</div>
     <div class="legend_item state" style="border-color:pink;">skipped</div>
     <div class="legend_item state" style="border-color:red;">failed</div>
     <div class="legend_item state" style="border-color:lime;">running</div>
     <div class="legend_item state" style="border-color:green;">success</div>
-    </div>
-    <div style="clear:both;"></div>
+  </div>
+  <div style="clear:both;"></div>
 </div>
 <div id="error" style="display: none; margin-top: 10px;" class="alert alert-danger" role="alert">
-        <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
-        <span id="error_msg">Oops.</span>
+  <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
+  <span id="error_msg">Oops.</span>
 </div>
 <hr style="margin-bottom: 0px;"/>
 <button class="btn btn-default pull-right" id="refresh_button">
-    <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
+  <span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
 </button>
 <div id="svg_container">
 
-    <svg width="{{ width }}" height="{{ height }}">
-        <g id='dig' transform="translate(20,20)"/>
-        <filter id="blur-effect-1">
-        <feGaussianBlur stdDeviation="3" />
-        </filter>
-    </svg>
-    <img id="loading" alt="spinner" src="{{ url_for('static', filename='loading.gif') }}">
+  <svg width="{{ width }}" height="{{ height }}">
+    <g id='dig' transform="translate(20,20)"/>
+    <filter id="blur-effect-1">
+    <feGaussianBlur stdDeviation="3" />
+    </filter>
+  </svg>
+  <img id="loading" alt="spinner" src="{{ url_for('static', filename='loading.gif') }}">
 </div>
 <hr>
 {% endblock %}
 
 {% block tail %}
-    {{ super() }}
+{{ super() }}
 
-    <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-    <script src="{{ url_for('static', filename='dagre-d3.js') }}"></script>
-    <script>
+<script src="{{ url_for_asset('d3.min.js') }}"></script>
+<script src="{{ url_for_asset('dagre-d3.min.js') }}"></script>
+<script>
 
     var highlight_color = "#000000";
     var upstream_color = "#2020A0";
     var downstream_color = "#0000FF";
+    var initialStrokeWidth = '3px';
+    var highlightStrokeWidth = '5px';
 
-    var nodes = {{ nodes|safe }};
-    var edges = {{ edges|safe }};
-    var tasks = {{ tasks|safe }};
-    var task_instances = {{ task_instances|safe }};
+    var nodes = {{ nodes|tojson|safe }};
+    var edges = {{ edges|tojson|safe }};
     var execution_date = "{{ execution_date }}";
     var arrange = "{{ arrange }}";
-    var g = dagreD3.json.decode(nodes, edges);
+
+    // Below variables are being used in dag.js
+    var tasks = {{ tasks|tojson|safe }};
+    var task_instances = {{ task_instances|tojson|safe }};
+    var getTaskInstanceURL = "{{ url_for('Airflow.task_instances') }}" +
+      "?dag_id=" + encodeURIComponent(dag_id) + "&execution_date=" +
+      encodeURIComponent(execution_date);
+
     var duration = 500;
     var stateFocusMap = {
-        'no status':false, 'failed':false, 'running':false,
-        'queued': false, 'success': false};
+        'success': false,
+        'running':false,
+        'failed':false,
+        'skipped': false,
+        'up_for_reschedule': false,
+        'up_for_retry': false,
+        'queued': false,
+        'no_status': false
+    };
+
+    // Preparation of DagreD3 data structures
+    var g = new dagreD3.graphlib.Graph().setGraph({
+        nodesep: 15,
+        ranksep: 15,
+        rankdir: arrange,
+      })
+      .setDefaultEdgeLabel(function() { return { lineInterpolate: 'basis' } });
+
+    // Set all nodes and styles
+    nodes.forEach(function(node) {
+      g.setNode(node.id, node.value)
+    });
+
+    // Set edges
+    edges.forEach(function(edge) {
+      g.setEdge(edge.u, edge.v)
+    });
 
+    var render = dagreD3.render(),
+      svg = d3.select("svg"),
+      innerSvg = d3.select("svg g");
+
+    innerSvg.call(render, g);
+
+    function setUpZoomSupport() {
+      // Set up zoom support for Graph
+      var zoom = d3.behavior.zoom().on("zoom", function() {
+            innerSvg.attr("transform", "translate(" + d3.event.translate + ")" +
+                                        "scale(" + d3.event.scale + ")");
+          });
+      svg.call(zoom);
+    }
 
-    var layout = dagreD3.layout().rankDir(arrange).nodeSep(15).rankSep(15);
-    var renderer = new dagreD3.Renderer();
-    renderer.layout(layout).run(g, d3.select("#dig"));
+    // Centering the DAG on load (Not implemented yet)
+    // https://github.com/dagrejs/dagre-d3/issues/245
+
+    setUpZoomSupport();
     inject_node_ids(tasks);
-    update_nodes_states(task_instances);
 
     d3.selectAll("g.node").on("click", function(d){
         task = tasks[d];
         if (task.task_type == "SubDagOperator")
-            call_modal(d, execution_date, true);
+            call_modal(d, execution_date, task_instances[d].try_number, true);
         else
-            call_modal(d, execution_date);
+            call_modal(d, execution_date,task_instances[d].try_number);
     });
 
 
     function highlight_nodes(nodes, color) {
         nodes.forEach (function (nodeid) {
-            my_node = d3.select('#' + nodeid + ' rect');
-            my_node.style("stroke", color) ;
+            my_node = d3.select('#' + nodeid).node().parentNode;
+            d3.select(my_node).selectAll("rect").style("stroke", color) ;
         })
     }
 
     d3.selectAll("g.node").on("mouseover", function(d){
         d3.select(this).selectAll("rect").style("stroke", highlight_color) ;
-        highlight_nodes(g.predecessors(d), upstream_color)
+        highlight_nodes(g.predecessors(d), upstream_color);
         highlight_nodes(g.successors(d), downstream_color)
 
     });
@@ -209,7 +252,7 @@
                     .transition().duration(duration)
                     .style("opacity", 1)
                     .selectAll("rect")
-                    .style("stroke-width", "2px");
+                    .style("stroke-width", initialStrokeWidth);
             }
             else{
                 d3.select("g.edgePaths")
@@ -222,17 +265,21 @@
                         .transition().duration(duration)
                         .style("opacity", 1)
                         .selectAll("rect")
-                        .style("stroke-width", "10px");
+                        .style("stroke-width", highlightStrokeWidth);
                 }
                 else {
                     d3.select(this)
                         .transition()
                         .style("opacity", 0.2).duration(duration)
                         .selectAll("rect")
-                        .style("stroke-width", "2px");
+                        .style("stroke-width", initialStrokeWidth);
                 }
             }
         });
+
+        // This moves the matched node in the center of the graph area
+        // ToDo: Should we keep this here as it has no added value
+        // and does not fit the graph on small screens, and has to scroll
         if(match) {
             var transform = d3.transform(d3.select(match).attr("transform"));
             transform.translate = [
@@ -244,8 +291,8 @@
             d3.select("g.zoom")
                 .transition()
                 .attr("transform", transform.toString());
-            renderer.zoom_obj.translate(transform.translate);
-            renderer.zoom_obj.scale(1);
+            innerSvg.attr("transform", "translate(" + transform.translate + ")" +
+                                        "scale(1)");
         }
     });
 
@@ -261,39 +308,13 @@
         });
     }
 
-
-    // Assigning css classes based on state to nodes
-    function update_nodes_states(task_instances) {
-        $.each(task_instances, function(task_id, ti) {
-          $('tspan').filter(function(index) { return $(this).text() === task_id; })
-            .parent().parent().parent()
-            .attr("class", "node enter " + ti.state)
-            .attr("data-toggle", "tooltip")
-            .attr("data-original-title", function(d) {
-              // Tooltip
-              task = tasks[task_id];
-              tt =  "Task_id: " + ti.task_id + "<br>";
-              tt += "Run: " + ti.execution_date + "<br>";
-              if(ti.run_id != undefined){
-                tt += "run_id: <nobr>" + ti.run_id + "</nobr><br>";
-              }
-              tt += "Operator: " + task.task_type + "<br>";
-              tt += "Started: " + ti.start_date + "<br>";
-              tt += "Ended: " + ti.end_date + "<br>";
-              tt += "Duration: " + ti.duration + "<br>";
-              tt += "State: " + ti.state + "<br>";
-              return tt;
-            });
-        });
-    }
-
     function clearFocus(){
         d3.selectAll("g.node")
             .transition(duration)
             .style("opacity", 1);
         d3.selectAll("g.node rect")
             .transition(duration)
-            .style("stroke-width", "2px");
+            .style("stroke-width", initialStrokeWidth);
         d3.select("g.edgePaths")
             .transition().duration(duration)
             .style("opacity", 1);
@@ -310,7 +331,7 @@
             .style("opacity", 1);
         d3.selectAll("g.node." + state + " rect")
             .transition(duration)
-            .style("stroke-width", "10px")
+            .style("stroke-width", highlightStrokeWidth)
             .style("opacity", 1);
         d3.select("g.edgePaths")
             .transition().duration(duration)
@@ -336,36 +357,8 @@
         }
         return false
     }
-
-    function error(msg){
-      $('#error_msg').html(msg);
-      $('#error').show();
-      $('#loading').hide();
-      $('#chart_section').hide(1000);
-      $('#datatable_section').hide(1000);
-    }
-
-    d3.select("#refresh_button").on("click",
-        function() {
-            $("#loading").css("display", "block");
-            $("div#svg_container").css("opacity", "0.2");
-            $.get(
-                "/airflow/object/task_instances",
-                {dag_id : "{{ dag.dag_id }}", execution_date : "{{ execution_date }}"})
-            .done(
-                function(task_instances) {
-                    update_nodes_states(JSON.parse(task_instances));
-                    $("#loading").hide();
-                    $("div#svg_container").css("opacity", "1");
-                    $('#error').hide();
-                }
-            ).fail(function(jqxhr, textStatus, err) {
-                error(textStatus + ': ' + err);
-            });
-        }
-    );
-
-    </script>
+</script>
+<script src="{{ url_for_asset('graph.js') }}"></script>
 
 
 {% endblock %}
diff --git a/airflow/www_rbac/templates/airflow/task.html b/airflow/www_rbac/templates/airflow/task.html
index 6c843b28e3e5e..2b4356f49ec60 100644
--- a/airflow/www_rbac/templates/airflow/task.html
+++ b/airflow/www_rbac/templates/airflow/task.html
@@ -37,7 +37,9 @@ <h5>Dependencies Blocking Task From Getting Scheduled</h5>
                 </tr>
             {% endfor %}
         </table>
-        {{ html_code|safe }}
+        {% if html_code is defined %}
+            {{ html_code|safe }}
+        {% endif %}
     </div>
     <div>
         {% for attr, value in special_attrs_rendered.items() %}
@@ -70,6 +72,8 @@ <h5>Task Attributes</h5>
                 </tr>
             {% endfor %}
         </table>
-        {{ html_code|safe }}
+        {% if html_code is defined %}
+            {{ html_code|safe }}
+        {% endif %}
     </div>
 {% endblock %}
diff --git a/airflow/www_rbac/templates/airflow/ti_code.html b/airflow/www_rbac/templates/airflow/ti_code.html
index d38eb7d6c936d..8c7db0d9d415d 100644
--- a/airflow/www_rbac/templates/airflow/ti_code.html
+++ b/airflow/www_rbac/templates/airflow/ti_code.html
@@ -21,23 +21,8 @@
 {% block content %}
     {{ super() }}
     <h4>{{ title }}</h4>
-    {% if html_code %}
-        {{ html_code|safe }}
-    {% endif %}
-    {% if code %}
-        <pre>{{ code }}</pre>
-    {% endif %}
-
-    {% if code_dict %}
-        {% for k, v in code_dict.items() %}
-            <h5>{{ k }}</h5>
-            <pre>{{ v }}</pre>
-        {% endfor %}
-    {% endif %}
-    {% if html_dict %}
-        {% for k, v in html_dict.items() %}
-            <h5>{{ k }}</h5>
-            {{ v|safe }}
-        {% endfor %}
-    {% endif %}
+    {% for k, v in html_dict.items() %}
+        <h5>{{ k }}</h5>
+        {{ v|safe }}
+    {% endfor %}
 {% endblock %}
diff --git a/airflow/www_rbac/templates/airflow/tree.html b/airflow/www_rbac/templates/airflow/tree.html
index 10288619238ec..904d0b007a42f 100644
--- a/airflow/www_rbac/templates/airflow/tree.html
+++ b/airflow/www_rbac/templates/airflow/tree.html
@@ -20,67 +20,68 @@
 
 {% block head_css %}
 {{ super() }}
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/tree.css') }}">
 <link rel="stylesheet" type="text/css"
-    href="{{ url_for('static', filename='tree.css') }}">
-<link href="{{ url_for('static', filename='appbuilder/daterangepicker/bootstrap-datepicker.css') }}" rel="stylesheet">
+    href="{{url_for('appbuilder.static',filename='datepicker/bootstrap-datepicker.css')}}" >
 {% endblock %}
 
 {% block content %}
 {{ super() }}
 <div style="float: left" class="form-inline">
-    <form method="get" style="float:left;">
-        Base date: {{ form.base_date(class_="form-control") }}
-        Number of runs: {{ form.num_runs(class_="form-control") }}
-        <input type="hidden" name="root" value="{{ root if root else '' }}">
-        <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
-        <input type="submit" value="Go" class="btn btn-default"
-        action="" method="get">
-        <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
-    </form>
+  <form method="get" style="float:left;">
+    Base date: {{ form.base_date(class_="form-control") }}
+    Number of runs: {{ form.num_runs(class_="form-control") }}
+    <input type="hidden" name="root" value="{{ root if root else '' }}">
+    <input type="hidden" value="{{ dag.dag_id }}" name="dag_id">
+    <input type="submit" value="Go" class="btn btn-default"
+                                    action="" method="get">
+    <input name="_csrf_token" type="hidden" value="{{ csrf_token() }}">
+  </form>
 </div>
 <div style="clear: both;"></div>
 <hr/>
 <div>
-    <div class="legend_item" style="border: none;">no status</div>
-    <div class="square" style="background: white;"></div>
-    <div class="legend_item" style="border: none;">queued</div>
-    <div class="square" style="background: grey;"></div>
-    <div class="legend_item" style="border: none;">retry</div>
-    <div class="square" style="background: gold;"></div>
-    <div class="legend_item" style="border: none;">skipped</div>
-    <div class="square" style="background: pink;"></div>
-    <div class="legend_item" style="border: none;">failed</div>
-    <div class="square" style="background: red;"></div>
-    <div class="legend_item" style="border: none;">running</div>
-    <div class="square" style="background: lime;"></div>
-    <div class="legend_item" style="border: none;">success</div>
-    <div class="square" style="background: green;"></div>
-    {% for op in operators %}
-        <div class="legend_circle" style="background:{{ op.ui_color }};">
-        </div>
-        <div class="legend_item" style="float:left;border-color:white;">{{ op.__name__ }}</div>
-    {% endfor %}
-    <div style="clear:both;"></div>
+  <div class="legend_item" style="border: none;">no_status</div>
+  <div class="square" style="background: white;"></div>
+  <div class="legend_item" style="border: none;">queued</div>
+  <div class="square" style="background: grey;"></div>
+  <div class="legend_item" style="border: none;">up_for_retry</div>
+  <div class="square" style="background: gold;"></div>
+  <div class="legend_item" style="border: none;">up_for_reschedule</div>
+  <div class="square" style="background: turquoise;"></div>
+  <div class="legend_item" style="border: none;">skipped</div>
+  <div class="square" style="background: pink;"></div>
+  <div class="legend_item" style="border: none;">failed</div>
+  <div class="square" style="background: red;"></div>
+  <div class="legend_item" style="border: none;">running</div>
+  <div class="square" style="background: lime;"></div>
+  <div class="legend_item" style="border: none;">success</div>
+  <div class="square" style="background: green;"></div>
+  {% for op in operators %}
+  <div class="legend_circle" style="background:{{ op.ui_color }};">
+  </div>
+  <div class="legend_item" style="float:left;border-color:white;">{{ op.__name__ }}</div>
+  {% endfor %}
+  <div style="clear:both;"></div>
 </div>
 <hr/>
 <div id="svg_container">
-  <img id='loading' width="50"
-    src="{{ url_for('static', filename='loading.gif') }}">
-    <svg class='tree' width="100%">
-        <filter id="blur-effect-1">
-            <feGaussianBlur stdDeviation="3" />
-        </filter>
-    </svg>
+  <img id='loading' width="50" src="{{ url_for('static', filename='loading.gif') }}">
+  <svg class='tree' width="100%">
+    <filter id="blur-effect-1">
+      <feGaussianBlur stdDeviation="3"></feGaussianBlur>
+    </filter>
+  </svg>
 </div>
 {% endblock %}
 
 {% block tail %}
-  {{ super() }}
-  <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
-  <script>
+{{ super() }}
+<script src="{{ url_for_asset('d3.min.js') }}"></script>
+<script>
 $('span.status_square').tooltip({html: true});
 
-var data = {{ data|safe }};
+var data = {{ data|tojson|safe }};
 var barHeight = 20;
 var axisHeight = 40;
 var square_x = 500;
@@ -198,13 +199,15 @@
       .attr("title", function(d){
         var tt = "";
         if (d.operator != undefined) {
-          tt += "operator: " + d.operator + "<br/>";
-          tt += "depends_on_past: " + d.depends_on_past + "<br/>";
-          tt += "upstream: " + d.num_dep + "<br/>";
-          tt += "retries: " + d.retries + "<br/>";
-          tt += "owner: " + d.owner + "<br/>";
-          tt += "start_date: " + d.start_date + "<br/>";
-          tt += "end_date: " + d.end_date + "<br/>";
+          if (d.operator != undefined) {
+            tt += "operator: " + escapeHtml(d.operator) + "<br/>";
+          }
+          tt += "depends_on_past: " + escapeHtml(d.depends_on_past) + "<br/>";
+          tt += "upstream: " + escapeHtml(d.num_dep) + "<br/>";
+          tt += "retries: " + escapeHtml(d.retries) + "<br/>";
+          tt += "owner: " + escapeHtml(d.owner) + "<br/>";
+          tt += "start_date: " + escapeHtml(d.start_date) + "<br/>";
+          tt += "end_date: " + escapeHtml(d.end_date) + "<br/>";
         }
         return tt;
       })
@@ -233,9 +236,9 @@
         if(d.task_id === undefined)
             call_modal_dag(d);
         else if(nodeobj[d.task_id].operator=='SubDagOperator')
-            call_modal(d.task_id, d.execution_date, true);
+            call_modal(d.task_id, d.execution_date, d.try_number, true);
         else
-            call_modal(d.task_id, d.execution_date);
+            call_modal(d.task_id, d.execution_date, d.try_number);
       })
       .attr("class", function(d) {return "state " + d.state})
       .attr("data-toggle", "tooltip")
@@ -245,17 +248,24 @@
       .style("stroke-width", function(d) {return (d.run_id != undefined)? "2": "1"})
       .style("stroke-opacity", function(d) {return d.external_trigger ? "0": "1"})
       .attr("title", function(d){
-        s =  "Task_id: " + d.task_id + "<br>";
-        s += "Run: " + d.execution_date + "<br>";
+        var s = "";
+        if (d.task_id != undefined) {
+          s += "Task_id: " + escapeHtml(d.task_id) + "<br>";
+        }
+        s += "Run: " + escapeHtml(d.execution_date) + "<br>";
         if(d.run_id != undefined){
-          s += "run_id: <nobr>" + d.run_id + "</nobr><br>";
+          s += "run_id: <nobr>" + escapeHtml(d.run_id) + "</nobr><br>";
+        }
+        if (d.operator != undefined) {
+          s += "Operator: " + escapeHtml(d.operator) + "<br>"
         }
-        s += "Operator: " + d.operator + "<br>"
         if(d.start_date != undefined){
-          s += "Started: " + d.start_date + "<br>";
-          s += "Ended: " + d.end_date + "<br>";
-          s += "Duration: " + d.duration + "<br>";
-          s += "State: " + d.state + "<br>";
+          s += "Started: " + escapeHtml(d.start_date) + "<br>";
+          s += "Ended: " + escapeHtml(d.end_date) + "<br>";
+          if (d.duration != undefined) {
+            s += "Duration: " + escapeHtml(d.duration) + "<br>";
+          }
+          s += "State: " + escapeHtml(d.state) + "<br>";
         }
         return s;
       })
@@ -377,5 +387,6 @@
   }
 }
 set_tooltip();
-  </script>
+
+</script>
 {% endblock %}
diff --git a/airflow/www_rbac/templates/appbuilder/baselayout.html b/airflow/www_rbac/templates/appbuilder/baselayout.html
index eeb3d5c04032e..9cf6145cb8053 100644
--- a/airflow/www_rbac/templates/appbuilder/baselayout.html
+++ b/airflow/www_rbac/templates/appbuilder/baselayout.html
@@ -20,9 +20,15 @@
 
 {% block head_css %}
   {{ super() }}
-  <link href="{{ url_for('static', filename='main.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='bootstrap-theme.css') }}" rel="stylesheet">
-  <link rel="icon" type="image/png" href="{{ url_for("static", filename="pin_30.png") }}">
+
+
+  {% if not appbuilder.app_theme %}
+    {# airflowDefaultTheme.css file contains the styles from local bootstrap-theme.css #}
+    <link href="{{ url_for_asset('airflowDefaultTheme.css') }}" rel="stylesheet">
+  {% endif %}
+  <link href="{{ url_for_asset('main.css') }}" rel="stylesheet">
+
+  <link rel="icon" type="image/png" href="{{ url_for('static', filename='pin_30.png') }}">
 {% endblock %}
 
 
@@ -34,7 +40,7 @@
         {% include 'appbuilder/navbar.html' %}
         </header>
     {% endblock %}
-   
+
 
     <div class="container">
       <div class="row">
@@ -60,25 +66,10 @@
 
 {% block tail_js %}
 {{ super() }}
-<script src="{{ url_for('static', filename='jqClock.min.js') }}" type="text/javascript"></script>
-<script>
-    x = new Date()
-    var UTCseconds = (x.getTime() + x.getTimezoneOffset()*60*1000);
-    $("#clock").clock({
-        "dateFormat":"Y-m-d ",
-        "timeFormat":"H:i:s %UTC%",
-        "timestamp":UTCseconds
-    }).click(function(){
-        alert('{{ hostname }}');
-    });
-    $('span').tooltip();
-
-    $.ajaxSetup({
-      beforeSend: function(xhr, settings) {
-        if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-          xhr.setRequestHeader("X-CSRFToken", "{{ csrf_token() }}");
-        }
-      }
-    });
+<script type="text/javascript">
+  // below variables are used in base.js
+  var hostName = '{{ hostname }}';
+  var csrfToken = '{{ csrf_token() }}';
 </script>
-{% endblock %}
\ No newline at end of file
+<script src="{{ url_for_asset('base.js') }}" type="text/javascript"></script>
+{% endblock %}
diff --git a/airflow/www_rbac/templates/appbuilder/navbar_menu.html b/airflow/www_rbac/templates/appbuilder/navbar_menu.html
index 5661e894762b3..9cf3f94f4094e 100644
--- a/airflow/www_rbac/templates/appbuilder/navbar_menu.html
+++ b/airflow/www_rbac/templates/appbuilder/navbar_menu.html
@@ -23,7 +23,7 @@
     {{_(item.label)}}</a>
 {% endmacro %}
 
-<li class="dropdown"><a href="/">DAGs</a></li>
+<li class="dropdown"><a href="{{ url_for('Airflow.index') }}">DAGs</a></li>
 
 {% for item1 in menu.get_list() %}
     {% if item1 | is_menu_visible %}
diff --git a/airflow/www_rbac/templates/appbuilder/navbar_right.html b/airflow/www_rbac/templates/appbuilder/navbar_right.html
index bf5aa43221884..d42f8e2e8a82c 100644
--- a/airflow/www_rbac/templates/appbuilder/navbar_right.html
+++ b/airflow/www_rbac/templates/appbuilder/navbar_right.html
@@ -47,7 +47,7 @@
 <!-- clock -->
 <li><a id="clock"></a></li>
 
-{% if not current_user.is_anonymous() %}
+{% if not current_user.is_anonymous %}
     <li class="dropdown">
         <a class="dropdown-toggle" data-toggle="dropdown" href="#">
            <span class="fa fa-user"></span> {{g.user.get_full_name()}}<b class="caret"></b>
diff --git a/airflow/www_rbac/utils.py b/airflow/www_rbac/utils.py
index 7bbdada5556f2..23a07f923b1aa 100644
--- a/airflow/www_rbac/utils.py
+++ b/airflow/www_rbac/utils.py
@@ -23,9 +23,11 @@
 import inspect
 import json
 import time
-import wtforms
-import bleach
 import markdown
+import re
+import zipfile
+import os
+import io
 
 from builtins import str
 from past.builtins import basestring
@@ -33,6 +35,11 @@
 from pygments import highlight, lexers
 from pygments.formatters import HtmlFormatter
 from flask import request, Response, Markup, url_for
+from flask_appbuilder.models.sqla.interface import SQLAInterface
+import flask_appbuilder.models.sqla.filters as fab_sqlafilters
+import sqlalchemy as sqla
+from six.moves.urllib.parse import urlencode
+
 from airflow import configuration
 from airflow.models import BaseOperator
 from airflow.operators.subdag_operator import SubDagOperator
@@ -54,22 +61,21 @@
 
 
 def should_hide_value_for_key(key_name):
-    return any(s in key_name.lower() for s in DEFAULT_SENSITIVE_VARIABLE_FIELDS) \
-        and configuration.getboolean('admin', 'hide_sensitive_variable_fields')
+    # It is possible via importing variables from file that a key is empty.
+    if key_name:
+        config_set = configuration.conf.getboolean('admin',
+                                                   'hide_sensitive_variable_fields')
+        field_comp = any(s in key_name.lower() for s in DEFAULT_SENSITIVE_VARIABLE_FIELDS)
+        return config_set and field_comp
+    return False
 
 
 def get_params(**kwargs):
-    params = []
-    for k, v in kwargs.items():
-        if k == 'showPaused':
-            # True is default or None
-            if v or v is None:
-                continue
-            params.append('{}={}'.format(k, v))
-        elif v:
-            params.append('{}={}'.format(k, v))
-    params = sorted(params, key=lambda x: x.split('=')[0])
-    return '&'.join(params)
+    if 'showPaused' in kwargs:
+        v = kwargs['showPaused']
+        if v or v is None:
+            kwargs.pop('showPaused')
+    return urlencode({d: v if v is not None else '' for d, v in kwargs.items()})
 
 
 def generate_pages(current_page, num_of_pages,
@@ -100,27 +106,27 @@ def generate_pages(current_page, num_of_pages,
     """
 
     void_link = 'javascript:void(0)'
-    first_node = """<li class="paginate_button {disabled}" id="dags_first">
+    first_node = Markup("""<li class="paginate_button {disabled}" id="dags_first">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&laquo;</a>
-</li>"""
+</li>""")
 
-    previous_node = """<li class="paginate_button previous {disabled}" id="dags_previous">
+    previous_node = Markup("""<li class="paginate_button previous {disabled}" id="dags_previous">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&lt;</a>
-</li>"""
+</li>""")
 
-    next_node = """<li class="paginate_button next {disabled}" id="dags_next">
+    next_node = Markup("""<li class="paginate_button next {disabled}" id="dags_next">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&gt;</a>
-</li>"""
+</li>""")
 
-    last_node = """<li class="paginate_button {disabled}" id="dags_last">
+    last_node = Markup("""<li class="paginate_button {disabled}" id="dags_last">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&raquo;</a>
-</li>"""
+</li>""")
 
-    page_node = """<li class="paginate_button {is_active}">
+    page_node = Markup("""<li class="paginate_button {is_active}">
     <a href="{href_link}" aria-controls="dags" data-dt-idx="2" tabindex="0">{page_num}</a>
-</li>"""
+</li>""")
 
-    output = ['<ul class="pagination" style="margin-top:0px;">']
+    output = [Markup('<ul class="pagination" style="margin-top:0px;">')]
 
     is_disabled = 'disabled' if current_page <= 0 else ''
     output.append(first_node.format(href_link="?{}"
@@ -176,9 +182,9 @@ def is_current(current, page):
                                                       showPaused=showPaused)),
                                    disabled=is_disabled))
 
-    output.append('</ul>')
+    output.append(Markup('</ul>'))
 
-    return wtforms.widgets.core.HTMLString('\n'.join(output))
+    return Markup('\n'.join(output))
 
 
 def epoch(dttm):
@@ -197,6 +203,22 @@ def json_response(obj):
         mimetype="application/json")
 
 
+def open_maybe_zipped(f, mode='r'):
+    """
+    Opens the given file. If the path contains a folder with a .zip suffix, then
+    the folder is treated as a zip archive, opening the file inside the archive.
+
+    :return: a file object, as in `open`, or as in `ZipFile.open`.
+    """
+
+    _, archive, filename = re.search(
+        r'((.*\.zip){})?(.*)'.format(re.escape(os.sep)), f).groups()
+    if archive and zipfile.is_zipfile(archive):
+        return zipfile.ZipFile(archive, mode=mode).open(filename)
+    else:
+        return io.open(f, mode=mode)
+
+
 def make_cache_key(*args, **kwargs):
     """
     Used by cache to get a unique key per URL
@@ -207,8 +229,8 @@ def make_cache_key(*args, **kwargs):
 
 
 def task_instance_link(attr):
-    dag_id = bleach.clean(attr.get('dag_id')) if attr.get('dag_id') else None
-    task_id = bleach.clean(attr.get('task_id')) if attr.get('task_id') else None
+    dag_id = attr.get('dag_id')
+    task_id = attr.get('task_id')
     execution_date = attr.get('execution_date')
     url = url_for(
         'Airflow.task',
@@ -229,14 +251,14 @@ def task_instance_link(attr):
             aria-hidden="true"></span>
         </a>
         </span>
-        """.format(**locals()))
+        """).format(**locals())
 
 
 def state_token(state):
     color = State.color(state)
     return Markup(
         '<span class="label" style="background-color:{color};">'
-        '{state}</span>'.format(**locals()))
+        '{state}</span>').format(**locals())
 
 
 def state_f(attr):
@@ -247,7 +269,7 @@ def state_f(attr):
 def nobr_f(attr_name):
     def nobr(attr):
         f = attr.get(attr_name)
-        return Markup("<nobr>{}</nobr>".format(f))
+        return Markup("<nobr>{}</nobr>").format(f)
     return nobr
 
 
@@ -257,24 +279,24 @@ def dt(attr):
         f = f.isoformat() if f else ''
         if timezone.utcnow().isoformat()[:4] == f[:4]:
             f = f[5:]
-        return Markup("<nobr>{}</nobr>".format(f))
+        return Markup("<nobr>{}</nobr>").format(f)
     return dt
 
 
 def dag_link(attr):
-    dag_id = bleach.clean(attr.get('dag_id')) if attr.get('dag_id') else None
+    dag_id = attr.get('dag_id')
     execution_date = attr.get('execution_date')
     url = url_for(
         'Airflow.graph',
         dag_id=dag_id,
         execution_date=execution_date)
     return Markup(
-        '<a href="{}">{}</a>'.format(url, dag_id))
+        '<a href="{}">{}</a>').format(url, dag_id)
 
 
 def dag_run_link(attr):
-    dag_id = bleach.clean(attr.get('dag_id')) if attr.get('dag_id') else None
-    run_id = bleach.clean(attr.get('run_id')) if attr.get('run_id') else None
+    dag_id = attr.get('dag_id')
+    run_id = attr.get('run_id')
     execution_date = attr.get('execution_date')
     url = url_for(
         'Airflow.graph',
@@ -282,7 +304,7 @@ def dag_run_link(attr):
         run_id=run_id,
         execution_date=execution_date)
     return Markup(
-        '<a href="{url}">{run_id}</a>'.format(**locals()))
+        '<a href="{url}">{run_id}</a>').format(**locals())
 
 
 def pygment_html_render(s, lexer=lexers.TextLexer):
@@ -353,3 +375,67 @@ def get_chart_height(dag):
     charts, that is charts that take up space based on the size of the components within.
     """
     return 600 + len(dag.tasks) * 10
+
+
+class UtcAwareFilterMixin(object):
+    def apply(self, query, value):
+        value = timezone.parse(value, timezone=timezone.utc)
+
+        return super(UtcAwareFilterMixin, self).apply(query, value)
+
+
+class UtcAwareFilterEqual(UtcAwareFilterMixin, fab_sqlafilters.FilterEqual):
+    pass
+
+
+class UtcAwareFilterGreater(UtcAwareFilterMixin, fab_sqlafilters.FilterGreater):
+    pass
+
+
+class UtcAwareFilterSmaller(UtcAwareFilterMixin, fab_sqlafilters.FilterSmaller):
+    pass
+
+
+class UtcAwareFilterNotEqual(UtcAwareFilterMixin, fab_sqlafilters.FilterNotEqual):
+    pass
+
+
+class UtcAwareFilterConverter(fab_sqlafilters.SQLAFilterConverter):
+
+    conversion_table = (
+        (('is_utcdatetime', [UtcAwareFilterEqual,
+                             UtcAwareFilterGreater,
+                             UtcAwareFilterSmaller,
+                             UtcAwareFilterNotEqual]),) +
+        fab_sqlafilters.SQLAFilterConverter.conversion_table
+    )
+
+
+class CustomSQLAInterface(SQLAInterface):
+    """
+    FAB does not know how to handle columns with leading underscores because
+    they are not supported by WTForm. This hack will remove the leading
+    '_' from the key to lookup the column names.
+
+    """
+    def __init__(self, obj):
+        super(CustomSQLAInterface, self).__init__(obj)
+
+        def clean_column_names():
+            if self.list_properties:
+                self.list_properties = dict(
+                    (k.lstrip('_'), v) for k, v in self.list_properties.items())
+            if self.list_columns:
+                self.list_columns = dict(
+                    (k.lstrip('_'), v) for k, v in self.list_columns.items())
+
+        clean_column_names()
+
+    def is_utcdatetime(self, col_name):
+        from airflow.utils.sqlalchemy import UtcDateTime
+        obj = self.list_columns[col_name].type
+        return isinstance(obj, UtcDateTime) or \
+            isinstance(obj, sqla.types.TypeDecorator) and \
+            isinstance(obj.impl, UtcDateTime)
+
+    filter_converter_class = UtcAwareFilterConverter
diff --git a/airflow/www_rbac/validators.py b/airflow/www_rbac/validators.py
index 7746e74338023..62b7db82523bc 100644
--- a/airflow/www_rbac/validators.py
+++ b/airflow/www_rbac/validators.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/airflow/www_rbac/views.py b/airflow/www_rbac/views.py
index 43e481ed0278c..044b553c531c6 100644
--- a/airflow/www_rbac/views.py
+++ b/airflow/www_rbac/views.py
@@ -18,67 +18,66 @@
 # under the License.
 #
 
-from past.builtins import unicode
-
+import copy
+from io import BytesIO
+import itertools
+import json
 import logging
+import math
 import os
 import socket
-from datetime import datetime, timedelta
-import copy
-import math
-import json
-from collections import defaultdict
 import traceback
+from collections import defaultdict
+from datetime import timedelta
+
+
 import markdown
-import nvd3
 import pendulum
-
 import sqlalchemy as sqla
-from sqlalchemy import or_, desc, and_, union_all
-
 from flask import (
-    g, redirect, request, Markup, Response, render_template,
-    make_response, flash, jsonify)
+    redirect, request, Markup, Response, render_template,
+    make_response, flash, jsonify, send_file, url_for)
 from flask._compat import PY2
-
 from flask_appbuilder import BaseView, ModelView, expose, has_access
-from flask_appbuilder.models.sqla.interface import SQLAInterface
 from flask_appbuilder.actions import action
-
+from flask_appbuilder.models.sqla.filters import BaseFilter
 from flask_babel import lazy_gettext
-
-from wtforms import Form, SelectField, validators
-
 from pygments import highlight, lexers
 from pygments.formatters import HtmlFormatter
+from sqlalchemy import func, or_, desc, and_, union_all
+from wtforms import SelectField, validators
 
 import airflow
 from airflow import configuration as conf
 from airflow import models, jobs
 from airflow import settings
-from airflow.api.common.experimental.mark_tasks import set_dag_run_state
-from airflow.models import XCom, DagRun
+from airflow.api.common.experimental.mark_tasks import (set_dag_run_state_to_success,
+                                                        set_dag_run_state_to_failed)
+from airflow.models import XCom, DagRun, errors, DagModel
+from airflow.models.connection import Connection
+from airflow.models.log import Log
+from airflow.models.taskfail import TaskFail
 from airflow.ti_deps.dep_context import DepContext, QUEUE_DEPS, SCHEDULER_DEPS
-
 from airflow.utils import timezone
-from airflow.utils.json import json_ser
-from airflow.utils.state import State
-from airflow.utils.db import provide_session
-from airflow.utils.helpers import alchemy_to_dict
 from airflow.utils.dates import infer_time_unit, scale_time_units
-
-
+from airflow.utils.db import provide_session
+from airflow.utils.helpers import alchemy_to_dict, render_log_filename
+from airflow.utils.state import State
+from airflow._vendor import nvd3
 from airflow.www_rbac import utils as wwwutils
-from airflow.www_rbac.app import app
-from airflow.www_rbac.decorators import action_logging, gzipped
+from airflow.www_rbac.app import app, appbuilder
+from airflow.www_rbac.decorators import action_logging, gzipped, has_dag_access
 from airflow.www_rbac.forms import (DateTimeForm, DateTimeWithNumRunsForm,
                                     DateTimeWithNumRunsWithDagRunsForm,
                                     DagRunForm, ConnectionForm)
-from airflow.www_rbac.security import is_view_only
 from airflow.www_rbac.widgets import AirflowModelListWidget
 
+
 PAGE_SIZE = conf.getint('webserver', 'page_size')
-dagbag = models.DagBag(settings.DAGS_FOLDER)
+if os.environ.get('SKIP_DAGS_PARSING') != 'True':
+    dagbag = models.DagBag(settings.DAGS_FOLDER)
+else:
+    dagbag = models.DagBag(os.devnull, include_examples=False)
 
 
 def get_date_time_num_runs_dag_runs_form_data(request, session, dag):
@@ -149,6 +148,42 @@ def render(self, template, **context):
 
 
 class Airflow(AirflowBaseView):
+    @expose('/health')
+    @provide_session
+    def health(self, session=None):
+        """
+        An endpoint helping check the health status of the Airflow instance,
+        including metadatabase and scheduler.
+        """
+
+        BJ = jobs.BaseJob
+        payload = {}
+        scheduler_health_check_threshold = timedelta(seconds=conf.getint('scheduler',
+                                                                         'scheduler_health_check_threshold'
+                                                                         ))
+
+        latest_scheduler_heartbeat = None
+        payload['metadatabase'] = {'status': 'healthy'}
+        try:
+            latest_scheduler_heartbeat = session.query(func.max(BJ.latest_heartbeat)).\
+                filter(BJ.state == 'running', BJ.job_type == 'SchedulerJob').\
+                scalar()
+        except Exception:
+            payload['metadatabase']['status'] = 'unhealthy'
+
+        if not latest_scheduler_heartbeat:
+            scheduler_status = 'unhealthy'
+        else:
+            if timezone.utcnow() - latest_scheduler_heartbeat <= scheduler_health_check_threshold:
+                scheduler_status = 'healthy'
+            else:
+                scheduler_status = 'unhealthy'
+
+        payload['scheduler'] = {'status': scheduler_status,
+                                'latest_scheduler_heartbeat': str(latest_scheduler_heartbeat)}
+
+        return wwwutils.json_response(payload)
+
     @expose('/home')
     @has_access
     @provide_session
@@ -159,6 +194,10 @@ def index(self, session=None):
                                                       'hide_paused_dags_by_default')
         show_paused_arg = request.args.get('showPaused', 'None')
 
+        default_dag_run = conf.getint('webserver', 'default_dag_run_display_number')
+        num_runs = request.args.get('num_runs')
+        num_runs = int(num_runs) if num_runs else default_dag_run
+
         def get_int_arg(value, default=0):
             try:
                 return int(value)
@@ -179,78 +218,55 @@ def get_int_arg(value, default=0):
             hide_paused = hide_paused_dags_by_default
 
         # read orm_dags from the db
-        sql_query = session.query(DM).filter(
+        query = session.query(DM).filter(
             ~DM.is_subdag, DM.is_active
         )
 
         # optionally filter out "paused" dags
         if hide_paused:
-            sql_query = sql_query.filter(~DM.is_paused)
+            query = query.filter(~DM.is_paused)
 
-        orm_dags = {dag.dag_id: dag for dag
-                    in sql_query
-                    .all()}
+        if arg_search_query:
+            query = query.filter(sqla.func.lower(DM.dag_id) == arg_search_query.lower())
 
-        import_errors = session.query(models.ImportError).all()
+        import_errors = session.query(errors.ImportError).all()
         for ie in import_errors:
             flash(
                 "Broken DAG: [{ie.filename}] {ie.stacktrace}".format(ie=ie),
                 "error")
 
-        # get a list of all non-subdag dags visible to everyone
-        # optionally filter out "paused" dags
-        if hide_paused:
-            unfiltered_webserver_dags = [dag for dag in dagbag.dags.values() if
-                                         not dag.parent_dag and not dag.is_paused]
-
-        else:
-            unfiltered_webserver_dags = [dag for dag in dagbag.dags.values() if
-                                         not dag.parent_dag]
-
-        webserver_dags = {
-            dag.dag_id: dag
-            for dag in unfiltered_webserver_dags
-        }
+        from airflow.plugins_manager import import_errors as plugin_import_errors
+        for filename, stacktrace in plugin_import_errors.items():
+            flash(
+                "Broken plugin: [{filename}] {stacktrace}".format(
+                    stacktrace=stacktrace,
+                    filename=filename),
+                "error")
 
-        if arg_search_query:
-            lower_search_query = arg_search_query.lower()
-            # filter by dag_id
-            webserver_dags_filtered = {
-                dag_id: dag
-                for dag_id, dag in webserver_dags.items()
-                if (lower_search_query in dag_id.lower() or
-                    lower_search_query in dag.owner.lower())
-            }
+        # Get all the dag id the user could access
+        filter_dag_ids = appbuilder.sm.get_accessible_dag_ids()
 
-            all_dag_ids = (set([dag.dag_id for dag in orm_dags.values()
-                                if lower_search_query in dag.dag_id.lower() or
-                                lower_search_query in dag.owners.lower()]) |
-                           set(webserver_dags_filtered.keys()))
+        if 'all_dags' not in filter_dag_ids:
+            query = query.filter(DM.dag_id.in_(filter_dag_ids))
 
-            sorted_dag_ids = sorted(all_dag_ids)
-        else:
-            webserver_dags_filtered = webserver_dags
-            sorted_dag_ids = sorted(set(orm_dags.keys()) | set(webserver_dags.keys()))
+        query = query.order_by(DM.dag_id)
 
         start = current_page * dags_per_page
         end = start + dags_per_page
 
-        num_of_all_dags = len(sorted_dag_ids)
-        page_dag_ids = sorted_dag_ids[start:end]
+        dags = query.offset(start).limit(dags_per_page).all()
+
+        num_of_all_dags = query.count()
         num_of_pages = int(math.ceil(num_of_all_dags / float(dags_per_page)))
 
         auto_complete_data = set()
-        for dag in webserver_dags_filtered.values():
-            auto_complete_data.add(dag.dag_id)
-            auto_complete_data.add(dag.owner)
-        for dag in orm_dags.values():
-            auto_complete_data.add(dag.dag_id)
-            auto_complete_data.add(dag.owners)
+        for row in query.with_entities(DM.dag_id, DM.owners):
+            auto_complete_data.add(row.dag_id)
+            auto_complete_data.add(row.owners)
 
         return self.render(
             'airflow/dags.html',
-            webserver_dags=webserver_dags_filtered,
-            orm_dags=orm_dags,
+            dags=dags,
             hide_paused=hide_paused,
             current_page=current_page,
             search_query=arg_search_query if arg_search_query else '',
@@ -262,43 +278,44 @@ def get_int_arg(value, default=0):
             paging=wwwutils.generate_pages(current_page, num_of_pages,
                                            search=arg_search_query,
                                            showPaused=not hide_paused),
-            dag_ids_in_page=page_dag_ids,
             auto_complete_data=auto_complete_data,
-            view_only=is_view_only(g.user, self.appbuilder))
+            num_runs=num_runs)
 
     @expose('/dag_stats')
     @has_access
     @provide_session
     def dag_stats(self, session=None):
-        ds = models.DagStat
-
-        ds.update()
+        dr = models.DagRun
 
-        qry = (
-            session.query(ds.dag_id, ds.state, ds.count)
-        )
+        filter_dag_ids = appbuilder.sm.get_accessible_dag_ids()
 
-        data = {}
-        for dag_id, state, count in qry:
-            if dag_id not in data:
-                data[dag_id] = {}
-            data[dag_id][state] = count
+        dag_state_stats = session.query(dr.dag_id, dr.state, sqla.func.count(dr.state))\
+            .group_by(dr.dag_id, dr.state)
 
         payload = {}
-        for dag in dagbag.dags.values():
-            payload[dag.safe_dag_id] = []
-            for state in State.dag_states:
-                try:
-                    count = data[dag.dag_id][state]
-                except Exception:
-                    count = 0
-                d = {
-                    'state': state,
-                    'count': count,
-                    'dag_id': dag.dag_id,
-                    'color': State.color(state)
-                }
-                payload[dag.safe_dag_id].append(d)
+        if filter_dag_ids:
+            if 'all_dags' not in filter_dag_ids:
+                dag_state_stats = dag_state_stats.filter(dr.dag_id.in_(filter_dag_ids))
+            data = {}
+            for dag_id, state, count in dag_state_stats:
+                if dag_id not in data:
+                    data[dag_id] = {}
+                data[dag_id][state] = count
+
+            if 'all_dags' in filter_dag_ids:
+                filter_dag_ids = [dag_id for dag_id, in session.query(models.DagModel.dag_id)]
+
+            for dag_id in filter_dag_ids:
+                if 'all_dags' in filter_dag_ids or dag_id in filter_dag_ids:
+                    payload[dag_id] = []
+                    for state in State.dag_states:
+                        count = data.get(dag_id, {}).get(state, 0)
+                        payload[dag_id].append({
+                            'state': state,
+                            'count': count,
+                            'dag_id': dag_id,
+                            'color': State.color(state)
+                        })
         return wwwutils.json_response(payload)
 
     @expose('/task_stats')
@@ -309,6 +326,12 @@ def task_stats(self, session=None):
         DagRun = models.DagRun
         Dag = models.DagModel
 
+        filter_dag_ids = appbuilder.sm.get_accessible_dag_ids()
+
+        payload = {}
+        if not filter_dag_ids:
+            return
+
         LastDagRun = (
             session.query(
                         DagRun.dag_id,
@@ -350,36 +373,37 @@ def task_stats(self, session=None):
 
         data = {}
         for dag_id, state, count in qry:
-            if dag_id not in data:
-                data[dag_id] = {}
-            data[dag_id][state] = count
+            if 'all_dags' in filter_dag_ids or dag_id in filter_dag_ids:
+                if dag_id not in data:
+                    data[dag_id] = {}
+                data[dag_id][state] = count
         session.commit()
 
-        payload = {}
-        for dag in dagbag.dags.values():
-            payload[dag.safe_dag_id] = []
-            for state in State.task_states:
-                try:
-                    count = data[dag.dag_id][state]
-                except Exception:
-                    count = 0
-                d = {
-                    'state': state,
-                    'count': count,
-                    'dag_id': dag.dag_id,
-                    'color': State.color(state)
-                }
-                payload[dag.safe_dag_id].append(d)
+        if 'all_dags' in filter_dag_ids:
+            filter_dag_ids = [dag_id for dag_id, in session.query(models.DagModel.dag_id)]
+        for dag_id in filter_dag_ids:
+            if 'all_dags' in filter_dag_ids or dag_id in filter_dag_ids:
+                payload[dag_id] = []
+                for state in State.task_states:
+                    count = data.get(dag_id, {}).get(state, 0)
+                    payload[dag_id].append({
+                        'state': state,
+                        'count': count,
+                        'dag_id': dag_id,
+                        'color': State.color(state)
+                    })
         return wwwutils.json_response(payload)
 
     @expose('/code')
+    @has_dag_access(can_dag_read=True)
     @has_access
-    def code(self):
+    @provide_session
+    def code(self, session=None):
+        dm = models.DagModel
         dag_id = request.args.get('dag_id')
-        dag = dagbag.get_dag(dag_id)
-        title = dag_id
+        dag = session.query(dm).filter(dm.dag_id == dag_id).first()
         try:
-            with open(dag.fileloc, 'r') as f:
+            with wwwutils.open_maybe_zipped(dag.fileloc, 'r') as f:
                 code = f.read()
             html_code = highlight(
                 code, lexers.PythonLexer(), HtmlFormatter(linenos=True))
@@ -387,17 +411,21 @@ def code(self):
             html_code = str(e)
 
         return self.render(
-            'airflow/dag_code.html', html_code=html_code, dag=dag, title=title,
+            'airflow/dag_code.html', html_code=html_code, dag=dag, title=dag_id,
             root=request.args.get('root'),
             demo_mode=conf.getboolean('webserver', 'demo_mode'))
 
     @expose('/dag_details')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @provide_session
     def dag_details(self, session=None):
         dag_id = request.args.get('dag_id')
-        dag = dagbag.get_dag(dag_id)
+        dag_orm = DagModel.get_dagmodel(dag_id)
+        # FIXME: items needed for this view should move to the database
+        dag = dag_orm.get_dag()
         title = "DAG details"
+        root = request.args.get('root', '')
 
         TI = models.TaskInstance
         states = (
@@ -406,9 +434,16 @@ def dag_details(self, session=None):
                    .group_by(TI.state)
                    .all()
         )
+
+        active_runs = models.DagRun.find(
+            dag_id=dag_id,
+            state=State.RUNNING,
+            external_trigger=False
+        )
+
         return self.render(
             'airflow/dag_details.html',
-            dag=dag, title=title, states=states, State=State)
+            dag=dag, title=title, root=root, states=states, State=State, active_runs=active_runs)
 
     @app.errorhandler(404)
     def circles(self):
@@ -428,14 +463,19 @@ def show_traceback(self):
     @has_access
     def pickle_info(self):
         d = {}
+        filter_dag_ids = appbuilder.sm.get_accessible_dag_ids()
+        if not filter_dag_ids:
+            return wwwutils.json_response({})
         dag_id = request.args.get('dag_id')
         dags = [dagbag.dags.get(dag_id)] if dag_id else dagbag.dags.values()
         for dag in dags:
-            if not dag.is_subdag:
-                d[dag.dag_id] = dag.pickle_info()
+            if 'all_dags' in filter_dag_ids or dag.dag_id in filter_dag_ids:
+                if not dag.is_subdag:
+                    d[dag.dag_id] = dag.pickle_info()
         return wwwutils.json_response(d)
 
     @expose('/rendered')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     def rendered(self):
@@ -444,6 +484,7 @@ def rendered(self):
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
+        root = request.args.get('root', '')
         dag = dagbag.get_dag(dag_id)
         task = copy.copy(dag.get_task(task_id))
         ti = models.TaskInstance(task=task, execution_date=dttm)
@@ -469,9 +510,11 @@ def rendered(self):
             task_id=task_id,
             execution_date=execution_date,
             form=form,
-            title=title, )
+            root=root,
+            title=title)
 
     @expose('/get_logs_with_metadata')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -480,9 +523,13 @@ def get_logs_with_metadata(self, session=None):
         task_id = request.args.get('task_id')
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
-        try_number = int(request.args.get('try_number'))
+        if request.args.get('try_number') is not None:
+            try_number = int(request.args.get('try_number'))
+        else:
+            try_number = None
         metadata = request.args.get('metadata')
         metadata = json.loads(metadata)
+        response_format = request.args.get('format', 'json')
 
         # metadata may be null
         if not metadata:
@@ -519,11 +566,21 @@ def get_logs_with_metadata(self, session=None):
                 ti.task = dag.get_task(ti.task_id)
                 logs, metadatas = handler.read(ti, try_number, metadata=metadata)
                 metadata = metadatas[0]
-            for i, log in enumerate(logs):
-                if PY2 and not isinstance(log, unicode):
-                    logs[i] = log.decode('utf-8')
-            message = logs[0]
-            return jsonify(message=message, metadata=metadata)
+
+            if response_format == 'json':
+                message = logs[0] if try_number is not None else logs
+                return jsonify(message=message, metadata=metadata)
+
+            file_obj = BytesIO(b'\n'.join(
+                log.encode('utf-8') for log in logs
+            ))
+            filename_template = conf.get('core', 'LOG_FILENAME_TEMPLATE')
+            attachment_filename = render_log_filename(
+                ti=ti,
+                try_number="all" if try_number is None else try_number,
+                filename_template=filename_template)
+            return send_file(file_obj, as_attachment=True,
+                             attachment_filename=attachment_filename)
         except AttributeError as e:
             error_message = ["Task log handler {} does not support read logs.\n{}\n"
                              .format(task_log_reader, str(e))]
@@ -531,6 +588,7 @@ def get_logs_with_metadata(self, session=None):
             return jsonify(message=error_message, error=True, metadata=metadata)
 
     @expose('/log')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -547,14 +605,23 @@ def log(self, session=None):
             models.TaskInstance.task_id == task_id,
             models.TaskInstance.execution_date == dttm).first()
 
-        logs = [''] * (ti.next_try_number - 1 if ti is not None else 0)
+        num_logs = 0
+        if ti is not None:
+            num_logs = ti.next_try_number - 1
+            if ti.state == State.UP_FOR_RESCHEDULE:
+                # Tasks in reschedule state decremented the try number
+                num_logs += 1
+        logs = [''] * num_logs
+        root = request.args.get('root', '')
         return self.render(
             'airflow/ti_log.html',
             logs=logs, dag=dag, title="Log by attempts",
             dag_id=dag.dag_id, task_id=task_id,
-            execution_date=execution_date, form=form)
+            execution_date=execution_date, form=form,
+            root=root)
 
     @expose('/task')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     def task(self):
@@ -567,6 +634,7 @@ def task(self):
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
+        root = request.args.get('root', '')
         dag = dagbag.get_dag(dag_id)
 
         if not dag or task_id not in dag.task_ids:
@@ -574,7 +642,7 @@ def task(self):
                 "Task [{}.{}] doesn't seem to exist"
                 " at the moment".format(dag_id, task_id),
                 "error")
-            return redirect('/')
+            return redirect(url_for('Airflow.index'))
         task = copy.copy(dag.get_task(task_id))
         task.resolve_template_files()
         ti = TI(task=task, execution_date=dttm)
@@ -629,9 +697,11 @@ def task(self):
             execution_date=execution_date,
             special_attrs_rendered=special_attrs_rendered,
             form=form,
+            root=root,
             dag=dag, title=title)
 
     @expose('/xcom')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -643,13 +713,18 @@ def xcom(self, session=None):
         execution_date = request.args.get('execution_date')
         dttm = pendulum.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
-        dag = dagbag.get_dag(dag_id)
-        if not dag or task_id not in dag.task_ids:
+        root = request.args.get('root', '')
+        dm_db = models.DagModel
+        ti_db = models.TaskInstance
+        dag = session.query(dm_db).filter(dm_db.dag_id == dag_id).first()
+        ti = session.query(ti_db).filter(ti_db.dag_id == dag_id and ti_db.task_id == task_id).first()
+
+        if not ti:
             flash(
                 "Task [{}.{}] doesn't seem to exist"
                 " at the moment".format(dag_id, task_id),
                 "error")
-            return redirect('/')
+            return redirect(url_for('Airflow.index'))
 
         xcomlist = session.query(XCom).filter(
             XCom.dag_id == dag_id, XCom.task_id == task_id,
@@ -667,34 +742,45 @@ def xcom(self, session=None):
             task_id=task_id,
             execution_date=execution_date,
             form=form,
+            root=root,
             dag=dag, title=title)
 
-    @expose('/run')
+    @expose('/run', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
     def run(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
         dag = dagbag.get_dag(dag_id)
         task = dag.get_task(task_id)
 
-        execution_date = request.args.get('execution_date')
+        execution_date = request.form.get('execution_date')
         execution_date = pendulum.parse(execution_date)
-        ignore_all_deps = request.args.get('ignore_all_deps') == "true"
-        ignore_task_deps = request.args.get('ignore_task_deps') == "true"
-        ignore_ti_state = request.args.get('ignore_ti_state') == "true"
+        ignore_all_deps = request.form.get('ignore_all_deps') == "true"
+        ignore_task_deps = request.form.get('ignore_task_deps') == "true"
+        ignore_ti_state = request.form.get('ignore_ti_state') == "true"
+
+        from airflow.executors import GetDefaultExecutor
+        executor = GetDefaultExecutor()
+        valid_celery_config = False
+        valid_kubernetes_config = False
 
         try:
-            from airflow.executors import GetDefaultExecutor
             from airflow.executors.celery_executor import CeleryExecutor
-            executor = GetDefaultExecutor()
-            if not isinstance(executor, CeleryExecutor):
-                flash("Only works with the CeleryExecutor, sorry", "error")
-                return redirect(origin)
+            valid_celery_config = isinstance(executor, CeleryExecutor)
         except ImportError:
-            # in case CeleryExecutor cannot be imported it is not active either
-            flash("Only works with the CeleryExecutor, sorry", "error")
+            pass
+
+        try:
+            from airflow.contrib.executors.kubernetes_executor import KubernetesExecutor
+            valid_kubernetes_config = isinstance(executor, KubernetesExecutor)
+        except ImportError:
+            pass
+
+        if not valid_celery_config and not valid_kubernetes_config:
+            flash("Only works with the Celery or Kubernetes executors, sorry", "error")
             return redirect(origin)
 
         ti = models.TaskInstance(task=task, execution_date=execution_date)
@@ -727,14 +813,43 @@ def run(self):
             "it should start any moment now.".format(ti))
         return redirect(origin)
 
-    @expose('/trigger')
+    @expose('/delete', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
-    def trigger(self):
-        dag_id = request.args.get('dag_id')
-        origin = request.args.get('origin') or "/"
-        dag = dagbag.get_dag(dag_id)
+    def delete(self):
+        from airflow.api.common.experimental import delete_dag
+        from airflow.exceptions import DagNotFound, DagFileExists
 
+        dag_id = request.values.get('dag_id')
+        origin = request.values.get('origin') or url_for('Airflow.index')
+
+        try:
+            delete_dag.delete_dag(dag_id)
+        except DagNotFound:
+            flash("DAG with id {} not found. Cannot delete".format(dag_id), 'error')
+            return redirect(request.referrer)
+        except DagFileExists:
+            flash("Dag id {} is still in DagBag. "
+                  "Remove the DAG file first.".format(dag_id),
+                  'error')
+            return redirect(request.referrer)
+
+        flash("Deleting DAG with id {}. May take a couple minutes to fully"
+              " disappear.".format(dag_id))
+
+        # Upon success return to origin.
+        return redirect(origin)
+
+    @expose('/trigger', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
+    @has_access
+    @action_logging
+    @provide_session
+    def trigger(self, session=None):
+        dag_id = request.values.get('dag_id')
+        origin = request.values.get('origin') or url_for('Airflow.index')
+        dag = session.query(models.DagModel).filter(models.DagModel.dag_id == dag_id).first()
         if not dag:
             flash("Cannot find dag {}".format(dag_id))
             return redirect(origin)
@@ -768,7 +883,9 @@ def _clear_dag_tis(self, dag, start_date, end_date, origin,
             count = dag.clear(
                 start_date=start_date,
                 end_date=end_date,
-                include_subdags=recursive)
+                include_subdags=recursive,
+                include_parentdag=recursive,
+            )
 
             flash("{0} task instances have been cleared".format(count))
             return redirect(origin)
@@ -777,7 +894,9 @@ def _clear_dag_tis(self, dag, start_date, end_date, origin,
             start_date=start_date,
             end_date=end_date,
             include_subdags=recursive,
-            dry_run=True)
+            include_parentdag=recursive,
+            dry_run=True,
+        )
         if not tis:
             flash("No task instances to clear", 'error')
             response = redirect(origin)
@@ -792,23 +911,24 @@ def _clear_dag_tis(self, dag, start_date, end_date, origin,
 
         return response
 
-    @expose('/clear')
+    @expose('/clear', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
     def clear(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
         dag = dagbag.get_dag(dag_id)
 
-        execution_date = request.args.get('execution_date')
+        execution_date = request.form.get('execution_date')
         execution_date = pendulum.parse(execution_date)
-        confirmed = request.args.get('confirmed') == "true"
-        upstream = request.args.get('upstream') == "true"
-        downstream = request.args.get('downstream') == "true"
-        future = request.args.get('future') == "true"
-        past = request.args.get('past') == "true"
-        recursive = request.args.get('recursive') == "true"
+        confirmed = request.form.get('confirmed') == "true"
+        upstream = request.form.get('upstream') == "true"
+        downstream = request.form.get('downstream') == "true"
+        future = request.form.get('future') == "true"
+        past = request.form.get('past') == "true"
+        recursive = request.form.get('recursive') == "true"
 
         dag = dag.sub_dag(
             task_regex=r"^{0}$".format(task_id),
@@ -821,14 +941,15 @@ def clear(self):
         return self._clear_dag_tis(dag, start_date, end_date, origin,
                                    recursive=recursive, confirmed=confirmed)
 
-    @expose('/dagrun_clear')
+    @expose('/dagrun_clear', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
     def dagrun_clear(self):
-        dag_id = request.args.get('dag_id')
-        origin = request.args.get('origin')
-        execution_date = request.args.get('execution_date')
-        confirmed = request.args.get('confirmed') == "true"
+        dag_id = request.form.get('dag_id')
+        origin = request.form.get('origin')
+        execution_date = request.form.get('execution_date')
+        confirmed = request.form.get('confirmed') == "true"
 
         dag = dagbag.get_dag(dag_id)
         execution_date = pendulum.parse(execution_date)
@@ -843,33 +964,60 @@ def dagrun_clear(self):
     @provide_session
     def blocked(self, session=None):
         DR = models.DagRun
-        dags = (
-            session.query(DR.dag_id, sqla.func.count(DR.id))
-                   .filter(DR.state == State.RUNNING)
-                   .group_by(DR.dag_id)
-                   .all()
-        )
+        filter_dag_ids = appbuilder.sm.get_accessible_dag_ids()
+
         payload = []
-        for dag_id, active_dag_runs in dags:
-            max_active_runs = 0
-            if dag_id in dagbag.dags:
-                max_active_runs = dagbag.dags[dag_id].max_active_runs
-            payload.append({
-                'dag_id': dag_id,
-                'active_dag_run': active_dag_runs,
-                'max_active_runs': max_active_runs,
-            })
+        if filter_dag_ids:
+            dags = (
+                session.query(DR.dag_id, sqla.func.count(DR.id))
+                       .filter(DR.state == State.RUNNING)
+                       .group_by(DR.dag_id)
+
+            )
+            if 'all_dags' not in filter_dag_ids:
+                dags = dags.filter(DR.dag_id.in_(filter_dag_ids))
+            dags = dags.all()
+
+            for dag_id, active_dag_runs in dags:
+                max_active_runs = 0
+                if dag_id in dagbag.dags:
+                    max_active_runs = dagbag.dags[dag_id].max_active_runs
+                payload.append({
+                    'dag_id': dag_id,
+                    'active_dag_run': active_dag_runs,
+                    'max_active_runs': max_active_runs,
+                })
         return wwwutils.json_response(payload)
 
-    @expose('/dagrun_success')
-    @has_access
-    @action_logging
-    def dagrun_success(self):
-        dag_id = request.args.get('dag_id')
-        execution_date = request.args.get('execution_date')
-        confirmed = request.args.get('confirmed') == 'true'
-        origin = request.args.get('origin')
+    def _mark_dagrun_state_as_failed(self, dag_id, execution_date, confirmed, origin):
+        if not execution_date:
+            flash('Invalid execution date', 'error')
+            return redirect(origin)
+
+        execution_date = pendulum.parse(execution_date)
+        dag = dagbag.get_dag(dag_id)
+
+        if not dag:
+            flash('Cannot find DAG: {}'.format(dag_id), 'error')
+            return redirect(origin)
+
+        new_dag_state = set_dag_run_state_to_failed(dag, execution_date, commit=confirmed)
+
+        if confirmed:
+            flash('Marked failed on {} task instances'.format(len(new_dag_state)))
+            return redirect(origin)
+
+        else:
+            details = '\n'.join([str(t) for t in new_dag_state])
+
+            response = self.render('airflow/confirm.html',
+                                   message=("Here's the list of task instances you are "
+                                            "about to mark as failed"),
+                                   details=details)
+
+            return response
 
+    def _mark_dagrun_state_as_success(self, dag_id, execution_date, confirmed, origin):
         if not execution_date:
             flash('Invalid execution date', 'error')
             return redirect(origin)
@@ -881,8 +1029,8 @@ def dagrun_success(self):
             flash('Cannot find DAG: {}'.format(dag_id), 'error')
             return redirect(origin)
 
-        new_dag_state = set_dag_run_state(dag, execution_date, state=State.SUCCESS,
-                                          commit=confirmed)
+        new_dag_state = set_dag_run_state_to_success(dag, execution_date,
+                                                     commit=confirmed)
 
         if confirmed:
             flash('Marked success on {} task instances'.format(len(new_dag_state)))
@@ -893,29 +1041,43 @@ def dagrun_success(self):
 
             response = self.render('airflow/confirm.html',
                                    message=("Here's the list of task instances you are "
-                                            "about to mark as successful:"),
+                                            "about to mark as success"),
                                    details=details)
 
             return response
 
-    @expose('/success')
+    @expose('/dagrun_failed', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
-    def success(self):
-        dag_id = request.args.get('dag_id')
-        task_id = request.args.get('task_id')
-        origin = request.args.get('origin')
+    def dagrun_failed(self):
+        dag_id = request.form.get('dag_id')
+        execution_date = request.form.get('execution_date')
+        confirmed = request.form.get('confirmed') == 'true'
+        origin = request.form.get('origin')
+        return self._mark_dagrun_state_as_failed(dag_id, execution_date,
+                                                 confirmed, origin)
+
+    @expose('/dagrun_success', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
+    @has_access
+    @action_logging
+    def dagrun_success(self):
+        dag_id = request.form.get('dag_id')
+        execution_date = request.form.get('execution_date')
+        confirmed = request.form.get('confirmed') == 'true'
+        origin = request.form.get('origin')
+        return self._mark_dagrun_state_as_success(dag_id, execution_date,
+                                                  confirmed, origin)
+
+    def _mark_task_instance_state(self, dag_id, task_id, origin, execution_date,
+                                  confirmed, upstream, downstream,
+                                  future, past, state):
         dag = dagbag.get_dag(dag_id)
         task = dag.get_task(task_id)
         task.dag = dag
 
-        execution_date = request.args.get('execution_date')
         execution_date = pendulum.parse(execution_date)
-        confirmed = request.args.get('confirmed') == "true"
-        upstream = request.args.get('upstream') == "true"
-        downstream = request.args.get('downstream') == "true"
-        future = request.args.get('future') == "true"
-        past = request.args.get('past') == "true"
 
         if not dag:
             flash("Cannot find DAG: {}".format(dag_id))
@@ -930,27 +1092,68 @@ def success(self):
         if confirmed:
             altered = set_state(task=task, execution_date=execution_date,
                                 upstream=upstream, downstream=downstream,
-                                future=future, past=past, state=State.SUCCESS,
+                                future=future, past=past, state=state,
                                 commit=True)
 
-            flash("Marked success on {} task instances".format(len(altered)))
+            flash("Marked {} on {} task instances".format(state, len(altered)))
             return redirect(origin)
 
         to_be_altered = set_state(task=task, execution_date=execution_date,
                                   upstream=upstream, downstream=downstream,
-                                  future=future, past=past, state=State.SUCCESS,
+                                  future=future, past=past, state=state,
                                   commit=False)
 
         details = "\n".join([str(t) for t in to_be_altered])
 
         response = self.render("airflow/confirm.html",
                                message=("Here's the list of task instances you are "
-                                        "about to mark as successful:"),
+                                        "about to mark as {}:".format(state)),
                                details=details)
 
         return response
 
+    @expose('/failed', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
+    @has_access
+    @action_logging
+    def failed(self):
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
+        execution_date = request.form.get('execution_date')
+
+        confirmed = request.form.get('confirmed') == "true"
+        upstream = request.form.get('upstream') == "true"
+        downstream = request.form.get('downstream') == "true"
+        future = request.form.get('future') == "true"
+        past = request.form.get('past') == "true"
+
+        return self._mark_task_instance_state(dag_id, task_id, origin, execution_date,
+                                              confirmed, upstream, downstream,
+                                              future, past, State.FAILED)
+
+    @expose('/success', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
+    @has_access
+    @action_logging
+    def success(self):
+        dag_id = request.form.get('dag_id')
+        task_id = request.form.get('task_id')
+        origin = request.form.get('origin')
+        execution_date = request.form.get('execution_date')
+
+        confirmed = request.form.get('confirmed') == "true"
+        upstream = request.form.get('upstream') == "true"
+        downstream = request.form.get('downstream') == "true"
+        future = request.form.get('future') == "true"
+        past = request.form.get('past') == "true"
+
+        return self._mark_task_instance_state(dag_id, task_id, origin, execution_date,
+                                              confirmed, upstream, downstream,
+                                              future, past, State.SUCCESS)
+
     @expose('/tree')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @gzipped
     @action_logging
@@ -960,6 +1163,10 @@ def tree(self, session=None):
         dag_id = request.args.get('dag_id')
         blur = conf.getboolean('webserver', 'demo_mode')
         dag = dagbag.get_dag(dag_id)
+        if dag_id not in dagbag.dags:
+            flash('DAG "{0}" seems to be missing.'.format(dag_id), "error")
+            return redirect(url_for('Airflow.index'))
+
         root = request.args.get('root')
         if root:
             dag = dag.sub_dag(
@@ -1060,7 +1267,6 @@ def set_duration(tid):
                 for d in dates],
         }
 
-        data = json.dumps(data, indent=4, default=json_ser)
         session.commit()
 
         form = DateTimeWithNumRunsForm(data={'base_date': max_date,
@@ -1076,6 +1282,7 @@ def set_duration(tid):
             dag=dag, data=data, blur=blur, num_runs=num_runs)
 
     @expose('/graph')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @gzipped
     @action_logging
@@ -1086,7 +1293,7 @@ def graph(self, session=None):
         dag = dagbag.get_dag(dag_id)
         if dag_id not in dagbag.dags:
             flash('DAG "{0}" seems to be missing.'.format(dag_id), "error")
-            return redirect('/')
+            return redirect(url_for('Airflow.index'))
 
         root = request.args.get('root')
         if root:
@@ -1106,6 +1313,8 @@ def graph(self, session=None):
                     'label': task.task_id,
                     'labelStyle': "fill:{0};".format(task.ui_fgcolor),
                     'style': "fill:{0};".format(task.ui_color),
+                    'rx': 5,
+                    'ry': 5,
                 }
             })
 
@@ -1168,12 +1377,13 @@ class GraphForm(DateTimeWithNumRunsWithDagRunsForm):
             ),
             blur=blur,
             root=root or '',
-            task_instances=json.dumps(task_instances, indent=2),
-            tasks=json.dumps(tasks, indent=2),
-            nodes=json.dumps(nodes, indent=2),
-            edges=json.dumps(edges, indent=2), )
+            task_instances=task_instances,
+            tasks=tasks,
+            nodes=nodes,
+            edges=edges)
 
     @expose('/duration')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -1185,6 +1395,10 @@ def duration(self, session=None):
         num_runs = request.args.get('num_runs')
         num_runs = int(num_runs) if num_runs else default_dag_run
 
+        if dag is None:
+            flash('DAG "{0}" seems to be missing.'.format(dag_id), "error")
+            return redirect(url_for('Airflow.index'))
+
         if base_date:
             base_date = pendulum.parse(base_date)
         else:
@@ -1212,7 +1426,7 @@ def duration(self, session=None):
 
         tis = dag.get_task_instances(
             session, start_date=min_date, end_date=base_date)
-        TF = models.TaskFail
+        TF = TaskFail
         ti_fails = (
             session.query(TF)
                    .filter(TF.dag_id == dag.dag_id,  # noqa
@@ -1281,6 +1495,7 @@ def duration(self, session=None):
         )
 
     @expose('/tries')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -1345,6 +1560,7 @@ def tries(self, session=None):
         )
 
     @expose('/landing_times')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -1423,6 +1639,7 @@ def landing_times(self, session=None):
         )
 
     @expose('/paused', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
     @provide_session
@@ -1443,13 +1660,14 @@ def paused(self, session=None):
         dagbag.get_dag(dag_id)
         return "OK"
 
-    @expose('/refresh')
+    @expose('/refresh', methods=['POST'])
+    @has_dag_access(can_dag_edit=True)
     @has_access
     @action_logging
     @provide_session
     def refresh(self, session=None):
         DagModel = models.DagModel
-        dag_id = request.args.get('dag_id')
+        dag_id = request.values.get('dag_id')
         orm_dag = session.query(
             DagModel).filter(DagModel.dag_id == dag_id).first()
 
@@ -1458,19 +1676,25 @@ def refresh(self, session=None):
             session.merge(orm_dag)
         session.commit()
 
+        # sync dag permission
+        appbuilder.sm.sync_perm_for_dag(dag_id)
+
         dagbag.get_dag(dag_id)
         flash("DAG [{}] is now fresh as a daisy".format(dag_id))
         return redirect(request.referrer)
 
-    @expose('/refresh_all')
+    @expose('/refresh_all', methods=['POST'])
     @has_access
     @action_logging
     def refresh_all(self):
         dagbag.collect_dags(only_if_updated=False)
+        # sync permissions for all dags
+        appbuilder.sm.sync_perm_for_dag()
         flash("All DAGs are now up to date")
-        return redirect('/')
+        return redirect(url_for('Airflow.index'))
 
     @expose('/gantt')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -1496,19 +1720,41 @@ def gantt(self, session=None):
             ti for ti in dag.get_task_instances(session, dttm, dttm)
             if ti.start_date]
         tis = sorted(tis, key=lambda ti: ti.start_date)
+        TF = TaskFail
+        ti_fails = list(itertools.chain(*[(
+            session
+            .query(TF)
+            .filter(TF.dag_id == ti.dag_id,
+                    TF.task_id == ti.task_id,
+                    TF.execution_date == ti.execution_date)
+            .all()
+        ) for ti in tis]))
 
-        tasks = []
+        # determine bars to show in the gantt chart
+        # all reschedules of one attempt are combinded into one bar
+        gantt_bar_items = []
         for ti in tis:
-            end_date = ti.end_date if ti.end_date else timezone.utcnow()
+            end_date = ti.end_date or timezone.utcnow()
+            gantt_bar_items.append((ti.task_id, ti.start_date, end_date, ti.state))
+        for tf in ti_fails:
+            end_date = tf.end_date or timezone.utcnow()
+            gantt_bar_items.append((tf.task_id, tf.start_date, end_date, State.FAILED))
+
+        tasks = []
+        for gantt_bar_item in gantt_bar_items:
+            task_id = gantt_bar_item[0]
+            start_date = gantt_bar_item[1]
+            end_date = gantt_bar_item[2]
+            state = gantt_bar_item[3]
             tasks.append({
-                'startDate': wwwutils.epoch(ti.start_date),
+                'startDate': wwwutils.epoch(start_date),
                 'endDate': wwwutils.epoch(end_date),
-                'isoStart': ti.start_date.isoformat()[:-4],
+                'isoStart': start_date.isoformat()[:-4],
                 'isoEnd': end_date.isoformat()[:-4],
-                'taskName': ti.task_id,
-                'duration': "{}".format(end_date - ti.start_date)[:-4],
-                'status': ti.state,
-                'executionDate': ti.execution_date.isoformat(),
+                'taskName': task_id,
+                'duration': "{}".format(end_date - start_date)[:-4],
+                'status': state,
+                'executionDate': dttm.isoformat(),
             })
         states = {ti.state: ti.state for ti in tis}
         data = {
@@ -1525,13 +1771,14 @@ def gantt(self, session=None):
             dag=dag,
             execution_date=dttm.isoformat(),
             form=form,
-            data=json.dumps(data, indent=2),
+            data=data,
             base_date='',
             demo_mode=demo_mode,
             root=root,
         )
 
     @expose('/object/task_instances')
+    @has_dag_access(can_dag_read=True)
     @has_access
     @action_logging
     @provide_session
@@ -1543,7 +1790,7 @@ def task_instances(self, session=None):
         if dttm:
             dttm = pendulum.parse(dttm)
         else:
-            return ("Error: Invalid execution_date")
+            return "Error: Invalid execution_date"
 
         task_instances = {
             ti.task_id: alchemy_to_dict(ti)
@@ -1586,11 +1833,18 @@ def conf(self):
         raw = request.args.get('raw') == "true"
         title = "Airflow Configuration"
         subtitle = conf.AIRFLOW_CONFIG
-        with open(conf.AIRFLOW_CONFIG, 'r') as f:
-            config = f.read()
-        table = [(section, key, value, source)
-                 for section, parameters in conf.as_dict(True, True).items()
-                 for key, (value, source) in parameters.items()]
+        # Don't show config when expose_config variable is False in airflow config
+        if conf.getboolean("webserver", "expose_config"):
+            with open(conf.AIRFLOW_CONFIG, 'r') as f:
+                config = f.read()
+            table = [(section, key, value, source)
+                     for section, parameters in conf.as_dict(True, True).items()
+                     for key, (value, source) in parameters.items()]
+        else:
+            config = (
+                "# Your Airflow administrator chose not to expose the "
+                "configuration, most likely for security reasons.")
+            table = None
 
         if raw:
             return Response(
@@ -1614,31 +1868,19 @@ def conf(self):
 #                                    ModelViews
 ######################################################################################
 
+class DagFilter(BaseFilter):
+    def apply(self, query, func): # noqa
+        if appbuilder.sm.has_all_dags_access():
+            return query
+        filter_dag_ids = appbuilder.sm.get_accessible_dag_ids()
+        return query.filter(self.model.dag_id.in_(filter_dag_ids))
+
+
 class AirflowModelView(ModelView):
     list_widget = AirflowModelListWidget
     page_size = PAGE_SIZE
 
-    class CustomSQLAInterface(SQLAInterface):
-        """
-        FAB does not know how to handle columns with leading underscores because
-        they are not supported by WTForm. This hack will remove the leading
-        '_' from the key to lookup the column names.
-
-        """
-        def __init__(self, obj):
-            super(AirflowModelView.CustomSQLAInterface, self).__init__(obj)
-
-            self.session = settings.Session()
-
-            def clean_column_names():
-                if self.list_properties:
-                    self.list_properties = dict(
-                        (k.lstrip('_'), v) for k, v in self.list_properties.items())
-                if self.list_columns:
-                    self.list_columns = dict(
-                        (k.lstrip('_'), v) for k, v in self.list_columns.items())
-
-            clean_column_names()
+    CustomSQLAInterface = wwwutils.CustomSQLAInterface
 
 
 class SlaMissModelView(AirflowModelView):
@@ -1653,6 +1895,8 @@ class SlaMissModelView(AirflowModelView):
     edit_columns = ['dag_id', 'task_id', 'execution_date', 'email_sent', 'timestamp']
     search_columns = ['dag_id', 'task_id', 'email_sent', 'timestamp', 'execution_date']
     base_order = ('execution_date', 'desc')
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     formatters_columns = {
         'task_id': wwwutils.task_instance_link,
         'execution_date': wwwutils.datetime_f('execution_date'),
@@ -1674,6 +1918,8 @@ class XComModelView(AirflowModelView):
     edit_columns = ['key', 'value', 'execution_date', 'task_id', 'dag_id']
     base_order = ('execution_date', 'desc')
 
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     @action('muldelete', 'Delete', "Are you sure you want to delete selected records?",
             single=False)
     def action_muldelete(self, items):
@@ -1685,7 +1931,7 @@ def action_muldelete(self, items):
 class ConnectionModelView(AirflowModelView):
     route_base = '/connection'
 
-    datamodel = AirflowModelView.CustomSQLAInterface(models.Connection)
+    datamodel = AirflowModelView.CustomSQLAInterface(Connection)
 
     base_permissions = ['can_add', 'can_list', 'can_edit', 'can_delete']
 
@@ -1706,6 +1952,7 @@ class ConnectionModelView(AirflowModelView):
 
     @action('muldelete', 'Delete', 'Are you sure you want to delete selected records?',
             single=False)
+    @has_dag_access(can_dag_edit=True)
     def action_muldelete(self, items):
         self.datamodel.delete_all(items)
         self.update_redirect()
@@ -1725,6 +1972,11 @@ def prefill_form(self, form, pk):
         except Exception:
             d = {}
 
+        if not hasattr(d, 'get'):
+            logging.warning('extra field for {} is not iterable'.format(
+                form.data.get('conn_id', '<unknown>')))
+            return
+
         for field in self.extra_fields:
             value = d.get(field, '')
             if value:
@@ -1755,8 +2007,8 @@ def action_muldelete(self, items):
     def pool_link(attr):
         pool_id = attr.get('pool')
         if pool_id is not None:
-            url = '/taskinstance/list/?_flt_3_pool=' + str(pool_id)
-            return Markup("<a href='{url}'>{pool_id}</a>".format(**locals()))
+            url = url_for('TaskInstanceModelView.list', _flt_3_pool=pool_id)
+            return Markup("<a href='{url}'>{pool_id}</a>").format(**locals())
         else:
             return Markup('<span class="label label-danger">Invalid</span>')
 
@@ -1764,9 +2016,8 @@ def fused_slots(attr):
         pool_id = attr.get('pool')
         used_slots = attr.get('used_slots')
         if pool_id is not None and used_slots is not None:
-            url = '/taskinstance/list/?_flt_3_pool=' + str(pool_id) + \
-                  '&_flt_3_state=running'
-            return Markup("<a href='{url}'>{used_slots}</a>".format(**locals()))
+            url = url_for('TaskInstanceModelView.list', _flt_3_pool=pool_id, _flt_3_state='running')
+            return Markup("<a href='{url}'>{used_slots}</a>").format(**locals())
         else:
             return Markup('<span class="label label-danger">Invalid</span>')
 
@@ -1774,9 +2025,8 @@ def fqueued_slots(attr):
         pool_id = attr.get('pool')
         queued_slots = attr.get('queued_slots')
         if pool_id is not None and queued_slots is not None:
-            url = '/taskinstance/list/?_flt_3_pool=' + str(pool_id) + \
-                  '&_flt_3_state=queued'
-            return Markup("<a href='{url}'>{queued_slots}</a>".format(**locals()))
+            url = url_for('TaskInstanceModelView.list', _flt_3_pool=pool_id, _flt_3_state='queued')
+            return Markup("<a href='{url}'>{queued_slots}</a>").format(**locals())
         else:
             return Markup('<span class="label label-danger">Invalid</span>')
 
@@ -1802,7 +2052,7 @@ class VariableModelView(AirflowModelView):
     base_permissions = ['can_add', 'can_list', 'can_edit', 'can_delete', 'can_varimport']
 
     list_columns = ['key', 'val', 'is_encrypted']
-    add_columns = ['key', 'val']
+    add_columns = ['key', 'val', 'is_encrypted']
     edit_columns = ['key', 'val']
     search_columns = ['key', 'val']
 
@@ -1842,7 +2092,6 @@ def action_varexport(self, items):
         var_dict = {}
         d = json.JSONDecoder()
         for var in items:
-            val = None
             try:
                 val = d.decode(var.val)
             except Exception:
@@ -1864,11 +2113,22 @@ def varimport(self):
             else:
                 d = json.loads(out)
         except Exception:
-            flash("Missing file or syntax error.")
+            self.update_redirect()
+            flash("Missing file or syntax error.", 'error')
+            return redirect(self.get_redirect())
         else:
+            suc_count = fail_count = 0
             for k, v in d.items():
-                models.Variable.set(k, v, serialize_json=isinstance(v, dict))
-            flash("{} variable(s) successfully updated.".format(len(d)))
+                try:
+                    models.Variable.set(k, v, serialize_json=isinstance(v, dict))
+                except Exception as e:
+                    logging.info('Variable import failed: {}'.format(repr(e)))
+                    fail_count += 1
+                else:
+                    suc_count += 1
+            flash("{} variable(s) successfully updated.".format(suc_count))
+            if fail_count:
+                flash("{} variable(s) failed to be updated.".format(fail_count), 'error')
             self.update_redirect()
             return redirect(self.get_redirect())
 
@@ -1889,6 +2149,8 @@ class JobModelView(AirflowModelView):
 
     base_order = ('start_date', 'desc')
 
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     formatters_columns = {
         'start_date': wwwutils.datetime_f('start_date'),
         'end_date': wwwutils.datetime_f('end_date'),
@@ -1910,6 +2172,8 @@ class DagRunModelView(AirflowModelView):
 
     base_order = ('execution_date', 'desc')
 
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     add_form = edit_form = DagRunForm
 
     formatters_columns = {
@@ -1926,56 +2190,95 @@ class DagRunModelView(AirflowModelView):
 
     @action('muldelete', "Delete", "Are you sure you want to delete selected records?",
             single=False)
+    @has_dag_access(can_dag_edit=True)
     @provide_session
     def action_muldelete(self, items, session=None):
         self.datamodel.delete_all(items)
         self.update_redirect()
-        return redirect(self.get_redirect())
         dirty_ids = []
         for item in items:
             dirty_ids.append(item.dag_id)
-        models.DagStat.update(dirty_ids, dirty_only=False, session=session)
+        return redirect(self.get_redirect())
 
     @action('set_running', "Set state to 'running'", '', single=False)
-    def action_set_running(self, drs):
-        return self.set_dagrun_state(drs, State.RUNNING)
-
-    @action('set_failed', "Set state to 'failed'", '', single=False)
-    def action_set_failed(self, drs):
-        return self.set_dagrun_state(drs, State.FAILED)
+    @provide_session
+    def action_set_running(self, drs, session=None):
+        try:
+            DR = models.DagRun
+            count = 0
+            dirty_ids = []
+            for dr in session.query(DR).filter(
+                    DR.id.in_([dagrun.id for dagrun in drs])).all():
+                dirty_ids.append(dr.dag_id)
+                count += 1
+                dr.start_date = timezone.utcnow()
+                dr.state = State.RUNNING
+            session.commit()
+            flash("{count} dag runs were set to running".format(**locals()))
+        except Exception as ex:
+            flash(str(ex), 'error')
+            flash('Failed to set state', 'error')
+        return redirect(self.get_default_url())
 
-    @action('set_success', "Set state to 'success'", '', single=False)
-    def action_set_success(self, drs):
-        return self.set_dagrun_state(drs, State.SUCCESS)
+    @action('set_failed', "Set state to 'failed'",
+            "All running task instances would also be marked as failed, are you sure?",
+            single=False)
+    @provide_session
+    def action_set_failed(self, drs, session=None):
+        try:
+            DR = models.DagRun
+            count = 0
+            dirty_ids = []
+            altered_tis = []
+            for dr in session.query(DR).filter(
+                    DR.id.in_([dagrun.id for dagrun in drs])).all():
+                dirty_ids.append(dr.dag_id)
+                count += 1
+                altered_tis += \
+                    set_dag_run_state_to_failed(dagbag.get_dag(dr.dag_id),
+                                                dr.execution_date,
+                                                commit=True,
+                                                session=session)
+            altered_ti_count = len(altered_tis)
+            flash(
+                "{count} dag runs and {altered_ti_count} task instances "
+                "were set to failed".format(**locals()))
+        except Exception:
+            flash('Failed to set state', 'error')
+        return redirect(self.get_default_url())
 
+    @action('set_success', "Set state to 'success'",
+            "All task instances would also be marked as success, are you sure?",
+            single=False)
     @provide_session
-    def set_dagrun_state(self, drs, target_state, session=None):
+    def action_set_success(self, drs, session=None):
         try:
             DR = models.DagRun
             count = 0
             dirty_ids = []
+            altered_tis = []
             for dr in session.query(DR).filter(
                     DR.id.in_([dagrun.id for dagrun in drs])).all():
                 dirty_ids.append(dr.dag_id)
                 count += 1
-                dr.state = target_state
-                if target_state == State.RUNNING:
-                    dr.start_date = timezone.utcnow()
-                else:
-                    dr.end_date = timezone.utcnow()
-            session.commit()
-            models.DagStat.update(dirty_ids, session=session)
+                altered_tis += \
+                    set_dag_run_state_to_success(dagbag.get_dag(dr.dag_id),
+                                                 dr.execution_date,
+                                                 commit=True,
+                                                 session=session)
+            altered_ti_count = len(altered_tis)
             flash(
-                "{count} dag runs were set to '{target_state}'".format(**locals()))
-        except Exception as ex:
+                "{count} dag runs and {altered_ti_count} task instances "
+                "were set to success".format(**locals()))
+        except Exception:
             flash('Failed to set state', 'error')
-        return redirect(self.route_base + '/list')
+        return redirect(self.get_default_url())
 
 
 class LogModelView(AirflowModelView):
     route_base = '/log'
 
-    datamodel = AirflowModelView.CustomSQLAInterface(models.Log)
+    datamodel = AirflowModelView.CustomSQLAInterface(Log)
 
     base_permissions = ['can_list']
 
@@ -1985,6 +2288,8 @@ class LogModelView(AirflowModelView):
 
     base_order = ('dttm', 'desc')
 
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     formatters_columns = {
         'dttm': wwwutils.datetime_f('dttm'),
         'execution_date': wwwutils.datetime_f('execution_date'),
@@ -2011,6 +2316,8 @@ class TaskInstanceModelView(AirflowModelView):
 
     base_order = ('job_id', 'asc')
 
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     def log_url_formatter(attr):
         log_url = attr.get('log_url')
         return Markup(
@@ -2059,7 +2366,7 @@ def action_clear(self, tis, session=None):
             self.update_redirect()
             return redirect(self.get_redirect())
 
-        except Exception as ex:
+        except Exception:
             flash('Failed to clear task instances', 'error')
 
     @provide_session
@@ -2071,28 +2378,32 @@ def set_task_instance_state(self, tis, target_state, session=None):
             session.commit()
             flash(
                 "{count} task instances were set to '{target_state}'".format(**locals()))
-        except Exception as ex:
+        except Exception:
             flash('Failed to set state', 'error')
 
     @action('set_running', "Set state to 'running'", '', single=False)
+    @has_dag_access(can_dag_edit=True)
     def action_set_running(self, tis):
         self.set_task_instance_state(tis, State.RUNNING)
         self.update_redirect()
         return redirect(self.get_redirect())
 
     @action('set_failed', "Set state to 'failed'", '', single=False)
+    @has_dag_access(can_dag_edit=True)
     def action_set_failed(self, tis):
         self.set_task_instance_state(tis, State.FAILED)
         self.update_redirect()
         return redirect(self.get_redirect())
 
     @action('set_success', "Set state to 'success'", '', single=False)
+    @has_dag_access(can_dag_edit=True)
     def action_set_success(self, tis):
         self.set_task_instance_state(tis, State.SUCCESS)
         self.update_redirect()
         return redirect(self.get_redirect())
 
     @action('set_retry', "Set state to 'up_for_retry'", '', single=False)
+    @has_dag_access(can_dag_edit=True)
     def action_set_retry(self, tis):
         self.set_task_instance_state(tis, State.UP_FOR_RETRY)
         self.update_redirect()
@@ -2116,7 +2427,7 @@ class DagModelView(AirflowModelView):
 
     datamodel = AirflowModelView.CustomSQLAInterface(models.DagModel)
 
-    base_permissions = ['can_list', 'can_show']
+    base_permissions = ['can_list', 'can_show', 'can_edit']
 
     list_columns = ['dag_id', 'is_paused', 'last_scheduler_run',
                     'last_expired', 'scheduler_lock', 'fileloc', 'owners']
@@ -2125,6 +2436,8 @@ class DagModelView(AirflowModelView):
         'dag_id': wwwutils.dag_link
     }
 
+    base_filters = [['dag_id', DagFilter, lambda: []]]
+
     def get_query(self):
         """
         Default filters for model
diff --git a/airflow/www_rbac/webpack.config.js b/airflow/www_rbac/webpack.config.js
new file mode 100644
index 0000000000000..16f394cc616b9
--- /dev/null
+++ b/airflow/www_rbac/webpack.config.js
@@ -0,0 +1,134 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+const webpack = require('webpack');
+const path = require('path');
+const ManifestPlugin = require('webpack-manifest-plugin');
+const CleanWebpackPlugin = require('clean-webpack-plugin');
+const CopyWebpackPlugin = require('copy-webpack-plugin');
+const MiniCssExtractPlugin = require('mini-css-extract-plugin');
+
+// Input Directory (airflow/www_rbac)
+// noinspection JSUnresolvedVariable
+const STATIC_DIR = path.resolve(__dirname, './static');
+
+// Output Directory (airflow/www_rbac/static/dist)
+// noinspection JSUnresolvedVariable
+const BUILD_DIR = path.resolve(__dirname, './static/dist');
+
+const config = {
+  entry: {
+    connectionForm: `${STATIC_DIR}/js/connection_form.js`,
+    base: `${STATIC_DIR}/js/base.js`,
+    graph: `${STATIC_DIR}/js/graph.js`,
+    ganttChartD3v2: `${STATIC_DIR}/js/gantt-chart-d3v2.js`,
+    main: `${STATIC_DIR}/css/main.css`,
+    airflowDefaultTheme : `${STATIC_DIR}/css/bootstrap-theme.css`,
+  },
+  output: {
+    path: BUILD_DIR,
+    filename: '[name].[chunkhash].js',
+    chunkFilename: '[name].[chunkhash].js',
+  },
+  resolve: {
+    extensions: [
+      '.js',
+      '.jsx',
+      '.css',
+    ],
+  },
+  module: {
+    rules: [
+      {
+        test: /datatables\.net.*/,
+        loader: 'imports-loader?define=>false',
+      },
+      {
+        test: /\.jsx?$/,
+        exclude: /node_modules/,
+        loader: 'babel-loader',
+      },
+      // Extract css files
+      {
+        test: /\.css$/,
+        include: STATIC_DIR,
+        use: [
+          MiniCssExtractPlugin.loader,
+          'css-loader',
+        ],
+      },
+      /* for css linking images */
+      {
+        test: /\.png$/,
+        loader: 'url-loader?limit=100000',
+      },
+      {
+        test: /\.jpg$/,
+        loader: 'file-loader',
+      },
+      {
+        test: /\.gif$/,
+        loader: 'file-loader',
+      },
+      /* for font-awesome */
+      {
+        test: /\.woff(2)?(\?v=[0-9]\.[0-9]\.[0-9])?$/,
+        loader: 'url-loader?limit=10000&mimetype=application/font-woff',
+      },
+      {
+        test: /\.(ttf|eot|svg)(\?v=[0-9]\.[0-9]\.[0-9])?$/,
+        loader: 'file-loader',
+      },
+    ],
+  },
+  plugins: [
+    new ManifestPlugin(),
+    new CleanWebpackPlugin(['static/dist']),
+    new MiniCssExtractPlugin({ filename: '[name].[chunkhash].css' }),
+
+    // MomentJS loads all the locale, making it a huge JS file.
+    // This will ignore the locales from momentJS
+    new webpack.IgnorePlugin(/^\.\/locale$/, /moment$/),
+
+    new webpack.DefinePlugin({
+      'process.env': {
+        NODE_ENV: JSON.stringify(process.env.NODE_ENV),
+      },
+    }),
+    // Since we have all the dependencies separated from hard-coded JS within HTML,
+    // this seems like an efficient solution for now. Will update that once
+    // we'll have the dependencies imported within the custom JS
+    new CopyWebpackPlugin([
+      { from: 'node_modules/nvd3/build/*.min.*', flatten: true },
+      // Update this when upgrade d3 package, as the path in new D3 is different
+      { from: 'node_modules/d3/d3.min.*', flatten: true },
+      { from: 'node_modules/dagre-d3/dist/*.min.*', flatten: true },
+      { from: 'node_modules/d3-tip/dist/index.js', to: 'd3-tip.js', flatten: true },
+      { from: 'node_modules/bootstrap-3-typeahead/*min.*', flatten: true },
+      {
+        from: 'node_modules/bootstrap-toggle/**/*bootstrap-toggle.min.*',
+        flatten: true,
+      },
+      { from: 'node_modules/datatables.net/**/**.min.*', flatten: true },
+      { from: 'node_modules/datatables.net-bs/**/**.min.*', flatten: true },
+    ], { copyUnmodified: true }),
+  ],
+};
+
+module.exports = config;
diff --git a/airflow/www_rbac/widgets.py b/airflow/www_rbac/widgets.py
index 3bca3869ce7ee..ea61d3ff94004 100644
--- a/airflow/www_rbac/widgets.py
+++ b/airflow/www_rbac/widgets.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/dags/test_dag.py b/dags/test_dag.py
index 3c385bbbefe43..a133dd5a1263b 100644
--- a/dags/test_dag.py
+++ b/dags/test_dag.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,7 +23,9 @@
 from datetime import datetime, timedelta
 
 now = datetime.now()
-now_to_the_hour = (now - timedelta(0, 0, 0, 0, 0, 3)).replace(minute=0, second=0, microsecond=0)
+now_to_the_hour = (
+    now - timedelta(0, 0, 0, 0, 0, 3)
+).replace(minute=0, second=0, microsecond=0)
 START_DATE = now_to_the_hour
 DAG_NAME = 'test_dag_v1'
 
diff --git a/dev/README.md b/dev/README.md
index 084020cfabb66..a7747526cf862 100755
--- a/dev/README.md
+++ b/dev/README.md
@@ -1,10 +1,29 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Development Tools
 
 ## Airflow Pull Request Tool
 
 The `airflow-pr` tool interactively guides committers through the process of merging GitHub PRs into Airflow and closing associated JIRA issues.
 
-It is very important that PRs reference a JIRA issue. The preferred way to do that is for the PR title to begin with [AIRFLOW-XX]. However, the PR tool can recognize and parse many other JIRA issue formats in the title and will offer to correct them if possible.
+It is very important that PRs reference a JIRA issue. The preferred way to do that is for the PR title to begin with [AIRFLOW-XXX]. However, the PR tool can recognize and parse many other JIRA issue formats in the title and will offer to correct them if possible.
 
 __Please note:__ this tool will restore your current branch when it finishes, but you will lose any uncommitted changes. Make sure you commit any changes you wish to keep before proceeding.
 
@@ -57,16 +76,14 @@ pip install click jira
 #### git Remotes
 tl;dr run `airflow-pr setup_git_remotes` before using the tool for the first time.
 
-Before using the merge tool, users need to make sure their git remotes are configured. By default, the tool assumes a setup like the one below, where the github repo remote is named `github` and the Apache repo remote is named `apache`. If users have other remote names, they can be supplied by setting environment variables `GITHUB_REMOTE_NAME` and `APACHE_REMOTE_NAME`, respectively.
+Before using the merge tool, users need to make sure their git remotes are configured. By default, the tool assumes a setup like the one below, where the github repo remote is named `github`. If users have other remote names, they can be supplied by setting environment variables `GITHUB_REMOTE_NAME`.
 
 Users can configure this automatically by running `airflow-pr setup_git_remotes`.
 
 ```bash
 $ git remote -v
-apache	https://git-wip-us.apache.org/repos/asf/incubator-airflow.git (fetch)
-apache	https://git-wip-us.apache.org/repos/asf/incubator-airflow.git (push)
-github	https://github.com/apache/incubator-airflow.git (fetch)
-github	https://github.com/apache/incubator-airflow.git (push)
+github	https://github.com/apache/airflow.git (fetch)
+github	https://github.com/apache/airflow.git (push)
 origin	https://github.com/<USER>/airflow (fetch)
 origin	https://github.com/<USER>/airflow (push)
 ```
@@ -81,5 +98,16 @@ Unauthenticated users can only make 60 requests/hour to the Github API. If you g
 The release signing tool can be used to create the SHA512/MD5 and ASC files that required for Apache releases.
 
 ### Execution
-To create a release tar ball execute `python setup.py sdist --formats=gztar` from Airflow's root. After that `cd dist` and
-execute `../dev/sign.sh <the_created_tar_ball.tar.gz`. Signing files will be created in the same directory.
+To create a release tar ball execute following command from Airflow's root. 
+
+`python setup.py compile_assets sdist --formats=gztar`
+
+*Note: `compile_assets` command build the frontend assets (JS and CSS) files for the 
+Web UI using webpack and npm. Please make sure you have `npm` installed on your local machine globally.
+Details on how to install `npm` can be found in CONTRIBUTING.md file.*
+
+After that navigate to relative directory i.e., `cd dist` and sign the release files.
+
+`../dev/sign.sh <the_created_tar_ball.tar.gz` 
+
+Signing files will be created in the same directory.
diff --git a/dev/airflow-jira b/dev/airflow-jira
index 4c9b947a0d49c..4a9833e7cb07b 100755
--- a/dev/airflow-jira
+++ b/dev/airflow-jira
@@ -39,10 +39,6 @@ TMP_CREDENTIALS = {}
 PROJECT = "AIRFLOW"
 
 # Python 3 compatibility
-try:
-    import urllib2 as urllib
-except ImportError:
-    import urllib.request as urllib
 if sys.version_info[0] == 3:
     raw_input = input
 
@@ -52,12 +48,6 @@ except ImportError:
     print("Could not find the click library. Run 'sudo pip install click' to install.")
     sys.exit(-1)
 
-try:
-    import keyring
-except ImportError:
-    print("Could not find the keyring library. Run 'sudo pip install keyring' to install.")
-    sys.exit(-1)
-
 try:
     import git
 except ImportError:
@@ -68,36 +58,61 @@ JIRA_BASE = "https://issues.apache.org/jira/browse"
 JIRA_API_BASE = "https://issues.apache.org/jira"
 
 GIT_COMMIT_FIELDS = ['id', 'author_name', 'author_email', 'date', 'subject', 'body']
-GIT_LOG_FORMAT = ['%H', '%an', '%ae', '%ad', '%s', '%b']
-GIT_LOG_FORMAT = '%x1f'.join(GIT_LOG_FORMAT) + '%x1e'
+GIT_LOG_FORMAT = '%x1f'.join(['%h', '%an', '%ae', '%ad', '%s', '%b']) + '%x1e'
+
+STATUS_COLUR_MAP = {
+    'Resolved': 'green',
+    'Open': 'red',
+    'Closed': 'yellow'
+}
 
 
 def get_jiras_for_version(version):
-    asf_jira = jira.client.JIRA(
-            {'server': JIRA_API_BASE})
+    asf_jira = jira.client.JIRA({'server': JIRA_API_BASE})
 
-    issues = asf_jira.search_issues('PROJECT={} and fixVersion={}'.format(PROJECT, version))
-    return issues
+    start_at = 0
+    page_size = 50
+    while True:
+        results = asf_jira.search_issues(
+            'PROJECT={} and fixVersion={} order by updated desc'.format(PROJECT, version),
+            maxResults=page_size,
+            startAt=start_at,
+        )
 
+        for r in results:
+            yield r
+
+        if len(results) < page_size:
+            break
+
+        start_at += page_size
+
+
+issue_re = re.compile(r".*? (AIRFLOW-[0-9]{1,6}) (?: \]|\s|: )", flags=re.X)
+pr_re = re.compile(r"(.*)Closes (#[0-9]{1,6})", flags=re.DOTALL)
+pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$")
+
+
+def get_merged_issues(repo, version, previous_version=None):
+    log_args = ['--format={}'.format(GIT_LOG_FORMAT)]
+    if previous_version:
+        log_args.append(previous_version + "..")
+    log = repo.git.log(*log_args)
 
-def get_merged_issues(version):
-    repo = git.Repo(".", search_parent_directories=True)
-    log = repo.git.log('--format={}'.format(GIT_LOG_FORMAT))
     log = log.strip('\n\x1e').split("\x1e")
     log = [row.strip().split("\x1f") for row in log]
     log = [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
 
-    issue_re = re.compile(".*(AIRFLOW-[0-9]{1,6})(\]|\s|:)")
-    pr_re = re.compile("(.*)Closes (#[0-9]{1,6})", flags=re.DOTALL)
-
     merges = {}
     for log_item in log:
         issue_id = None
 
-        match = issue_re.match(log_item['subject'])
+        issue_ids = issue_re.findall(log_item['subject'])
+
+        match = pr_title_re.match(log_item['subject'])
         if match:
-            issue_id = match.group(1)
-        if log_item.has_key('body'):
+            log_item['pull_request'] = match.group(1)
+        elif 'body' in log_item:
             match = pr_re.match(log_item['body'])
             if match:
                 log_item['pull_request'] = match.group(2)
@@ -106,11 +121,34 @@ def get_merged_issues(version):
         else:
             log_item['pull_request'] = '#na'
 
-        if issue_id:
+        for issue_id in issue_ids:
             merges[issue_id] = log_item
 
     return merges
 
+
+def get_commits_from_master(repo, issue):
+    log = repo.git.log(
+        '--format=%h%x1f%s',
+        '-1',
+        '--grep',
+        r'.*{issue}\(\]\|:\|\s\)'.format(issue=issue.key),
+        'origin/master')
+    if not log:
+        return None
+    commit, subject = log.split('\x1f')
+
+    merge = {'id': commit}
+    match = pr_title_re.match(subject)
+
+    if match:
+        merge['pull_request'] = match.group(1)
+    else:
+        merge['pull_request'] = '-'
+
+    return merge
+
+
 @click.group()
 def cli():
     r"""
@@ -123,26 +161,57 @@ def cli():
 
 @cli.command(short_help='Compare a jira target version against git merges')
 @click.argument('target_version', default=None)
-def compare(target_version):
-    merges = get_merged_issues(target_version)
+@click.argument('previous_version', default="")
+@click.option('--unmerged', 'unmerged_only', help="Show unmerged issues only", is_flag=True)
+def compare(target_version, previous_version=None, unmerged_only=False):
+    repo = git.Repo(".", search_parent_directories=True)
+    merges = get_merged_issues(repo, target_version, previous_version)
     issues = get_jiras_for_version(target_version)
 
-    print("{:<18}|{:<12}||{:<10}||{:<10}|{:<50}|{:<6}|{:<6}|{:<40}"
-          .format("ISSUE ID", "TYPE", "PRIORITY",
-                  "STATUS", "DESCRIPTION", "MERGED",
-                  "PR", "COMMIT"))
+    # :<18 says left align, pad to 18
+    # :<50.50 truncates after 50 chars
+    # !s forces as string - some of the Jira objects have a string method, but
+    #    Py3 doesn't call by default
+    formatstr = "{id:<18}|{typ!s:<12}||{priority!s:<10}||{status!s}|" \
+                "{description:<83.83}|{merged:<6}|{pr:<6}|{commit:<7}"
+
+    print(formatstr.format(
+        id="ISSUE ID",
+        typ="TYPE",
+        priority="PRIORITY",
+        status="STATUS".ljust(10),
+        description="DESCRIPTION",
+        merged="MERGED",
+        pr="PR",
+        commit="COMMIT"))
 
     for issue in issues:
         is_merged = issue.key in merges
-        print("{:<18}|{:<12}||{:<10}||{:<10}|{:<50}|{:<6}|{:<6}|{:<40}"
-              .format(issue.key,
-                      issue.fields.issuetype,
-                      issue.fields.priority,
-                      issue.fields.status,
-                      issue.fields.summary[:50],
-                      is_merged,
-                      merges[issue.key]['pull_request'] if is_merged else "-",
-                      merges[issue.key]['id'] if is_merged else "-"))
+
+        if unmerged_only and is_merged:
+            continue
+
+        # Put colour on the status field. Since it will have non-printable
+        # characters we can't us string format to limit the length
+        status = issue.fields.status.name
+        if status in STATUS_COLUR_MAP:
+            status = click.style(status[:10].ljust(10), STATUS_COLUR_MAP[status])
+        else:
+            status = status[:10].ljust(10)
+
+        merge = merges.get(issue.key)
+        if not merge and issue.fields.status.name in {'Resolved', 'Closed'}:
+            merge = get_commits_from_master(repo, issue)
+
+        print(formatstr.format(
+            id=issue.key,
+            typ=issue.fields.issuetype,
+            priority=issue.fields.priority,
+            status=status,
+            description=issue.fields.summary,
+            merged=is_merged,
+            pr=merge['pull_request'] if merge else "-",
+            commit=merge['id'] if merge else "-"))
 
 
 if __name__ == "__main__":
@@ -152,7 +221,5 @@ if __name__ == "__main__":
         exit(-1)
     try:
         cli()
-    except:
+    except Exception:
         raise
-
-
diff --git a/dev/airflow-pr b/dev/airflow-pr
index 65243ddf876ae..9b499ec0c2a53 100755
--- a/dev/airflow-pr
+++ b/dev/airflow-pr
@@ -54,7 +54,8 @@ except ImportError:
 try:
     import keyring
 except ImportError:
-    print("Could not find the keyring library. Run 'sudo pip install keyring' to install.")
+    print("Could not find the keyring library. "
+          "Run 'sudo pip install keyring' to install.")
     sys.exit(-1)
 
 # Location of your Airflow git development area
@@ -62,18 +63,18 @@ AIRFLOW_GIT_LOCATION = os.environ.get(
     "AIRFLOW_GIT",
     os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 
-# Remote name which points to the Gihub site
+# Remote name which points to the GitHub site
 GITHUB_REMOTE_NAME = os.environ.get("GITHUB_REMOTE_NAME", "github")
-# Remote name which points to Apache git
-APACHE_REMOTE_NAME = os.environ.get("APACHE_REMOTE_NAME", "apache")
-# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests
-# will be unauthenticated. You should only need to configure this if you find yourself regularly
-# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at
-# https://github.com/settings/tokens. This tool only requires the "public_repo" scope.
+# OAuth key used for issuing requests against the GitHub API. If this is not
+# defined, then requests will be unauthenticated. You should only need to
+# configure this if you find yourself regularly exceeding your IP's
+# unauthenticated request rate limit. You can create an OAuth key at
+# https://github.com/settings/tokens. This tool only requires the "public_repo"
+# scope.
 GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY")
 
-GITHUB_BASE = "https://github.com/apache/incubator-airflow/pull"
-GITHUB_API_BASE = "https://api.github.com/repos/apache/incubator-airflow"
+GITHUB_BASE = "https://github.com/apache/airflow/pull"
+GITHUB_API_BASE = "https://api.github.com/repos/apache/airflow"
 GITHUB_USER = 'asfgit'
 
 JIRA_BASE = "https://issues.apache.org/jira/browse"
@@ -172,7 +173,7 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
     pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
     target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
     run_cmd("git fetch %s pull/%s/head:%s" % (GITHUB_REMOTE_NAME, pr_num, pr_branch_name))
-    run_cmd("git fetch %s %s:%s" % (APACHE_REMOTE_NAME, target_ref, target_branch_name))
+    run_cmd("git fetch %s %s:%s" % (GITHUB_REMOTE_NAME, target_ref, target_branch_name))
     run_cmd("git checkout %s" % target_branch_name)
 
     had_conflicts = False
@@ -205,7 +206,8 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
     except Exception as e:
         msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
         continue_maybe(msg)
-        msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
+        msg = ("Okay, please fix any conflicts and 'git add' conflicting files... " +
+               "Finished?")
         continue_maybe(msg)
         had_conflicts = True
 
@@ -216,7 +218,6 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
     if pr_commits:
         all_text += ' '.join(c['commit']['message'] for c in pr_commits)
     all_jira_refs = standardize_jira_ref(all_text, only_jira=True)
-    all_jira_issues = re.findall("AIRFLOW-[0-9]{1,6}", all_jira_refs)
 
     merge_message_flags = []
 
@@ -315,7 +316,6 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
         if primary_author == "":
             primary_author = distinct_authors[0]
 
-        authors = "\n".join(["Author: %s" % a for a in distinct_authors])
         merge_message_flags.append(u'--author="{}"'.format(primary_author))
 
     else:
@@ -327,7 +327,7 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
     # reflow commit message
     seen_first_line = False
     for i in range(1, len(merge_message_flags)):
-        if merge_message_flags[i-1] == '-m':
+        if merge_message_flags[i - 1] == '-m':
             # let the first line be as long as the user wants
             if not seen_first_line:
                 if '\n\n' in merge_message_flags[i]:
@@ -376,7 +376,7 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
             run_cmd(['git', 'commit'] + commit_flags, echo_cmd=False)
 
     if local:
-        msg ='\n' + reflow("""
+        msg = '\n' + reflow("""
             The PR has been merged locally in branch {}.
             You may leave this program running while you work on it. When
             you are finished, press any key to delete the PR branch and
@@ -390,13 +390,13 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
     else:
         continue_maybe(
             '\n\nThe local merge is complete ({}).\n'.format(
-                target_branch_name)
-            + click.style(
-                'Push to Apache ({})?'.format(APACHE_REMOTE_NAME), 'red'))
+                target_branch_name) +
+            click.style(
+                'Push to Gitbox ({})?'.format(GITHUB_REMOTE_NAME), 'red'))
 
     try:
         run_cmd('git push %s %s:%s' % (
-            APACHE_REMOTE_NAME, target_branch_name, target_ref))
+            GITHUB_REMOTE_NAME, target_branch_name, target_ref))
     except Exception as e:
         clean_up()
         fail("Exception while pushing: %s" % e)
@@ -419,7 +419,7 @@ def cherry_pick(pr_num, merge_hash, default_branch):
         BRANCH_PREFIX, pr_num, pick_ref.upper())
 
     run_cmd("git fetch %s %s:%s" % (
-        APACHE_REMOTE_NAME, pick_ref, pick_branch_name))
+        GITHUB_REMOTE_NAME, pick_ref, pick_branch_name))
     run_cmd("git checkout %s" % pick_branch_name)
 
     try:
@@ -435,12 +435,12 @@ def cherry_pick(pr_num, merge_hash, default_branch):
         continue_maybe(msg)
 
     continue_maybe("Pick complete (local ref %s). Push to %s?" % (
-        pick_branch_name, APACHE_REMOTE_NAME))
+        pick_branch_name, GITHUB_REMOTE_NAME))
 
     try:
         run_cmd(
             'git push %s %s:%s' % (
-                APACHE_REMOTE_NAME, pick_branch_name, pick_ref))
+                GITHUB_REMOTE_NAME, pick_branch_name, pick_ref))
     except Exception as e:
         clean_up()
         fail("Exception while pushing: %s" % e)
@@ -459,7 +459,7 @@ def fix_version_from_branch(branch, versions):
     if branch == "master":
         return versions[0]
     else:
-        #TODO adopt a release scheme with branches. Spark uses branch-XX.
+        # TODO adopt a release scheme with branches. Spark uses branch-XX.
         branch_ver = branch.replace("branch-", "")
         versions = list(filter(
             lambda x: x.name.startswith(branch_ver), versions))
@@ -692,18 +692,19 @@ def resolve_jira_issue(comment=None, jira_id=None, merge_branches=None):
         action["id"],
         fixVersions=jira_fix_versions,
         comment=comment or None,
-        resolution = {'id': resolution.raw['id']})
+        resolution={'id': resolution.raw['id']})
 
     click.echo("Successfully resolved {id}{fv}!".format(
         id=jira_id,
         fv=' with fix versions={}'.format(fix_versions) if fix_versions else ''
-        ))
+    ))
 
 
 def standardize_jira_ref(text, only_jira=False):
     """
     Standardize the [AIRFLOW-XXXXX] [MODULE] prefix
-    Converts "[AIRFLOW-XXX][mllib] Issue", "[MLLib] AIRFLOW-XXX. Issue" or "AIRFLOW XXX [MLLIB]: Issue" to "[AIRFLOW-XXX][MLLIB] Issue"
+    Converts "[AIRFLOW-XXX][mllib] Issue", "[MLLib] AIRFLOW-XXX. Issue" or
+    "AIRFLOW XXX [MLLIB]: Issue" to "[AIRFLOW-XXX][MLLIB] Issue"
 
     >>> standardize_jira_ref("[AIRFLOW-5821] [SQL] ParquetRelation2 CTAS should check if delete is successful")
     '[AIRFLOW-5821][SQL] ParquetRelation2 CTAS should check if delete is successful'
@@ -713,8 +714,8 @@ def standardize_jira_ref(text, only_jira=False):
     '[AIRFLOW-5954][MLLIB] Top by key'
     >>> standardize_jira_ref("[AIRFLOW-979] a LRU scheduler for load balancing in TaskSchedulerImpl")
     '[AIRFLOW-979] a LRU scheduler for load balancing in TaskSchedulerImpl'
-    >>> standardize_jira_ref("AIRFLOW-1094 Support MiMa for reporting binary compatibility accross versions.")
-    '[AIRFLOW-1094] Support MiMa for reporting binary compatibility accross versions.'
+    >>> standardize_jira_ref("AIRFLOW-1094 Support MiMa for reporting binary compatibility across versions.")
+    '[AIRFLOW-1094] Support MiMa for reporting binary compatibility across versions.'
     >>> standardize_jira_ref("[WIP]  [AIRFLOW-1146] Vagrant support for Spark")
     '[AIRFLOW-1146][WIP] Vagrant support for Spark'
     >>> standardize_jira_ref("AIRFLOW-1032. If Yarn app fails before registering, app master stays aroun...")
@@ -725,7 +726,7 @@ def standardize_jira_ref(text, only_jira=False):
     'Additional information for users building from source code'
     >>> standardize_jira_ref('AIRFLOW 35 AIRFLOW--36 AIRFLOW  37 test', only_jira=True)
     '[AIRFLOW-35][AIRFLOW-36][AIRFLOW-37]'
-    """
+    """  # noqa
     jira_refs = []
     components = []
 
@@ -757,7 +758,7 @@ def standardize_jira_ref(text, only_jira=False):
 
     # Cleanup any remaining symbols:
     pattern = re.compile(r'^\W+(.*)', re.IGNORECASE)
-    if (pattern.search(text) is not None):
+    if pattern.search(text) is not None:
         text = pattern.search(text).groups()[0]
 
     def unique(seq):
@@ -770,8 +771,8 @@ def standardize_jira_ref(text, only_jira=False):
     # Assemble full text (JIRA ref(s), module(s), remaining text)
     clean_text = ''.join(unique(jira_refs)).strip()
     if not only_jira:
-         clean_text += (
-             ''.join(unique(components)).strip() + " " + text.strip())
+        clean_text += (
+            ''.join(unique(components)).strip() + " " + text.strip())
 
     # Replace multiple spaces with a single space, e.g. if no jira refs
     # and/or components were included
@@ -806,9 +807,6 @@ def main(pr_num, local=False):
     - GITHUB_REMOTE_NAME
         GitHub remote name (defaults to "github")
 
-    - APACHE_REMOTE_NAME
-        Apache git remote name (defaults to "apache")
-
     - JIRA_USERNAME
         ASF JIRA username for automatically closing JIRA issues. Users will be
         prompted if it is not set.
@@ -847,7 +845,6 @@ def main(pr_num, local=False):
         click.echo('Working with pull request {}'.format(pr_num))
 
     pr = get_json("{}/pulls/{}".format(GITHUB_API_BASE, pr_num))
-    pr_events = get_json("{}/issues/{}/events".format(GITHUB_API_BASE, pr_num))
 
     url = pr["url"]
 
@@ -875,33 +872,6 @@ def main(pr_num, local=False):
     base_ref = pr["head"]["ref"]
     pr_repo_desc = "%s/%s" % (user_login, base_ref)
 
-    # Merged pull requests are either closed or merged by asfgit
-    merge_commits = [
-        e for e in pr_events
-        if e["actor"]["login"] == GITHUB_USER and
-        (e["event"] == "closed" or e["event"] == "merged")]
-
-    if merge_commits and False:
-        merge_hash = merge_commits[0]["commit_id"]
-        message = get_json(
-            "%s/commits/%s" % (GITHUB_API_BASE, merge_hash)
-            )["commit"]["message"]
-
-        continue_maybe(
-            "Pull request %s has already been merged. "
-            "Do you want to backport?" % pr_num)
-        commit_is_downloaded = run_cmd(
-            ['git', 'rev-parse', '--quiet', '--verify',
-             "%s^{commit}" % merge_hash]) != ""
-        if not commit_is_downloaded:
-            fail(
-                "Couldn't find any merge commit for #%s, "
-                "you may need to update HEAD." % pr_num)
-
-        click.echo("Found commit %s:\n%s" % (merge_hash, message))
-        cherry_pick(pr_num, merge_hash, latest_branch)
-        sys.exit(0)
-
     if not bool(pr["mergeable"]):
         msg = ('Pull request {} is not mergeable in its current form.\n'
                'Continue anyway? (experts only!)'.format(pr_num))
@@ -961,11 +931,10 @@ def cli():
     This tool should be used by Airflow committers to test PRs, merge them
     into the master branch, and close related JIRA issues.
 
-    Before you begin, make sure you have created the 'apache' and 'github' git
-    remotes. You can use the "setup_git_remotes" command to do this
-    automatically. If you do not want to use these remote names, you can tell
-    the PR tool by setting the appropriate environment variables. For more
-    information, run:
+    Before you begin, make sure you have created the 'github' git remote. You
+    can use the "setup_git_remotes" command to do this automatically. If you do
+    not want to use these remote names, you can tell the PR tool by setting the
+    appropriate environment variable. For more information, run:
 
         airflow-pr merge --help
     """
@@ -973,7 +942,7 @@ def cli():
     status = run_cmd('git status --porcelain', echo_cmd=False)
     if status:
         msg = (
-            'You have uncomitted changes in this branch. Running this tool\n'
+            'You have uncommitted changes in this branch. Running this tool\n'
             'will delete them permanently. Continue?')
         if click.confirm(click.style(msg, fg='red', bold=True)):
             run_cmd('git reset --hard', echo_cmd=False)
@@ -986,11 +955,10 @@ def cli():
 def merge(pr_num):
     """
     Utility for creating well-formed pull request merges and pushing them
-    to Apache, as well as closing JIRA issues.
+    to Gitbox (a.k.a. GitHub), as well as closing JIRA issues.
 
     This tool assumes you already have a local Airflow git folder and that you
-    have added remotes corresponding to both (i) the github apache Airflow
-    mirror and (ii) the apache git repo.
+    have added remote corresponding to the github apache Airflow repo.
 
     To configure the tool, set the following env vars:
     - AIRFLOW_GIT
@@ -1000,9 +968,6 @@ def merge(pr_num):
     - GITHUB_REMOTE_NAME
         GitHub remote name (defaults to "github")
 
-    - APACHE_REMOTE_NAME
-        Apache git remote name (defaults to "apache")
-
     - JIRA_USERNAME
         ASF JIRA username for automatically closing JIRA issues. Users will be
         prompted if it is not set.
@@ -1045,13 +1010,11 @@ def setup_git_remotes():
     click.echo(reflow("""
         This command will create git remotes to mirror the following
         structure. If you do not want to use these names, you must set the
-        GITHUB_REMOTE_NAME and APACHE_REMOTE_NAME environment variables:
+        GITHUB_REMOTE_NAME environment variable:
 
           git remote -v
-          apache https://git-wip-us.apache.org/repos/asf/incubator-airflow.git (fetch)
-          apache https://git-wip-us.apache.org/repos/asf/incubator-airflow.git (push)
-          github https://github.com/apache/incubator-airflow.git (fetch)
-          github https://github.com/apache/incubator-airflow.git (push)
+          github https://github.com/apache/airflow.git (fetch)
+          github https://github.com/apache/airflow.git (push)
 
         If these remotes already exist, the tool will display an error.
         """))
@@ -1059,20 +1022,14 @@ def setup_git_remotes():
 
     error = False
     try:
-        run_cmd('git remote add apache https://git-wip-us.apache.org/repos/asf/incubator-airflow.git')
-    except:
-        click.echo(click.style(reflow(
-            '>>ERROR: Could not create apache remote. If it already exists, '
-            'run `git remote remove apache` to delete it.', fg='red')))
-        error = True
-    try:
-        run_cmd('git remote add github https://github.com/apache/incubator-airflow.git')
+        run_cmd('git remote add github https://github.com/apache/airflow.git')
     except:
         click.echo(click.style(reflow(
             '>>ERROR: Could not create github remote. If it already exists, '
             'run `git remote remove github` to delete it.', fg='red')))
         error = True
-    click.echo('Done setting up git remotes. Run git remote -v to see them.')
+    if not error:
+        click.echo('Done setting up git remotes. Run git remote -v to see them.')
 
 
 if __name__ == "__main__":
diff --git a/dev/sign.sh b/dev/sign.sh
index da0f7a94576b2..33054a175e46c 100755
--- a/dev/sign.sh
+++ b/dev/sign.sh
@@ -25,5 +25,4 @@
 NAME=${1}
 
 gpg --armor --output ${NAME}.asc --detach-sig ${NAME}
-gpg --print-md SHA512 ${NAME} > ${NAME}.sha
-gpg --print-md MD5 ${NAME} > ${NAME}.md5
+gpg --print-md SHA512 ${NAME} > ${NAME}.sha512
diff --git a/docs/api.rst b/docs/api.rst
index 4ea19c8969f92..8ddfa8a4fdac7 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Experimental Rest API
 =====================
 
@@ -7,10 +24,74 @@ available at /api/experimental/. Please note that we expect the endpoint definit
 Endpoints
 ---------
 
-This is a place holder until the swagger definitions are active
+.. http:post:: /api/experimental/dags/<DAG_ID>/dag_runs
+
+
+  Creates a dag_run for a given dag id.
+
+
+  **Trigger DAG with config, example:**
+
+  .. code-block:: bash
+
+    curl -X POST \
+      http://localhost:8080/api/experimental/dags/<DAG_ID>/dag_runs \
+      -H 'Cache-Control: no-cache' \
+      -H 'Content-Type: application/json' \
+      -d '{"conf":"{\"key\":\"value\"}"}'
+
+
+.. http:get:: /api/experimental/dags/<DAG_ID>/dag_runs
+
+  Returns a list of Dag Runs for a specific DAG ID.
+
+.. http:get:: /api/experimental/dags/<string:dag_id>/dag_runs/<string:execution_date>
+
+  Returns a JSON with a dag_run's public instance variables. The format for the <string:execution_date> is expected to be "YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15".
+
+
+.. http:get:: /api/experimental/test
+
+  To check REST API server correct work. Return status 'OK'.
+
+
+.. http:get:: /api/experimental/dags/<DAG_ID>/tasks/<TASK_ID>
+
+  Returns info for a task.
+
+
+.. http:get:: /api/experimental/dags/<DAG_ID>/dag_runs/<string:execution_date>/tasks/<TASK_ID>
+
+  Returns a JSON with a task instance's public instance variables. The format for the <string:execution_date> is expected to be "YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15".
+
+
+.. http:get:: /api/experimental/dags/<DAG_ID>/paused/<string:paused>
+
+  '<string:paused>' must be a 'true' to pause a DAG and 'false' to unpause.
+
+
+.. http:get:: /api/experimental/latest_runs
+
+  Returns the latest DagRun for each DAG formatted for the UI.
+
+
+.. http:get:: /api/experimental/pools
+
+  Get all pools.
+
+
+.. http:get:: /api/experimental/pools/<string:name>
+
+  Get pool by a given name.
+
+.. http:post:: /api/experimental/pools
+
+  Create a pool.
+
+.. http:delete:: /api/experimental/pools/<string:name>
+
+  Delete pool.
 
-* /api/experimental/dags/<DAG_ID>/tasks/<TASK_ID> returns info for a task (GET).
-* /api/experimental/dags/<DAG_ID>/dag_runs creates a dag_run for a given dag id (POST).
 
 CLI
 -----
diff --git a/docs/autoapi_templates/index.rst b/docs/autoapi_templates/index.rst
new file mode 100644
index 0000000000000..ab32d3aa29c0c
--- /dev/null
+++ b/docs/autoapi_templates/index.rst
@@ -0,0 +1,129 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+API Reference
+=============
+
+Operators
+---------
+Operators allow for generation of certain types of tasks that become nodes in
+the DAG when instantiated. All operators derive from :class:`~airflow.models.BaseOperator` and
+inherit many attributes and methods that way.
+
+There are 3 main types of operators:
+
+- Operators that performs an **action**, or tell another system to
+  perform an action
+- **Transfer** operators move data from one system to another
+- **Sensors** are a certain type of operator that will keep running until a
+  certain criterion is met. Examples include a specific file landing in HDFS or
+  S3, a partition appearing in Hive, or a specific time of the day. Sensors
+  are derived from :class:`~airflow.sensors.base_sensor_operator.BaseSensorOperator` and run a poke
+  method at a specified :attr:`~airflow.sensors.base_sensor_operator.BaseSensorOperator.poke_interval` until it returns ``True``.
+
+BaseOperator
+''''''''''''
+All operators are derived from :class:`~airflow.models.BaseOperator` and acquire much
+functionality through inheritance. Since this is the core of the engine,
+it's worth taking the time to understand the parameters of :class:`~airflow.models.BaseOperator`
+to understand the primitive features that can be leveraged in your
+DAGs.
+
+BaseSensorOperator
+''''''''''''''''''
+All sensors are derived from :class:`~airflow.sensors.base_sensor_operator.BaseSensorOperator`. All sensors inherit
+the :attr:`~airflow.sensors.base_sensor_operator.BaseSensorOperator.timeout` and :attr:`~airflow.sensors.base_sensor_operator.BaseSensorOperator.poke_interval` on top of the :class:`~airflow.models.BaseOperator`
+attributes.
+
+Operators packages
+''''''''''''''''''
+All operators are in the following packages:
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/operators/index
+
+  airflow/sensors/index
+
+  airflow/contrib/operators/index
+
+  airflow/contrib/sensors/index
+
+
+Hooks
+-----
+Hooks are interfaces to external platforms and databases, implementing a common
+interface when possible and acting as building blocks for operators. All hooks
+are derived from :class:`~airflow.hooks.base_hook.BaseHook`.
+
+Hooks packages
+''''''''''''''
+All hooks are in the following packages:
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/hooks/index
+
+  airflow/contrib/hooks/index
+
+
+Executors
+---------
+Executors are the mechanism by which task instances get run. All executors are
+derived from :class:`~airflow.executors.base_executor.BaseExecutor`.
+
+Executors packages
+''''''''''''''''''
+All executors are in the following packages:
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/executors/index
+
+  airflow/contrib/executors/index
+
+
+Models
+------
+Models are built on top of the SQLAlchemy ORM Base class, and instances are
+persisted in the database.
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/models/index
+
+
+Core and community package
+--------------------------
+Formerly the core code was maintained by the original creators - Airbnb. The code
+that was in the contrib package was supported by the community. The project
+was passed to the Apache community and currently the entire code is
+maintained by the community, so now the division has no justification,
+and it is only due to historical reasons. Currently, all new classes are
+added only to the contrib package.
diff --git a/docs/build.sh b/docs/build.sh
index b3a9cbef99c7f..7d7a433f5ac02 100755
--- a/docs/build.sh
+++ b/docs/build.sh
@@ -18,5 +18,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-rm -r _build
-make html
+set -e
+
+FWDIR="$(cd "`dirname "$0"`"; pwd)"
+cd "$FWDIR"
+
+[[ -d "_build" ]] && rm -r _build
+[[ -d "_api" ]] && rm -r _api
+
+SUCCEED_LINE=$(make html |\
+    tee /dev/tty |\
+    grep 'build succeeded' |\
+    head -1)
+
+NUM_CURRENT_WARNINGS=$(echo $SUCCEED_LINE |\
+    sed -E 's/build succeeded, ([0-9]+) warnings?\./\1/g')
+
+if echo $SUCCEED_LINE | grep -q "warning"; then
+    echo
+    echo "Unexpected problems found in the documentation. "
+    echo "Currently, ${NUM_CURRENT_WARNINGS} warnings found"
+    echo
+    exit 1
+fi
diff --git a/docs/changelog.rst b/docs/changelog.rst
new file mode 100644
index 0000000000000..8fa1965686619
--- /dev/null
+++ b/docs/changelog.rst
@@ -0,0 +1,21 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Changelog
+=========
+
+.. include:: ../CHANGELOG.txt
diff --git a/docs/cli.rst b/docs/cli.rst
index f05cbfbe27ed2..0c0a67454b297 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Command Line Interface
 ======================
 
@@ -5,6 +22,10 @@ Airflow has a very rich command line interface that allows for
 many types of operation on a DAG, starting services, and supporting
 development and testing.
 
+.. contents:: Content
+    :local:
+    :depth: 2
+
 .. argparse::
    :module: airflow.bin.cli
    :func: get_parser
diff --git a/docs/code.rst b/docs/code.rst
deleted file mode 100644
index f055fc60cffe6..0000000000000
--- a/docs/code.rst
+++ /dev/null
@@ -1,421 +0,0 @@
-API Reference
-=============
-
-Operators
----------
-Operators allow for generation of certain types of tasks that become nodes in
-the DAG when instantiated. All operators derive from ``BaseOperator`` and
-inherit many attributes and methods that way. Refer to the BaseOperator_
-documentation for more details.
-
-There are 3 main types of operators:
-
-- Operators that performs an **action**, or tell another system to
-  perform an action
-- **Transfer** operators move data from one system to another
-- **Sensors** are a certain type of operator that will keep running until a
-  certain criterion is met. Examples include a specific file landing in HDFS or
-  S3, a partition appearing in Hive, or a specific time of the day. Sensors
-  are derived from ``BaseSensorOperator`` and run a poke
-  method at a specified ``poke_interval`` until it returns ``True``.
-
-BaseOperator
-''''''''''''
-All operators are derived from ``BaseOperator`` and acquire much
-functionality through inheritance. Since this is the core of the engine,
-it's worth taking the time to understand the parameters of ``BaseOperator``
-to understand the primitive features that can be leveraged in your
-DAGs.
-
-
-.. autoclass:: airflow.models.BaseOperator
-
-
-BaseSensorOperator
-'''''''''''''''''''
-All sensors are derived from ``BaseSensorOperator``. All sensors inherit
-the ``timeout`` and ``poke_interval`` on top of the ``BaseOperator``
-attributes.
-
-.. autoclass:: airflow.sensors.base_sensor_operator.BaseSensorOperator
-
-
-Core Operators
-''''''''''''''
-
-Operators
-^^^^^^^^^
-
-.. autoclass:: airflow.operators.bash_operator.BashOperator
-.. autoclass:: airflow.operators.python_operator.BranchPythonOperator
-.. autoclass:: airflow.operators.check_operator.CheckOperator
-.. autoclass:: airflow.operators.docker_operator.DockerOperator
-.. autoclass:: airflow.operators.dummy_operator.DummyOperator
-.. autoclass:: airflow.operators.druid_check_operator.DruidCheckOperator
-.. autoclass:: airflow.operators.email_operator.EmailOperator
-.. autoclass:: airflow.operators.generic_transfer.GenericTransfer
-.. autoclass:: airflow.operators.hive_to_druid.HiveToDruidTransfer
-.. autoclass:: airflow.operators.hive_to_mysql.HiveToMySqlTransfer
-.. autoclass:: airflow.operators.hive_to_samba_operator.Hive2SambaOperator
-.. autoclass:: airflow.operators.hive_operator.HiveOperator
-.. autoclass:: airflow.operators.hive_stats_operator.HiveStatsCollectionOperator
-.. autoclass:: airflow.operators.check_operator.IntervalCheckOperator
-.. autoclass:: airflow.operators.jdbc_operator.JdbcOperator
-.. autoclass:: airflow.operators.latest_only_operator.LatestOnlyOperator
-.. autoclass:: airflow.operators.mssql_operator.MsSqlOperator
-.. autoclass:: airflow.operators.mssql_to_hive.MsSqlToHiveTransfer
-.. autoclass:: airflow.operators.mysql_operator.MySqlOperator
-.. autoclass:: airflow.operators.mysql_to_hive.MySqlToHiveTransfer
-.. autoclass:: airflow.operators.oracle_operator.OracleOperator
-.. autoclass:: airflow.operators.pig_operator.PigOperator
-.. autoclass:: airflow.operators.postgres_operator.PostgresOperator
-.. autoclass:: airflow.operators.presto_check_operator.PrestoCheckOperator
-.. autoclass:: airflow.operators.presto_check_operator.PrestoIntervalCheckOperator
-.. autoclass:: airflow.operators.presto_to_mysql.PrestoToMySqlTransfer
-.. autoclass:: airflow.operators.presto_check_operator.PrestoValueCheckOperator
-.. autoclass:: airflow.operators.python_operator.PythonOperator
-.. autoclass:: airflow.operators.python_operator.PythonVirtualenvOperator
-.. autoclass:: airflow.operators.s3_file_transform_operator.S3FileTransformOperator
-.. autoclass:: airflow.operators.s3_to_hive_operator.S3ToHiveTransfer
-.. autoclass:: airflow.operators.s3_to_redshift_operator.S3ToRedshiftTransfer
-.. autoclass:: airflow.operators.python_operator.ShortCircuitOperator
-.. autoclass:: airflow.operators.http_operator.SimpleHttpOperator
-.. autoclass:: airflow.operators.slack_operator.SlackAPIOperator
-.. autoclass:: airflow.operators.slack_operator.SlackAPIPostOperator
-.. autoclass:: airflow.operators.sqlite_operator.SqliteOperator
-.. autoclass:: airflow.operators.subdag_operator.SubDagOperator
-.. autoclass:: airflow.operators.dagrun_operator.TriggerDagRunOperator
-.. autoclass:: airflow.operators.check_operator.ValueCheckOperator
-.. autoclass:: airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer
-
-Sensors
-^^^^^^^
-.. autoclass:: airflow.sensors.external_task_sensor.ExternalTaskSensor
-.. autoclass:: airflow.sensors.hdfs_sensor.HdfsSensor
-.. autoclass:: airflow.sensors.hive_partition_sensor.HivePartitionSensor
-.. autoclass:: airflow.sensors.http_sensor.HttpSensor
-.. autoclass:: airflow.sensors.metastore_partition_sensor.MetastorePartitionSensor
-.. autoclass:: airflow.sensors.named_hive_partition_sensor.NamedHivePartitionSensor
-.. autoclass:: airflow.sensors.s3_key_sensor.S3KeySensor
-.. autoclass:: airflow.sensors.s3_prefix_sensor.S3PrefixSensor
-.. autoclass:: airflow.sensors.sql_sensor.SqlSensor
-.. autoclass:: airflow.sensors.time_sensor.TimeSensor
-.. autoclass:: airflow.sensors.time_delta_sensor.TimeDeltaSensor
-.. autoclass:: airflow.sensors.web_hdfs_sensor.WebHdfsSensor
-
-Community-contributed Operators
-'''''''''''''''''''''''''''''''
-
-Operators
-^^^^^^^^^
-.. Alphabetize this list
-
-.. autoclass:: airflow.contrib.operators.awsbatch_operator.AWSBatchOperator
-.. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryCheckOperator
-.. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator
-.. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator
-.. autoclass:: airflow.contrib.operators.bigquery_get_data.BigQueryGetDataOperator
-.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator
-.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator
-.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryOperator
-.. autoclass:: airflow.contrib.operators.bigquery_table_delete_operator.BigQueryTableDeleteOperator
-.. autoclass:: airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator
-.. autoclass:: airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator
-.. autoclass:: airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator
-.. autoclass:: airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator
-.. autoclass:: airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcPigOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHiveOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateBaseOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateOperator
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateInlineOperator
-.. autoclass:: airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator
-.. autoclass:: airflow.contrib.operators.datastore_import_operator.DatastoreImportOperator
-.. autoclass:: airflow.contrib.operators.discord_webhook_operator.DiscordWebhookOperator
-.. autoclass:: airflow.contrib.operators.druid_operator.DruidOperator
-.. autoclass:: airflow.contrib.operators.ecs_operator.ECSOperator
-.. autoclass:: airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator
-.. autoclass:: airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator
-.. autoclass:: airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator
-.. autoclass:: airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.file_to_wasb.FileToWasbOperator
-.. autoclass:: airflow.contrib.operators.gcp_container_operator.GKEClusterCreateOperator
-.. autoclass:: airflow.contrib.operators.gcp_container_operator.GKEClusterDeleteOperator
-.. autoclass:: airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator
-.. autoclass:: airflow.contrib.operators.gcs_list_operator.GoogleCloudStorageListOperator
-.. autoclass:: airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator
-.. autoclass:: airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator
-.. autoclass:: airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator
-.. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPIOperator
-.. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPISendRoomNotificationOperator
-.. autoclass:: airflow.contrib.operators.hive_to_dynamodb.HiveToDynamoDBTransferOperator
-.. autoclass:: airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator
-.. autoclass:: airflow.contrib.operators.jira_operator.JiraOperator
-.. autoclass:: airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineModelOperator
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineVersionOperator
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator
-.. autoclass:: airflow.contrib.operators.mongo_to_s3.MongoToS3Operator
-.. autoclass:: airflow.contrib.operators.mysql_to_gcs.MySqlToGoogleCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.postgres_to_gcs_operator.PostgresToGoogleCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubTopicCreateOperator
-.. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubTopicDeleteOperator
-.. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubSubscriptionCreateOperator
-.. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubSubscriptionDeleteOperator
-.. autoclass:: airflow.contrib.operators.pubsub_operator.PubSubPublishOperator
-.. autoclass:: airflow.contrib.operators.qubole_check_operator.QuboleCheckOperator
-.. autoclass:: airflow.contrib.operators.qubole_check_operator.QuboleValueCheckOperator
-.. autoclass:: airflow.contrib.operators.qubole_operator.QuboleOperator
-.. autoclass:: airflow.contrib.operators.s3_list_operator.S3ListOperator
-.. autoclass:: airflow.contrib.operators.s3_to_gcs_operator.S3ToGoogleCloudStorageOperator
-.. autoclass:: airflow.contrib.operators.segment_track_event_operator.SegmentTrackEventOperator
-.. autoclass:: airflow.contrib.operators.sftp_operator.SFTPOperator
-.. autoclass:: airflow.contrib.operators.slack_webhook_operator.SlackWebhookOperator
-.. autoclass:: airflow.contrib.operators.snowflake_operator.SnowflakeOperator
-.. autoclass:: airflow.contrib.operators.spark_jdbc_operator.SparkJDBCOperator
-.. autoclass:: airflow.contrib.operators.spark_sql_operator.SparkSqlOperator
-.. autoclass:: airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator
-.. autoclass:: airflow.contrib.operators.sqoop_operator.SqoopOperator
-.. autoclass:: airflow.contrib.operators.ssh_operator.SSHOperator
-.. autoclass:: airflow.contrib.operators.vertica_operator.VerticaOperator
-.. autoclass:: airflow.contrib.operators.vertica_to_hive.VerticaToHiveTransfer
-.. autoclass:: airflow.contrib.operators.winrm_operator.WinRMOperator
-
-Sensors
-^^^^^^^
-
-.. autoclass:: airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor
-.. autoclass:: airflow.contrib.sensors.bash_sensor.BashSensor
-.. autoclass:: airflow.contrib.sensors.bigquery_sensor.BigQueryTableSensor
-.. autoclass:: airflow.contrib.sensors.cassandra_sensor.CassandraRecordSensor
-.. autoclass:: airflow.contrib.sensors.datadog_sensor.DatadogSensor
-.. autoclass:: airflow.contrib.sensors.emr_base_sensor.EmrBaseSensor
-.. autoclass:: airflow.contrib.sensors.emr_job_flow_sensor.EmrJobFlowSensor
-.. autoclass:: airflow.contrib.sensors.emr_step_sensor.EmrStepSensor
-.. autoclass:: airflow.contrib.sensors.file_sensor.FileSensor
-.. autoclass:: airflow.contrib.sensors.ftp_sensor.FTPSensor
-.. autoclass:: airflow.contrib.sensors.ftp_sensor.FTPSSensor
-.. autoclass:: airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageObjectSensor
-.. autoclass:: airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageObjectUpdatedSensor
-.. autoclass:: airflow.contrib.sensors.gcs_sensor.GoogleCloudStoragePrefixSensor
-.. autoclass:: airflow.contrib.sensors.hdfs_sensor.HdfsSensorFolder
-.. autoclass:: airflow.contrib.sensors.hdfs_sensor.HdfsSensorRegex
-.. autoclass:: airflow.contrib.sensors.jira_sensor.JiraSensor
-.. autoclass:: airflow.contrib.sensors.pubsub_sensor.PubSubPullSensor
-.. autoclass:: airflow.contrib.sensors.qubole_sensor.QuboleSensor
-.. autoclass:: airflow.contrib.sensors.redis_key_sensor.RedisKeySensor
-.. autoclass:: airflow.contrib.sensors.sftp_sensor.SFTPSensor
-.. autoclass:: airflow.contrib.sensors.wasb_sensor.WasbBlobSensor
-
-.. _macros:
-
-Macros
----------
-Here's a list of variables and macros that can be used in templates
-
-
-Default Variables
-'''''''''''''''''
-The Airflow engine passes a few variables by default that are accessible
-in all templates
-
-=================================   ====================================
-Variable                            Description
-=================================   ====================================
-``{{ ds }}``                        the execution date as ``YYYY-MM-DD``
-``{{ ds_nodash }}``                 the execution date as ``YYYYMMDD``
-``{{ prev_ds }}``                   the previous execution date as ``YYYY-MM-DD``.
-                                    if ``{{ ds }}`` is ``2016-01-08`` and ``schedule_interval`` is ``@weekly``,
-                                    ``{{ prev_ds }}`` will be ``2016-01-01``.
-``{{ next_ds }}``                   the next execution date as ``YYYY-MM-DD``.
-                                    if ``{{ ds }}`` is ``2016-01-01`` and ``schedule_interval`` is ``@weekly``,
-                                    ``{{ prev_ds }}`` will be ``2016-01-08``.
-``{{ yesterday_ds }}``              yesterday's date as ``YYYY-MM-DD``
-``{{ yesterday_ds_nodash }}``       yesterday's date as ``YYYYMMDD``
-``{{ tomorrow_ds }}``               tomorrow's date as ``YYYY-MM-DD``
-``{{ tomorrow_ds_nodash }}``        tomorrow's date as ``YYYYMMDD``
-``{{ ts }}``                        same as ``execution_date.isoformat()``
-``{{ ts_nodash }}``                 same as ``ts`` without ``-`` and ``:``
-``{{ execution_date }}``            the execution_date, (datetime.datetime)
-``{{ prev_execution_date }}``       the previous execution date (if available) (datetime.datetime)
-``{{ next_execution_date }}``       the next execution date (datetime.datetime)
-``{{ dag }}``                       the DAG object
-``{{ task }}``                      the Task object
-``{{ macros }}``                    a reference to the macros package, described below
-``{{ task_instance }}``             the task_instance object
-``{{ end_date }}``                  same as ``{{ ds }}``
-``{{ latest_date }}``               same as ``{{ ds }}``
-``{{ ti }}``                        same as ``{{ task_instance }}``
-``{{ params }}``                    a reference to the user-defined params dictionary which can be overridden by
-                                    the dictionary passed through ``trigger_dag -c`` if you enabled
-                                    ``dag_run_conf_overrides_params` in ``airflow.cfg``
-``{{ var.value.my_var }}``          global defined variables represented as a dictionary
-``{{ var.json.my_var.path }}``      global defined variables represented as a dictionary
-                                    with deserialized JSON object, append the path to the
-                                    key within the JSON object
-``{{ task_instance_key_str }}``     a unique, human-readable key to the task instance
-                                    formatted ``{dag_id}_{task_id}_{ds}``
-``{{ conf }}``                      the full configuration object located at
-                                    ``airflow.configuration.conf`` which
-                                    represents the content of your
-                                    ``airflow.cfg``
-``{{ run_id }}``                    the ``run_id`` of the current DAG run
-``{{ dag_run }}``                   a reference to the DagRun object
-``{{ test_mode }}``                 whether the task instance was called using
-                                    the CLI's test subcommand
-=================================   ====================================
-
-Note that you can access the object's attributes and methods with simple
-dot notation. Here are some examples of what is possible:
-``{{ task.owner }}``, ``{{ task.task_id }}``, ``{{ ti.hostname }}``, ...
-Refer to the models documentation for more information on the objects'
-attributes and methods.
-
-The ``var`` template variable allows you to access variables defined in Airflow's
-UI. You can access them as either plain-text or JSON. If you use JSON, you are
-also able to walk nested structures, such as dictionaries like:
-``{{ var.json.my_dict_var.key1 }}``
-
-Macros
-''''''
-Macros are a way to expose objects to your templates and live under the
-``macros`` namespace in your templates.
-
-A few commonly used libraries and methods are made available.
-
-
-=================================   ====================================
-Variable                            Description
-=================================   ====================================
-``macros.datetime``                 The standard lib's ``datetime.datetime``
-``macros.timedelta``                 The standard lib's ``datetime.timedelta``
-``macros.dateutil``                 A reference to the ``dateutil`` package
-``macros.time``                     The standard lib's ``time``
-``macros.uuid``                     The standard lib's ``uuid``
-``macros.random``                   The standard lib's ``random``
-=================================   ====================================
-
-
-Some airflow specific macros are also defined:
-
-.. automodule:: airflow.macros
-    :show-inheritance:
-    :members:
-
-.. autofunction:: airflow.macros.hive.closest_ds_partition
-.. autofunction:: airflow.macros.hive.max_partition
-
-.. _models_ref:
-
-Models
-------
-
-Models are built on top of the SQLAlchemy ORM Base class, and instances are
-persisted in the database.
-
-
-.. automodule:: airflow.models
-    :show-inheritance:
-    :members: DAG, BaseOperator, TaskInstance, DagBag, Connection
-
-Hooks
------
-
-Hooks are interfaces to external platforms and databases, implementing a common
-interface when possible and acting as building blocks for operators.
-
-.. autoclass:: airflow.hooks.dbapi_hook.DbApiHook
-.. autoclass:: airflow.hooks.docker_hook.DockerHook
-.. automodule:: airflow.hooks.hive_hooks
-    :members:
-      HiveCliHook,
-      HiveMetastoreHook,
-      HiveServer2Hook
-.. autoclass:: airflow.hooks.http_hook.HttpHook
-.. autoclass:: airflow.hooks.druid_hook.DruidDbApiHook
-.. autoclass:: airflow.hooks.druid_hook.DruidHook
-.. autoclass:: airflow.hooks.hdfs_hook.HDFSHook
-.. autoclass:: airflow.hooks.jdbc_hook.JdbcHook
-.. autoclass:: airflow.hooks.mssql_hook.MsSqlHook
-.. autoclass:: airflow.hooks.mysql_hook.MySqlHook
-.. autoclass:: airflow.hooks.oracle_hook.OracleHook
-.. autoclass:: airflow.hooks.pig_hook.PigCliHook
-.. autoclass:: airflow.hooks.postgres_hook.PostgresHook
-.. autoclass:: airflow.hooks.presto_hook.PrestoHook
-.. autoclass:: airflow.hooks.S3_hook.S3Hook
-.. autoclass:: airflow.hooks.samba_hook.SambaHook
-.. autoclass:: airflow.hooks.slack_hook.SlackHook
-.. autoclass:: airflow.hooks.sqlite_hook.SqliteHook
-.. autoclass:: airflow.hooks.webhdfs_hook.WebHDFSHook
-.. autoclass:: airflow.hooks.zendesk_hook.ZendeskHook
-
-Community contributed hooks
-'''''''''''''''''''''''''''
-.. Alphabetize this list
-.. autoclass:: airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook
-.. autoclass:: airflow.contrib.hooks.aws_hook.AwsHook
-.. autoclass:: airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook
-.. autoclass:: airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook
-.. autoclass:: airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook
-.. autoclass:: airflow.contrib.hooks.bigquery_hook.BigQueryHook
-.. autoclass:: airflow.contrib.hooks.cassandra_hook.CassandraHook
-.. autoclass:: airflow.contrib.hooks.cloudant_hook.CloudantHook
-.. autoclass:: airflow.contrib.hooks.databricks_hook.DatabricksHook
-.. autoclass:: airflow.contrib.hooks.datadog_hook.DatadogHook
-.. autoclass:: airflow.contrib.hooks.datastore_hook.DatastoreHook
-.. autoclass:: airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook
-.. autoclass:: airflow.contrib.hooks.emr_hook.EmrHook
-.. autoclass:: airflow.contrib.hooks.fs_hook.FSHook
-.. autoclass:: airflow.contrib.hooks.ftp_hook.FTPHook
-.. autoclass:: airflow.contrib.hooks.ftp_hook.FTPSHook
-.. autoclass:: airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook
-.. autoclass:: airflow.contrib.hooks.gcp_container_hook.GKEClusterHook
-.. autoclass:: airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook
-.. autoclass:: airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook
-.. autoclass:: airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook
-.. autoclass:: airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook
-.. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook
-.. autoclass:: airflow.contrib.hooks.jenkins_hook.JenkinsHook
-.. autoclass:: airflow.contrib.hooks.jira_hook.JiraHook
-.. autoclass:: airflow.contrib.hooks.mongo_hook.MongoHook
-.. autoclass:: airflow.contrib.hooks.pinot_hook.PinotDbApiHook
-.. autoclass:: airflow.contrib.hooks.qubole_hook.QuboleHook
-.. autoclass:: airflow.contrib.hooks.redis_hook.RedisHook
-.. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook
-.. autoclass:: airflow.contrib.hooks.salesforce_hook.SalesforceHook
-.. autoclass:: airflow.contrib.hooks.segment_hook.SegmentHook
-.. autoclass:: airflow.contrib.hooks.sftp_hook.SFTPHook
-.. autoclass:: airflow.contrib.hooks.slack_webhook_hook.SlackWebhookHook
-.. autoclass:: airflow.contrib.hooks.snowflake_hook.SnowflakeHook
-.. autoclass:: airflow.contrib.hooks.spark_jdbc_hook.SparkJDBCHook
-.. autoclass:: airflow.contrib.hooks.spark_sql_hook.SparkSqlHook
-.. autoclass:: airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook
-.. autoclass:: airflow.contrib.hooks.sqoop_hook.SqoopHook
-.. autoclass:: airflow.contrib.hooks.ssh_hook.SSHHook
-.. autoclass:: airflow.contrib.hooks.vertica_hook.VerticaHook
-.. autoclass:: airflow.contrib.hooks.wasb_hook.WasbHook
-.. autoclass:: airflow.contrib.hooks.winrm_hook.WinRMHook
-
-Executors
----------
-Executors are the mechanism by which task instances get run.
-
-.. autoclass:: airflow.executors.local_executor.LocalExecutor
-.. autoclass:: airflow.executors.celery_executor.CeleryExecutor
-.. autoclass:: airflow.executors.sequential_executor.SequentialExecutor
-
-Community-contributed executors
-'''''''''''''''''''''''''''''''
-
-.. autoclass:: airflow.contrib.executors.mesos_executor.MesosExecutor
diff --git a/docs/concepts.rst b/docs/concepts.rst
index 9b10224a817d8..5b1294d9d890a 100644
--- a/docs/concepts.rst
+++ b/docs/concepts.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Concepts
 ########
 
@@ -35,8 +52,10 @@ the ``DAG`` objects. You can have as many DAGs as you want, each describing an
 arbitrary number of tasks. In general, each one should correspond to a single
 logical workflow.
 
-.. note:: When searching for DAGs, Airflow will only consider files where the string
-   "airflow" and "DAG" both appear in the contents of the ``.py`` file.
+.. note:: When searching for DAGs, Airflow only considers python files
+   that contain the strings "airflow" and "DAG" by default. To consider
+   all python files instead, disable the ``DAG_DISCOVERY_SAFE_MODE``
+   configuration flag.
 
 Scope
 -----
@@ -90,6 +109,8 @@ DAGs can be used as context managers to automatically assign new operators to th
 
     op.dag is dag # True
 
+.. _concepts-operators:
+
 Operators
 =========
 
@@ -110,25 +131,28 @@ described elsewhere in this document.
 
 Airflow provides operators for many common tasks, including:
 
-- ``BashOperator`` - executes a bash command
-- ``PythonOperator`` - calls an arbitrary Python function
-- ``EmailOperator`` - sends an email
-- ``HTTPOperator`` - sends an HTTP request
-- ``MySqlOperator``, ``SqliteOperator``, ``PostgresOperator``, ``MsSqlOperator``, ``OracleOperator``, ``JdbcOperator``, etc. - executes a SQL command
+- :class:`~airflow.operators.bash_operator.BashOperator` - executes a bash command
+- :class:`~airflow.operators.python_operator.PythonOperator` - calls an arbitrary Python function
+- :class:`~airflow.operators.email_operator.EmailOperator` - sends an email
+- :class:`~airflow.operators.http_operator.SimpleHttpOperator` - sends an HTTP request
+- :class:`~airflow.operators.mysql_operator.MySqlOperator`,
+  :class:`~airflow.operators.sqlite_operator.SqliteOperator`,
+  :class:`~airflow.operators.postgres_operator.PostgresOperator`,
+  :class:`~airflow.operators.mssql_operator.MsSqlOperator`,
+  :class:`~airflow.operators.oracle_operator.OracleOperator`,
+  :class:`~airflow.operators.jdbc_operator.JdbcOperator`, etc. - executes a SQL command
 - ``Sensor`` - waits for a certain time, file, database row, S3 key, etc...
 
-
 In addition to these basic building blocks, there are many more specific
-operators: ``DockerOperator``, ``HiveOperator``, ``S3FileTransferOperator``,
-``PrestoToMysqlOperator``, ``SlackOperator``... you get the idea!
-
-The ``airflow/contrib/`` directory contains yet more operators built by the
-community. These operators aren't always as complete or well-tested as those in
-the main distribution, but allow users to more easily add new functionality to
-the platform.
+operators: :class:`~airflow.operators.docker_operator.DockerOperator`,
+:class:`~airflow.operators.hive_operator.HiveOperator`, :class:`~airflow.operators.s3_file_transform_operator.S3FileTransformOperator(`,
+:class:`~airflow.operators.presto_to_mysql.PrestoToMySqlTransfer`,
+:class:`~airflow.operators.slack_operator.SlackAPIOperator`... you get the idea!
 
 Operators are only loaded by Airflow if they are assigned to a DAG.
 
+See :doc:`howto/operator/index` for how to use Airflow operators.
+ 
 DAG Assignment
 --------------
 
@@ -264,7 +288,7 @@ Hooks
 Hooks are interfaces to external platforms and databases like Hive, S3,
 MySQL, Postgres, HDFS, and Pig. Hooks implement a common interface when
 possible, and act as a building block for operators. They also use
-the ``airflow.models.Connection`` model to retrieve hostnames
+the ``airflow.models.connection.Connection`` model to retrieve hostnames
 and authentication information. Hooks keep authentication code and
 information out of pipelines, centralized in the metadata database.
 
@@ -317,7 +341,7 @@ Connections
 ===========
 
 The connection information to external systems is stored in the Airflow
-metadata database and managed in the UI (``Menu -> Admin -> Connections``)
+metadata database and managed in the UI (``Menu -> Admin -> Connections``).
 A ``conn_id`` is defined there and hostname / login / password / schema
 information attached to it. Airflow pipelines can simply refer to the
 centrally managed ``conn_id`` without having to hard code any of this
@@ -329,17 +353,28 @@ from ``BaseHook``, Airflow will choose one connection randomly, allowing
 for some basic load balancing and fault tolerance when used in conjunction
 with retries.
 
+Airflow also has the ability to reference connections via environment
+variables from the operating system. Then connection parameters must
+be saved in URI format.
+
+If connections with the same ``conn_id`` are defined in both Airflow metadata
+database and environment variables, only the one in environment variables
+will be referenced by Airflow (for example, given ``conn_id``
+``postgres_master``, Airflow will search for ``AIRFLOW_CONN_POSTGRES_MASTER``
+in environment variables first and directly reference it if found,
+before it starts to search in metadata database).
+
 Many hooks have a default ``conn_id``, where operators using that hook do not
 need to supply an explicit connection ID. For example, the default
 ``conn_id`` for the :class:`~airflow.hooks.postgres_hook.PostgresHook` is
 ``postgres_default``.
 
-See :doc:`howto/manage-connections` for how to create and manage connections.
+See :doc:`howto/connection/index` for how to create and manage connections.
 
 Queues
 ======
 
-When using the CeleryExecutor, the celery queues that tasks are sent to
+When using the CeleryExecutor, the Celery queues that tasks are sent to
 can be specified. ``queue`` is an attribute of BaseOperator, so any
 task can be assigned to any queue. The default queue for the environment
 is defined in the ``airflow.cfg``'s ``celery -> default_queue``. This defines
@@ -347,7 +382,7 @@ the queue that tasks get assigned to when not specified, as well as which
 queue Airflow workers listen to when started.
 
 Workers can listen to one or multiple queues of tasks. When a worker is
-started (using the command ``airflow worker``), a set of comma delimited
+started (using the command ``airflow worker``), a set of comma-delimited
 queue names can be specified (e.g. ``airflow worker -q spark``). This worker
 will then only pick up tasks wired to the specified queue(s).
 
@@ -357,6 +392,8 @@ could take thousands of tasks without a problem), or from an environment
 perspective (you want a worker running from within the Spark cluster
 itself because it needs a very specific environment and security rights).
 
+.. _concepts:xcom:
+
 XComs
 =====
 
@@ -397,7 +434,7 @@ passed, then a corresponding list of XCom values is returned.
 It is also possible to pull XCom directly in a template, here's an example
 of what this may look like:
 
-.. code:: sql
+.. code:: jinja
 
     SELECT * FROM {{ task_instance.xcom_pull(task_ids='foo', key='table_name') }}
 
@@ -423,10 +460,14 @@ accessible and modifiable through the UI.
     from airflow.models import Variable
     foo = Variable.get("foo")
     bar = Variable.get("bar", deserialize_json=True)
+    baz = Variable.get("baz", default_var=None)
 
 The second call assumes ``json`` content and will be deserialized into
 ``bar``. Note that ``Variable`` is a sqlalchemy model and can be used
-as such.
+as such. The third call uses the ``default_var`` parameter with the value
+``None``, which either returns an existing value or ``None`` if the variable
+isn't defined. The get function will throw a ``KeyError`` if the variable
+doesn't exist and no default is provided.
 
 You can use a variable from a jinja template with the syntax :
 
@@ -450,8 +491,8 @@ that happened in an upstream task. One way to do this is by using the
 ``BranchPythonOperator``.
 
 The ``BranchPythonOperator`` is much like the PythonOperator except that it
-expects a python_callable that returns a task_id. The task_id returned
-is followed, and all of the other paths are skipped.
+expects a python_callable that returns a task_id (or list of task_ids). The
+task_id returned is followed, and all of the other paths are skipped.
 The task_id returned by the Python function has to be referencing a task
 directly downstream from the BranchPythonOperator task.
 
@@ -579,9 +620,37 @@ Service Level Agreements, or time by which a task or DAG should have
 succeeded, can be set at a task level as a ``timedelta``. If
 one or many instances have not succeeded by that time, an alert email is sent
 detailing the list of tasks that missed their SLA. The event is also recorded
-in the database and made available in the web UI under ``Browse->Missed SLAs``
+in the database and made available in the web UI under ``Browse->SLA Misses``
 where events can be analyzed and documented.
 
+Email Configuration
+-------------------
+
+You can configure the email that is being sent in your ``airflow.cfg``
+by setting a ``subject_template`` and/or a ``html_content_template``
+in the ``email`` section.
+
+.. code::
+
+  [email]
+
+  email_backend = airflow.utils.email.send_email_smtp
+
+  subject_template = /path/to/my_subject_template_file
+  html_content_template = /path/to/my_html_content_template_file
+
+To access the task's information you use `Jinja Templating <http://jinja.pocoo.org/docs/dev/>`_  in your template files.
+
+For example a ``html_content_template`` file could look like this:
+
+.. code::
+
+  Try {{try_number}} out of {{max_tries + 1}}<br>
+  Exception:<br>{{exception_html}}<br>
+  Log: <a href="{{ti.log_url}}">Link</a><br>
+  Host: {{ti.hostname}}<br>
+  Log file: {{ti.log_filepath}}<br>
+  Mark success: <a href="{{ti.mark_success_url}}">Link</a><br>
 
 Trigger Rules
 =============
@@ -602,12 +671,75 @@ while creating tasks:
 * ``all_done``: all parents are done with their execution
 * ``one_failed``: fires as soon as at least one parent has failed, it does not wait for all parents to be done
 * ``one_success``: fires as soon as at least one parent succeeds, it does not wait for all parents to be done
+* ``none_failed``: all parents have not failed (``failed`` or ``upstream_failed``) i.e. all parents have succeeded or been skipped
+* ``none_skipped``: no parent is in a ``skipped`` state, i.e. all parents are in a ``success``, ``failed``, or ``upstream_failed`` state
 * ``dummy``: dependencies are just for show, trigger at will
 
 Note that these can be used in conjunction with ``depends_on_past`` (boolean)
 that, when set to ``True``, keeps a task from getting triggered if the
 previous schedule for the task hasn't succeeded.
 
+One must be aware of the interaction between trigger rules and skipped tasks
+in schedule level. Skipped tasks will cascade through trigger rules 
+``all_success`` and ``all_failed`` but not ``all_done``, ``one_failed``, ``one_success``,
+``none_failed``, ``none_skipped`` and ``dummy``.
+
+For example, consider the following DAG:
+
+.. code:: python
+
+  #dags/branch_without_trigger.py
+  import datetime as dt  
+  
+  from airflow.models import DAG
+  from airflow.operators.dummy_operator import DummyOperator
+  from airflow.operators.python_operator import BranchPythonOperator  
+  
+  dag = DAG(
+      dag_id='branch_without_trigger',
+      schedule_interval='@once',
+      start_date=dt.datetime(2019, 2, 28)
+  )  
+  
+  run_this_first = DummyOperator(task_id='run_this_first', dag=dag)  
+  branching = BranchPythonOperator(
+      task_id='branching', dag=dag,
+      python_callable=lambda: 'branch_a'
+  )  
+  
+  branch_a = DummyOperator(task_id='branch_a', dag=dag)
+  follow_branch_a = DummyOperator(task_id='follow_branch_a', dag=dag)  
+  
+  branch_false = DummyOperator(task_id='branch_false', dag=dag)  
+  
+  join = DummyOperator(task_id='join', dag=dag)  
+  
+  run_this_first >> branching
+  branching >> branch_a >> follow_branch_a >> join
+  branching >> branch_false >> join
+
+In the case of this DAG, ``join`` is downstream of ``follow_branch_a`` 
+and ``branch_false``. The ``join`` task will show up as skipped 
+because its ``trigger_rule`` is set to ``all_success`` by default and 
+skipped tasks will cascade through ``all_success``. 
+
+.. image:: img/branch_without_trigger.png
+
+By setting ``trigger_rule`` to ``none_failed`` in ``join`` task, 
+
+.. code:: python
+  
+  #dags/branch_with_trigger.py
+  ...
+  join = DummyOperator(task_id='join', dag=dag, trigger_rule='none_failed')
+  ...
+
+The ``join`` task will be triggered as soon as 
+``branch_false`` has been skipped (a valid completion state) and 
+``follow_branch_a`` has succeeded. Because skipped tasks **will not** 
+cascade through ``none_failed``. 
+
+.. image:: img/branch_with_trigger.png
 
 Latest Run Only
 ===============
@@ -620,21 +752,9 @@ a pause just wastes CPU cycles.
 
 For situations like this, you can use the ``LatestOnlyOperator`` to skip
 tasks that are not being run during the most recent scheduled run for a
-DAG. The ``LatestOnlyOperator`` skips all immediate downstream tasks, and
-itself, if the time right now is not between its ``execution_time`` and the
-next scheduled ``execution_time``.
-
-One must be aware of the interaction between skipped tasks and trigger
-rules. Skipped tasks will cascade through trigger rules ``all_success``
-and ``all_failed`` but not ``all_done``, ``one_failed``, ``one_success``,
-and ``dummy``. If you would like to use the ``LatestOnlyOperator`` with
-trigger rules that do not cascade skips, you will need to ensure that the
-``LatestOnlyOperator`` is **directly** upstream of the task you would like
-to skip.
-
-It is possible, through use of trigger rules to mix tasks that should run
-in the typical date/time dependent mode and those using the
-``LatestOnlyOperator``.
+DAG. The ``LatestOnlyOperator`` skips all downstream tasks, if the time 
+right now is not between its ``execution_time`` and the next scheduled 
+``execution_time``.
 
 For example, consider the following dag:
 
@@ -651,8 +771,8 @@ For example, consider the following dag:
 
   dag = DAG(
       dag_id='latest_only_with_trigger',
-      schedule_interval=dt.timedelta(hours=4),
-      start_date=dt.datetime(2016, 9, 20),
+      schedule_interval=dt.timedelta(hours=1),
+      start_date=dt.datetime(2019, 2, 28),
   )
 
   latest_only = LatestOnlyOperator(task_id='latest_only', dag=dag)
@@ -676,9 +796,8 @@ for all runs except the latest run. ``task1`` is directly downstream of
 scheduled periods. ``task3`` is downstream of ``task1`` and ``task2`` and
 because of the default ``trigger_rule`` being ``all_success`` will receive
 a cascaded skip from ``task1``. ``task4`` is downstream of ``task1`` and
-``task2`` but since its ``trigger_rule`` is set to ``all_done`` it will
-trigger as soon as ``task1`` has been skipped (a valid completion state)
-and ``task2`` has succeeded.
+``task2``. It will be first skipped directly by ``LatestOnlyOperator``, 
+even its ``trigger_rule`` is set to ``all_done``.
 
 .. image:: img/latest_only_with_trigger.png
 
@@ -777,7 +896,7 @@ Jinja Templating
 
 Airflow leverages the power of
 `Jinja Templating <http://jinja.pocoo.org/docs/dev/>`_ and this can be a
-powerful tool to use in combination with macros (see the :ref:`macros` section).
+powerful tool to use in combination with macros (see the :doc:`macros` section).
 
 For example, say you want to pass the execution date as an environment variable
 to a Bash script using the ``BashOperator``.
@@ -849,3 +968,32 @@ do the same, but then it is more to use a virtualenv and pip.
    to be available on the system if a module needs those. In other words only
    pure python modules can be packaged.
 
+
+.airflowignore
+''''''''''''''
+
+A ``.airflowignore`` file specifies the directories or files in ``DAG_FOLDER``
+that Airflow should intentionally ignore. Each line in ``.airflowignore``
+specifies a regular expression pattern, and directories or files whose names
+(not DAG id) match any of the patterns would be ignored (under the hood,
+``re.findall()`` is used to match the pattern). Overall it works like a
+``.gitignore`` file.
+
+``.airflowignore`` file should be put in your ``DAG_FOLDER``.
+For example, you can prepare a ``.airflowignore`` file with contents
+
+.. code::
+
+    project_a
+    tenant_[\d]
+
+
+Then files like "project_a_dag_1.py", "TESTING_project_a.py", "tenant_1.py",
+"project_a/dag_1.py", and "tenant_1/dag_1.py" in your ``DAG_FOLDER`` would be ignored
+(If a directory's name matches any of the patterns, this directory and all its subfolders
+would not be scanned by Airflow at all. This improves efficiency of DAG finding).
+
+The scope of a ``.airflowignore`` file is the directory it is in plus all its subfolders.
+You can also prepare ``.airflowignore`` file for a subfolder in ``DAG_FOLDER`` and it
+would only be applicable for that subfolder.
+
diff --git a/docs/conf.py b/docs/conf.py
index 8b5d8305bbb15..dcc409764907e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,5 +1,25 @@
 # -*- coding: utf-8 -*-
 #
+# flake8: noqa
+# Disable Flake8 because of all the sphinx imports
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # Airflow documentation build configuration file, created by
 # sphinx-quickstart on Thu Oct  9 20:50:01 2014.
 #
@@ -13,40 +33,77 @@
 # serve to show the default.
 import os
 import sys
-import mock
 
-MOCK_MODULES = [
-    'apiclient',
-    'apiclient.discovery',
-    'apiclient.http',
-    'mesos',
-    'mesos.interface',
-    'mesos.native',
-    'google.auth.default',
+import airflow
+
+autodoc_mock_imports = [
+    'MySQLdb',
+    'adal',
+    'analytics',
+    'azure',
+    'azure.cosmos',
+    'azure.datalake',
+    'azure.mgmt',
+    'boto3',
+    'botocore',
+    'bson',
+    'cassandra',
+    'celery',
+    'cloudant',
+    'cx_Oracle',
+    'datadog',
+    'distributed',
+    'docker',
+    'google',
     'google_auth_httplib2',
-    'google.oauth2.service_account',
-    'pandas.io.gbq',
+    'googleapiclient',
+    'hdfs',
+    'httplib2',
+    'jaydebeapi',
+    'jenkins',
+    'jira',
+    'kubernetes',
+    'mesos',
+    'msrestazure',
+    'pandas',
+    'pandas_gbq',
+    'paramiko',
+    'pinotdb',
+    'psycopg2',
+    'pydruid',
+    'pyhive',
+    'pyhive',
+    'pymongo',
+    'pymssql',
+    'pysftp',
+    'qds_sdk',
+    'redis',
+    'simple_salesforce',
+    'slackclient',
+    'smbclient',
+    'snowflake',
+    'sshtunnel',
+    'tenacity',
     'vertica_python',
-    'pymssql'
+    'winrm',
+    'zdesk',
 ]
-for mod_name in MOCK_MODULES:
-    sys.modules[mod_name] = mock.Mock()
-
 
 # Hack to allow changing for piece of the code to behave differently while
 # the docs are being built. The main objective was to alter the
 # behavior of the utils.apply_default that was hiding function headers
 os.environ['BUILDING_AIRFLOW_DOCS'] = 'TRUE'
-from airflow import settings
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 
+sys.path.append(os.path.join(os.path.dirname(__file__), 'exts'))
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -56,6 +113,11 @@
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
     'sphinxarg.ext',
+    'sphinxcontrib.httpdomain',
+    'sphinx.ext.intersphinx',
+    'autoapi.extension',
+    'exampleinclude',
+    'docroles'
 ]
 
 autodoc_default_flags = ['show-inheritance', 'members']
@@ -63,68 +125,116 @@
 viewcode_import = True
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ['templates']
 
 # The suffix of source filenames.
 source_suffix = '.rst'
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
 master_doc = 'index'
 
 # General information about the project.
 project = u'Airflow'
-#copyright = u''
+# copyright = u''
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-#version = '1.0.0'
+# version = '1.0.0'
+version = airflow.__version__
 # The full version, including alpha/beta/rc tags.
-#release = '1.0.0'
+# release = '1.0.0'
+release = airflow.__version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = [
+    '_api/airflow/_vendor',
+    '_api/airflow/api',
+    '_api/airflow/bin',
+    '_api/airflow/config_templates',
+    '_api/airflow/configuration',
+    '_api/airflow/contrib/auth',
+    '_api/airflow/contrib/example_dags',
+    '_api/airflow/contrib/index.rst',
+    '_api/airflow/contrib/kubernetes',
+    '_api/airflow/contrib/task_runner',
+    '_api/airflow/contrib/utils',
+    '_api/airflow/dag',
+    '_api/airflow/default_login',
+    '_api/airflow/example_dags',
+    '_api/airflow/exceptions',
+    '_api/airflow/index.rst',
+    '_api/airflow/jobs',
+    '_api/airflow/lineage',
+    '_api/airflow/logging_config',
+    '_api/airflow/macros',
+    '_api/airflow/migrations',
+    '_api/airflow/plugins_manager',
+    '_api/airflow/security',
+    '_api/airflow/settings',
+    '_api/airflow/stats',
+    '_api/airflow/task',
+    '_api/airflow/ti_deps',
+    '_api/airflow/utils',
+    '_api/airflow/version',
+    '_api/airflow/www',
+    '_api/airflow/www_rbac',
+    '_api/main',
+    'autoapi_templates',
+    'howto/operator/gcp/_partials',
+]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
-
+keep_warnings = True
+
+
+intersphinx_mapping = {
+    'boto3': ('https://boto3.amazonaws.com/v1/documentation/api/latest/', None),
+    'google-cloud-python': (
+        'https://googleapis.github.io/google-cloud-python/latest/', None),
+    'mongodb': ('https://api.mongodb.com/python/current/', None),
+    'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None),
+    'python': ('https://docs.python.org/3/', None),
+    'requests': ('http://docs.python-requests.org/en/master/', None),
+    'sqlalchemy': ('https://docs.sqlalchemy.org/en/latest/', None),
+}
 
 # -- Options for HTML output ----------------------------------------------
 
@@ -135,11 +245,12 @@
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 import sphinx_rtd_theme
+
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # The name for this set of Sphinx documents.  If None, it defaults to
@@ -151,52 +262,52 @@
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
 html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
 html_show_copyright = False
@@ -204,55 +315,54 @@
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'Airflowdoc'
 
-
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
+    # The paper size ('letterpaper' or 'a4paper').
+    # 'papersize': 'letterpaper',
 
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
+    # The font size ('10pt', '11pt' or '12pt').
+    # 'pointsize': '10pt',
 
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # Additional stuff for the LaTeX preamble.
+    # 'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'Airflow.tex', u'Airflow Documentation',
-   u'Apache Airflow', 'manual'),
+    ('index', 'Airflow.tex', u'Airflow Documentation',
+     u'Apache Airflow', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
@@ -265,7 +375,7 @@
 ]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -278,16 +388,46 @@
     u'Apache Airflow', 'Airflow',
     'Airflow is a system to programmaticaly author, schedule and monitor data pipelines.',
     'Miscellaneous'
-),]
+), ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False
+
+# sphinx-autoapi configuration
+# See:
+# https://sphinx-autoapi.readthedocs.io/en/latest/config.html
+
+# Paths (relative or absolute) to the source code that you wish to generate
+# your API documentation from.
+autoapi_dirs = [
+    os.path.abspath('../airflow'),
+]
+
+# A directory that has user-defined templates to override our default templates.
+autoapi_template_dir = 'autoapi_templates'
+
+# A list of patterns to ignore when finding files
+autoapi_ignore = [
+    os.path.abspath('../airflow/contrib/operators/s3_to_gcs_transfer_operator.py'),
+    os.path.abspath('../airflow/contrib/operators/gcs_to_gcs_transfer_operator.py'),
+    os.path.abspath('../airflow/contrib/operators/gcs_to_gcs_transfer_operator.py'),
+]
+# Keep the AutoAPI generated files on the filesystem after the run.
+# Useful for debugging.
+autoapi_keep_files = False
+
+# Relative path to output the AutoAPI files into. This can also be used to place the generated documentation
+# anywhere in your documentation hierarchy.
+autoapi_root = '_api'
+
+# -- Options for examole include ------------------------------------------
+exampleinclude_sourceroot = os.path.abspath('..')
diff --git a/docs/exts/__init__.py b/docs/exts/__init__.py
new file mode 100644
index 0000000000000..645848f26c3b3
--- /dev/null
+++ b/docs/exts/__init__.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# flake8: noqa
+# Disable Flake8 because of all the sphinx imports
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/docs/exts/docroles.py b/docs/exts/docroles.py
new file mode 100644
index 0000000000000..00f88f1eba2b2
--- /dev/null
+++ b/docs/exts/docroles.py
@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from docutils import nodes, utils
+from sphinx.ext.autodoc.importer import import_module, mock
+from functools import partial
+
+
+class RoleException(Exception):
+    pass
+
+
+def get_template_field(env, fullname):
+    """
+    Gets template fields for specific operator class.
+
+    :param fullname: Full path to operator class.
+        For example: ``airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator``
+    :return: List of template field
+    :rtype: list[str]
+    """
+    modname, classname = fullname.rsplit(".", 1)
+
+    try:
+        with mock(env.config.autodoc_mock_imports):
+            mod = import_module(modname)
+    except ImportError:
+        raise RoleException("Error loading %s module." % (modname, ))
+
+    clazz = getattr(mod, classname)
+    if not clazz:
+        raise RoleException("Error finding %s class in %s module." % (classname, modname))
+
+    template_fields = getattr(clazz, "template_fields")
+
+    if not template_fields:
+        raise RoleException(
+            "Could not find the template fields for %s class in %s module." % (classname, modname)
+        )
+
+    return list(template_fields)
+
+
+def template_field_role(app, typ, rawtext, text, lineno, inliner, options={}, content=[]):
+    """
+    A role that allows you to include a list of template fields in the middle of the text. This is especially
+    useful when writing guides describing how to use the operator.
+    The result is a list of fields where each field is shorted in the literal block.
+
+    Sample usage::
+
+    :template-fields:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentimentOperator`
+
+    For further information look at:
+
+    * [http://docutils.sourceforge.net/docs/howto/rst-roles.html](Creating reStructuredText Interpreted
+      Text Roles)
+    """
+    text = utils.unescape(text)
+
+    try:
+        template_fields = get_template_field(app.env, text)
+    except RoleException as e:
+        msg = inliner.reporter.error("invalid class name %s \n%s" % (text, e, ), line=lineno)
+        prb = inliner.problematic(rawtext, rawtext, msg)
+        return [prb], [msg]
+
+    node = nodes.inline(rawtext=rawtext)
+    for i, field in enumerate(template_fields):
+        if i != 0:
+            node += nodes.Text(", ")
+        node += nodes.literal(field, "", nodes.Text(field))
+
+    return [node], []
+
+
+def setup(app):
+    from docutils.parsers.rst import roles
+    roles.register_local_role("template-fields", partial(template_field_role, app))
+
+    return {"version": "builtin", "parallel_read_safe": True, "parallel_write_safe": True}
diff --git a/docs/exts/exampleinclude.py b/docs/exts/exampleinclude.py
new file mode 100644
index 0000000000000..79eadd641da82
--- /dev/null
+++ b/docs/exts/exampleinclude.py
@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+#
+# flake8: noqa
+# Disable Flake8 because of all the sphinx imports
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from os import path
+
+from docutils import nodes
+from docutils.parsers.rst import directives
+from six import text_type
+from sphinx import addnodes
+from sphinx.directives.code import LiteralIncludeReader
+from sphinx.locale import _
+from sphinx.pycode import ModuleAnalyzer
+from sphinx.util import logging
+from sphinx.util import parselinenos
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import set_source_info
+
+logger = logging.getLogger(__name__)
+
+
+class example_header(nodes.reference, nodes.FixedTextElement):
+    pass
+
+
+class ExampleInclude(SphinxDirective):
+    """
+    Like ``.. literalinclude:: ``, but it does not support caption option.
+    Adds a header with a reference to  the full source code
+
+    Based on:
+    https://raw.githubusercontent.com/sphinx-doc/sphinx/v1.8.3/sphinx/directives/code.py
+    """
+
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {
+        "dedent": int,
+        "linenos": directives.flag,
+        "lineno-start": int,
+        "lineno-match": directives.flag,
+        "tab-width": int,
+        "language": directives.unchanged_required,
+        "encoding": directives.encoding,
+        "pyobject": directives.unchanged_required,
+        "lines": directives.unchanged_required,
+        "start-after": directives.unchanged_required,
+        "end-before": directives.unchanged_required,
+        "start-at": directives.unchanged_required,
+        "end-at": directives.unchanged_required,
+        "prepend": directives.unchanged_required,
+        "append": directives.unchanged_required,
+        "emphasize-lines": directives.unchanged_required,
+        "class": directives.class_option,
+        "name": directives.unchanged,
+        "diff": directives.unchanged_required,
+    }
+
+    def run(self):
+        document = self.state.document
+        if not document.settings.file_insertion_enabled:
+            return [document.reporter.warning("File insertion disabled", line=self.lineno)]
+        # convert options['diff'] to absolute path
+        if "diff" in self.options:
+            _, path = self.env.relfn2path(self.options["diff"])
+            self.options["diff"] = path
+
+        try:
+            location = self.state_machine.get_source_and_line(self.lineno)
+            rel_filename, filename = self.env.relfn2path(self.arguments[0])
+            self.env.note_dependency(rel_filename)
+
+            reader = LiteralIncludeReader(filename, self.options, self.config)
+            text, lines = reader.read(location=location)
+
+            retnode = nodes.literal_block(text, text, source=filename)
+            set_source_info(self, retnode)
+            if self.options.get("diff"):  # if diff is set, set udiff
+                retnode["language"] = "udiff"
+            elif "language" in self.options:
+                retnode["language"] = self.options["language"]
+            retnode["linenos"] = (
+                "linenos" in self.options or "lineno-start" in self.options or "lineno-match" in self.options
+            )
+            retnode["classes"] += self.options.get("class", [])
+            extra_args = retnode["highlight_args"] = {}
+            if "emphasize-lines" in self.options:
+                hl_lines = parselinenos(self.options["emphasize-lines"], lines)
+                if any(i >= lines for i in hl_lines):
+                    logger.warning(
+                        "line number spec is out of range(1-%d): %r", lines, self.options["emphasize-lines"]
+                    )
+                extra_args["hl_lines"] = [x + 1 for x in hl_lines if x < lines]
+            extra_args["linenostart"] = reader.lineno_start
+
+            container_node = nodes.container("", literal_block=True, classes=["example-block-wrapper"])
+            container_node += example_header(filename=filename)
+            container_node += retnode
+            retnode = container_node
+
+            return [retnode]
+        except Exception as exc:
+            return [document.reporter.warning(text_type(exc), line=self.lineno)]
+
+
+def register_source(app, env, modname):
+    entry = env._viewcode_modules.get(modname, None)  # type: ignore
+    if entry is False:
+        print("[%s] Entry is false for " % modname)
+        return
+
+    code_tags = app.emit_firstresult("viewcode-find-source", modname)
+    if code_tags is None:
+        try:
+            analyzer = ModuleAnalyzer.for_module(modname)
+        except Exception as ex:
+            logger.info("Module \"%s\" could not be loaded. Full source will not be available.", modname)
+            env._viewcode_modules[modname] = False  # type: ignore
+            return False
+
+        if not isinstance(analyzer.code, text_type):
+            code = analyzer.code.decode(analyzer.encoding)
+        else:
+            code = analyzer.code
+
+        analyzer.find_tags()
+        tags = analyzer.tags
+
+    else:
+        code, tags = code_tags
+
+    if entry is None or entry[0] != code:
+        # print("Registeted", entry)
+
+        entry = code, tags, {}, ""
+        env._viewcode_modules[modname] = entry  # type: ignore
+
+    return True
+
+
+def create_node(app, env, relative_path, show_button):
+    pagename = "_modules/" + relative_path[:-3]
+
+    header_classes = ["example-header"]
+    if show_button:
+        header_classes += ["example-header--with-button"]
+    paragraph = nodes.paragraph(relative_path, classes=header_classes)
+    paragraph += nodes.inline("", relative_path, classes=["example-title"])
+    if show_button:
+        pending_ref = addnodes.pending_xref(
+            "",
+            reftype="viewcode",
+            refdomain="std",
+            refexplicit=False,
+            reftarget=pagename,
+            refid="",
+            refdoc=env.docname,
+            classes=["example-header-button viewcode-button"],
+        )
+        pending_ref += nodes.inline("", _("View Source"))
+        paragraph += pending_ref
+
+    return paragraph
+
+
+def doctree_read(app, doctree):
+    env = app.builder.env
+    if not hasattr(env, "_viewcode_modules"):
+        env._viewcode_modules = {}  # type: ignore
+
+    if app.builder.name == "singlehtml":
+        return
+
+    for objnode in doctree.traverse(example_header):
+        filepath = objnode.get("filename")
+        relative_path = path.relpath(
+            filepath, path.commonprefix([app.config.exampleinclude_sourceroot, filepath])
+        )
+        modname = relative_path.replace("/", ".")[:-3]
+        show_button = register_source(app, env, modname)
+        onlynode = create_node(app, env, relative_path, show_button)
+
+        objnode.replace_self(onlynode)
+
+
+def setup(app):
+    directives.register_directive("exampleinclude", ExampleInclude)
+    app.connect("doctree-read", doctree_read)
+    app.add_config_value("exampleinclude_sourceroot", None, "env")
+
+    return {"version": "builtin", "parallel_read_safe": False, "parallel_write_safe": False}
diff --git a/docs/exts/removemarktransform.py b/docs/exts/removemarktransform.py
new file mode 100644
index 0000000000000..01ec440d0e112
--- /dev/null
+++ b/docs/exts/removemarktransform.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+#
+# flake8: noqa
+# Disable Flake8 because of all the sphinx imports
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+
+from docutils import nodes
+from pygments.lexers import guess_lexer, Python3Lexer, PythonLexer
+from sphinx.transforms import SphinxTransform
+from sphinx.transforms.post_transforms.code import TrimDoctestFlagsTransform
+
+docmark_re = re.compile(r"\s*#\s*\[(START|END)\s*[a-z_A-Z]+\].*$", re.MULTILINE)
+
+
+class TrimDocMarkerFlagsTransform(SphinxTransform):
+    """
+    Trim doc marker like ``# [START howto_concept]` from python code-blocks.
+
+    Based on:
+    https://github.com/sphinx-doc/sphinx/blob/master/sphinx/transforms/post_transforms/code.py
+    class TrimDoctestFlagsTransform
+    """
+
+    default_priority = TrimDoctestFlagsTransform.default_priority + 1
+
+    def apply(self, **kwargs):
+        for node in self.document.traverse(nodes.literal_block):
+            if self.is_pycode(node):
+                source = node.rawsource
+                source = docmark_re.sub("", source)
+                node.rawsource = source
+                node[:] = [nodes.Text(source)]
+
+    @staticmethod
+    def is_pycode(node):
+        # type: (nodes.literal_block) -> bool
+        if node.rawsource != node.astext():
+            return False  # skip parsed-literal node
+
+        language = node.get("language")
+        if language in ("py", "py3", "python", "python3", "default"):
+            return True
+        elif language == "guess":
+            try:
+                lexer = guess_lexer(node.rawsource)
+                return isinstance(lexer, PythonLexer) or isinstance(lexer, Python3Lexer)
+            except Exception:
+                pass
+
+        return False
+
+
+def setup(app):
+    app.add_post_transform(TrimDocMarkerFlagsTransform)
+
+    return {"version": "builtin", "parallel_read_safe": False, "parallel_write_safe": False}
diff --git a/docs/faq.rst b/docs/faq.rst
index 46212084c58fd..42dfb278f832a 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 FAQ
 ========
 
diff --git a/docs/howto/check-health.rst b/docs/howto/check-health.rst
new file mode 100644
index 0000000000000..ca0e63d8f4dc0
--- /dev/null
+++ b/docs/howto/check-health.rst
@@ -0,0 +1,49 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Checking Airflow Health Status
+==============================
+
+To check the health status of your Airflow instance, you can simply access the endpoint
+``/health``. It will return a JSON object in which a high-level glance is provided.
+
+.. code-block:: JSON
+
+  {
+    "metadatabase":{
+      "status":"healthy"
+    },
+    "scheduler":{
+      "status":"healthy",
+      "latest_scheduler_heartbeat":"2018-12-26 17:15:11+00:00"
+    }
+  }
+
+* The ``status`` of each component can be either "healthy" or "unhealthy"
+
+  * The status of ``metadatabase`` depends on whether a valid connection can be initiated with the database
+
+  * The status of ``scheduler`` depends on when the latest scheduler heartbeat was received
+
+    * If the last heartbeat was received more than 30 seconds (default value) earlier than the current time, the scheduler is
+      considered unhealthy
+    * This threshold value can be specified using the option ``scheduler_health_check_threshold`` within the
+      ``scheduler`` section in ``airflow.cfg``
+
+Please keep in mind that the HTTP response code of ``/health`` endpoint **should not** be used to determine the health
+status of the application. The return code is only indicative of the state of the rest call (200 for success).
+
diff --git a/docs/howto/connection/aws.rst b/docs/howto/connection/aws.rst
new file mode 100644
index 0000000000000..0cda3d6bdb6ca
--- /dev/null
+++ b/docs/howto/connection/aws.rst
@@ -0,0 +1,64 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Amazon Web Services Connection
+==============================
+
+The Amazon Web Services connection type enables the :ref:`AWS Integrations
+<AWS>`.
+
+Authenticating to AWS
+---------------------
+
+Authentication may be performed using any of the `boto3 options <https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials>`_. Alternatively, one can pass credentials in as a Connection initialisation parameter.
+
+To use IAM instance profile, create an "empty" connection (i.e. one with no Login or Password specified).
+
+Default Connection IDs
+-----------------------
+
+The default connection ID is ``aws_default``.
+
+Configuring the Connection
+--------------------------
+
+
+Login (optional)
+    Specify the AWS access key ID.
+
+Password (optional)
+    Specify the AWS secret access key.
+
+Extra (optional)
+    Specify the extra parameters (as json dictionary) that can be used in AWS
+    connection. The following parameters are supported:
+
+    * **aws_account_id**: AWS account ID for the connection
+    * **aws_iam_role**: AWS IAM role for the connection
+    * **external_id**: AWS external ID for the connection
+    * **host**: Endpoint URL for the connection
+    * **region_name**: AWS region for the connection
+    * **role_arn**: AWS role ARN for the connection
+
+    Example "extras" field:
+
+    .. code-block:: json
+
+       {
+          "aws_iam_role": "aws_iam_role_name",
+          "region_name": "ap-southeast-2"
+       }
diff --git a/docs/howto/connection/gcp.rst b/docs/howto/connection/gcp.rst
new file mode 100644
index 0000000000000..9f56bd4d35c75
--- /dev/null
+++ b/docs/howto/connection/gcp.rst
@@ -0,0 +1,117 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. _howto/connection:gcp:
+
+Google Cloud Platform Connection
+================================
+
+The Google Cloud Platform connection type enables the :ref:`GCP Integrations
+<GCP>`.
+
+Authenticating to GCP
+---------------------
+
+There are two ways to connect to GCP using Airflow.
+
+1. Use `Application Default Credentials
+   <https://google-auth.readthedocs.io/en/latest/reference/google.auth.html#google.auth.default>`_,
+   such as via the metadata server when running on Google Compute Engine.
+2. Use a `service account
+   <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
+   file (JSON format) on disk.
+
+Default Connection IDs
+----------------------
+
+The following connection IDs are used by default.
+
+``bigquery_default``
+    Used by the :class:`~airflow.contrib.hooks.bigquery_hook.BigQueryHook`
+    hook.
+
+``google_cloud_datastore_default``
+    Used by the :class:`~airflow.contrib.hooks.datastore_hook.DatastoreHook`
+    hook.
+
+``google_cloud_default``
+    Used by those hooks:
+
+    * :class:`~airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook`
+    * :class:`~airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook`
+    * :class:`~airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook`
+    * :class:`~airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook`
+    * :class:`~airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook`
+    * :class:`~airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook`
+    * :class:`~airflow.contrib.hooks.gcp_compute_hook.GceHook`
+    * :class:`~airflow.contrib.hooks.gcp_function_hook.GcfHook`
+    * :class:`~airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook`
+    * :class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook`
+
+
+Configuring the Connection
+--------------------------
+
+Project Id (optional)
+    The Google Cloud project ID to connect to. It is used as default project id by operators using it and
+    can usually be overridden at the operator level.
+
+Keyfile Path
+    Path to a `service account
+    <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
+    file (JSON format) on disk.
+
+    Not required if using application default credentials.
+
+Keyfile JSON
+    Contents of a `service account
+    <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
+    file (JSON format) on disk. It is recommended to :doc:`Secure your connections <../secure-connections>` if using this method to authenticate.
+
+    Not required if using application default credentials.
+
+Scopes (comma separated)
+    A list of comma-separated `Google Cloud scopes
+    <https://developers.google.com/identity/protocols/googlescopes>`_ to
+    authenticate with.
+
+    .. note::
+        Scopes are ignored when using application default credentials. See
+        issue `AIRFLOW-2522
+        <https://issues.apache.org/jira/browse/AIRFLOW-2522>`_.
+
+    When specifying the connection in environment variable you should specify
+    it using URI syntax, with the following requirements:
+
+      * scheme part should be equals ``google-cloud-platform`` (Note: look for a
+        hyphen character)
+      * authority (username, password, host, port), path is ignored
+      * query parameters contains information specific to this type of
+        connection. The following keys are accepted:
+
+        * ``extra__google_cloud_platform__project`` - Project Id
+        * ``extra__google_cloud_platform__key_path`` - Keyfile Path
+        * ``extra__google_cloud_platform__key_dict`` - Keyfile JSON
+        * ``extra__google_cloud_platform__scope`` - Scopes
+
+    Note that all components of the URI should be URL-encoded.
+
+    For example:
+
+    .. code-block:: bash
+
+       google-cloud-platform://?extra__google_cloud_platform__key_path=%2Fkeys%2Fkey.json&extra__google_cloud_platform__scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform&extra__google_cloud_platform__project=airflow
diff --git a/docs/howto/connection/gcp_sql.rst b/docs/howto/connection/gcp_sql.rst
new file mode 100644
index 0000000000000..6f19fd78f8051
--- /dev/null
+++ b/docs/howto/connection/gcp_sql.rst
@@ -0,0 +1,76 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud SQL Connection
+===========================
+
+The gcpcloudsql:// connection is used by
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator` to perform query
+on a Google Cloud SQL database. Google Cloud SQL database can be either
+Postgres or MySQL, so this is a "meta" connection type. It introduces common schema
+for both MySQL and Postgres, including what kind of connectivity should be used.
+Google Cloud SQL supports connecting via public IP or via Cloud SQL Proxy.
+In the latter case the
+:class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook` uses
+:class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlProxyRunner` to automatically prepare
+and use temporary Postgres or MySQL connection that will use the proxy to connect
+(either via TCP or UNIX socket.
+
+Configuring the Connection
+--------------------------
+
+Host (required)
+    The host to connect to.
+
+Schema (optional)
+    Specify the schema name to be used in the database.
+
+Login (required)
+    Specify the user name to connect.
+
+Password (required)
+    Specify the password to connect.
+
+Extra (optional)
+    Specify the extra parameters (as JSON dictionary) that can be used in Google Cloud SQL
+    connection.
+
+    Details of all the parameters supported in extra field can be found in
+    :class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook`
+
+    Example "extras" field:
+
+    .. code-block:: json
+
+       {
+          "database_type": "mysql",
+          "project_id": "example-project",
+          "location": "europe-west1",
+          "instance": "testinstance",
+          "use_proxy": true,
+          "sql_proxy_use_tcp": false
+       }
+
+    When specifying the connection as URI (in AIRFLOW_CONN_* variable), you should specify
+    it following the standard syntax of DB connection, where extras are passed as
+    parameters of the URI. Note that all components of the URI should be URL-encoded.
+
+    For example:
+
+    .. code-block:: bash
+
+        gcpcloudsql://user:XXXXXXXXX@1.1.1.1:3306/mydb?database_type=mysql&project_id=example-project&location=europe-west1&instance=testinstance&use_proxy=True&sql_proxy_use_tcp=False
diff --git a/docs/howto/connection/index.rst b/docs/howto/connection/index.rst
new file mode 100644
index 0000000000000..1f7a13a432cf2
--- /dev/null
+++ b/docs/howto/connection/index.rst
@@ -0,0 +1,92 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Managing Connections
+====================
+
+Airflow needs to know how to connect to your environment. Information
+such as hostname, port, login and passwords to other systems and services is
+handled in the ``Admin->Connections`` section of the UI. The pipeline code you
+will author will reference the 'conn_id' of the Connection objects.
+
+.. image:: ../../img/connections.png
+
+Connections can be created and managed using either the UI or environment
+variables.
+
+See the :ref:`Connections Concepts <concepts-connections>` documentation for
+more information.
+
+Creating a Connection with the UI
+---------------------------------
+
+Open the ``Admin->Connections`` section of the UI. Click the ``Create`` link
+to create a new connection.
+
+.. image:: ../../img/connection_create.png
+
+1. Fill in the ``Conn Id`` field with the desired connection ID. It is
+   recommended that you use lower-case characters and separate words with
+   underscores.
+2. Choose the connection type with the ``Conn Type`` field.
+3. Fill in the remaining fields. See
+   :ref:`manage-connections-connection-types` for a description of the fields
+   belonging to the different connection types.
+4. Click the ``Save`` button to create the connection.
+
+Editing a Connection with the UI
+--------------------------------
+
+Open the ``Admin->Connections`` section of the UI. Click the pencil icon next
+to the connection you wish to edit in the connection list.
+
+.. image:: ../../img/connection_edit.png
+
+Modify the connection properties and click the ``Save`` button to save your
+changes.
+
+Creating a Connection with Environment Variables
+------------------------------------------------
+
+Connections in Airflow pipelines can be created using environment variables.
+The environment variable needs to have a prefix of ``AIRFLOW_CONN_`` for
+Airflow with the value in a URI format to use the connection properly.
+
+When referencing the connection in the Airflow pipeline, the ``conn_id``
+should be the name of the variable without the prefix. For example, if the
+``conn_id`` is named ``postgres_master`` the environment variable should be
+named ``AIRFLOW_CONN_POSTGRES_MASTER`` (note that the environment variable
+must be all uppercase).
+
+Airflow assumes the value returned from the environment variable to be in a URI
+format (e.g.``postgres://user:password@localhost:5432/master`` or
+``s3://accesskey:secretkey@S3``). The underscore character is not allowed
+in the scheme part of URI, so it must be changed to a hyphen character
+(e.g. `google-compute-platform` if `conn_type` is `google_compute_platform`).
+Query parameters are parsed to one-dimensional dict and then used to fill extra.
+
+
+.. _manage-connections-connection-types:
+
+Connection Types
+----------------
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    *
diff --git a/docs/howto/connection/mysql.rst b/docs/howto/connection/mysql.rst
new file mode 100644
index 0000000000000..ab2162eb2c1b7
--- /dev/null
+++ b/docs/howto/connection/mysql.rst
@@ -0,0 +1,94 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+MySQL Connection
+================
+The MySQL connection type provides connection to a MySQL database.
+
+Configuring the Connection
+--------------------------
+Host (required)
+    The host to connect to.
+
+Schema (optional)
+    Specify the schema name to be used in the database.
+
+Login (required)
+    Specify the user name to connect.
+
+Password (required)
+    Specify the password to connect.
+
+Extra (optional)
+    Specify the extra parameters (as json dictionary) that can be used in MySQL
+    connection. The following parameters are supported:
+
+    * **charset**: specify charset of the connection
+    * **cursor**: one of "sscursor", "dictcursor, "ssdictcursor" . Specifies cursor class to be
+      used
+    * **local_infile**: controls MySQL's LOCAL capability (permitting local data loading by
+      clients). See `MySQLdb docs <https://mysqlclient.readthedocs.io/user_guide.html>`_
+      for details.
+    * **unix_socket**: UNIX socket used instead of the default socket.
+    * **ssl**: Dictionary of SSL parameters that control connecting using SSL. Those
+      parameters are server specific and should contain "ca", "cert", "key", "capath",
+      "cipher" parameters. See
+      `MySQLdb docs <https://mysqlclient.readthedocs.io/user_guide.html>`_ for details.
+      Note that to be useful in URL notation, this parameter might also be
+      a string where the SSL dictionary is a string-encoded JSON dictionary.
+
+    Example "extras" field:
+
+    .. code-block:: json
+
+       {
+          "charset": "utf8",
+          "cursorclass": "sscursor",
+          "local_infile": true,
+          "unix_socket": "/var/socket",
+          "ssl": {
+            "cert": "/tmp/client-cert.pem",
+            "ca": "/tmp/server-ca.pem'",
+            "key": "/tmp/client-key.pem"
+          }
+       }
+
+    or
+
+    .. code-block:: json
+
+       {
+          "charset": "utf8",
+          "cursorclass": "sscursor",
+          "local_infile": true,
+          "unix_socket": "/var/socket",
+          "ssl": "{\"cert\": \"/tmp/client-cert.pem\", \"ca\": \"/tmp/server-ca.pem\", \"key\": \"/tmp/client-key.pem\"}"
+       }
+
+    When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it
+    following the standard syntax of DB connections - where extras are passed as parameters
+    of the URI. Note that all components of the URI should be URL-encoded.
+
+    For example:
+
+    .. code-block:: bash
+
+       mysql://mysql_user:XXXXXXXXXXXX@1.1.1.1:3306/mysqldb?ssl=%7B%22cert%22%3A+%22%2Ftmp%2Fclient-cert.pem%22%2C+%22ca%22%3A+%22%2Ftmp%2Fserver-ca.pem%22%2C+%22key%22%3A+%22%2Ftmp%2Fclient-key.pem%22%7D
+
+    .. note::
+        If encounter UnicodeDecodeError while working with MySQL connection, check
+        the charset defined is matched to the database charset.
diff --git a/docs/howto/connection/oracle.rst b/docs/howto/connection/oracle.rst
new file mode 100644
index 0000000000000..6a3b23f88bc54
--- /dev/null
+++ b/docs/howto/connection/oracle.rst
@@ -0,0 +1,85 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Oracle Connection
+=================
+The Oracle connection type provides connection to a Oracle database.
+
+Configuring the Connection
+--------------------------
+Dsn (required)
+    The Data Source Name. The host address for the Oracle server.
+
+Sid (optional)
+    The Oracle System ID. The uniquely identify a particular database on a system.
+
+Service_name (optional)
+    The db_unique_name of the database.
+
+Port (optional)
+    The port for the Oracle server, Default 1521.
+
+Login (required)
+    Specify the user name to connect.
+
+Password (required)
+    Specify the password to connect.
+
+Extra (optional)
+    Specify the extra parameters (as json dictionary) that can be used in Oracle
+    connection. The following parameters are supported:
+
+    * **encoding** - The encoding to use for regular database strings. If not specified,
+      the environment variable `NLS_LANG` is used. If the environment variable `NLS_LANG`
+      is not set, `ASCII` is used.
+    * **nencoding** - The encoding to use for national character set database strings.
+      If not specified, the environment variable `NLS_NCHAR` is used. If the environment
+      variable `NLS_NCHAR` is not used, the environment variable `NLS_LANG` is used instead,
+      and if the environment variable `NLS_LANG` is not set, `ASCII` is used.
+    * **threaded** - Whether or not Oracle should wrap accesses to connections with a mutex.
+      Default value is False.
+    * **events** - Whether or not to initialize Oracle in events mode.
+    * **mode** - one of `sysdba`, `sysasm`, `sysoper`, `sysbkp`, `sysdgd`, `syskmt` or `sysrac`
+      which are defined at the module level, Default mode is connecting.
+    * **purity** - one of `new`, `self`, `default`. Specify the session acquired from the pool.
+      configuration parameter.
+
+    More details on all Oracle connect parameters supported can be found in
+    `cx_Oracle documentation <https://cx-oracle.readthedocs.io/en/latest/module.html#cx_Oracle.connect>`_.
+
+    Example "extras" field:
+
+    .. code-block:: json
+
+       {
+          "encoding": "UTF-8",
+          "nencoding": "UTF-8",
+          "threaded": false,
+          "events": false,
+          "mode": "sysdba",
+          "purity": "new"
+       }
+
+    When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it
+    following the standard syntax of DB connections, where extras are passed as parameters
+    of the URI (note that all components of the URI should be URL-encoded).
+
+    For example:
+
+    .. code-block:: bash
+
+        oracle://oracle_user:XXXXXXXXXXXX@1.1.1.1:1521?encoding=UTF-8&nencoding=UTF-8&threaded=False&events=False&mode=sysdba&purity=new
diff --git a/docs/howto/connection/postgres.rst b/docs/howto/connection/postgres.rst
new file mode 100644
index 0000000000000..2ca8b6df1814f
--- /dev/null
+++ b/docs/howto/connection/postgres.rst
@@ -0,0 +1,79 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+PostgresSQL Connection
+======================
+The Postgres connection type provides connection to a Postgres database.
+
+Configuring the Connection
+--------------------------
+Host (required)
+    The host to connect to.
+
+Schema (optional)
+    Specify the schema name to be used in the database.
+
+Login (required)
+    Specify the user name to connect.
+
+Password (required)
+    Specify the password to connect.
+
+Extra (optional)
+    Specify the extra parameters (as json dictionary) that can be used in postgres
+    connection. The following parameters out of the standard python parameters
+    are supported:
+
+    * **sslmode** - This option determines whether or with what priority a secure SSL
+      TCP/IP connection will be negotiated with the server. There are six modes:
+      'disable', 'allow', 'prefer', 'require', 'verify-ca', 'verify-full'.
+    * **sslcert** - This parameter specifies the file name of the client SSL certificate,
+      replacing the default.
+    * **sslkey** - This parameter specifies the file name of the client SSL key,
+      replacing the default.
+    * **sslrootcert** - This parameter specifies the name of a file containing SSL
+      certificate authority (CA) certificate(s).
+    * **sslcrl** - This parameter specifies the file name of the SSL certificate
+      revocation list (CRL).
+    * **application_name** - Specifies a value for the application_name
+      configuration parameter.
+    * **keepalives_idle** - Controls the number of seconds of inactivity after which TCP
+      should send a keepalive message to the server.
+
+    More details on all Postgres parameters supported can be found in
+    `Postgres documentation <https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING>`_.
+
+    Example "extras" field:
+
+    .. code-block:: json
+
+       {
+          "sslmode": "verify-ca",
+          "sslcert": "/tmp/client-cert.pem",
+          "sslca": "/tmp/server-ca.pem",
+          "sslkey": "/tmp/client-key.pem"
+       }
+
+    When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it
+    following the standard syntax of DB connections, where extras are passed as parameters
+    of the URI (note that all components of the URI should be URL-encoded).
+
+    For example:
+
+    .. code-block:: bash
+
+        postgresql://postgres_user:XXXXXXXXXXXX@1.1.1.1:5432/postgresdb?sslmode=verify-ca&sslcert=%2Ftmp%2Fclient-cert.pem&sslkey=%2Ftmp%2Fclient-key.pem&sslrootcert=%2Ftmp%2Fserver-ca.pem
diff --git a/docs/howto/connection/ssh.rst b/docs/howto/connection/ssh.rst
new file mode 100644
index 0000000000000..cd7c048ebd508
--- /dev/null
+++ b/docs/howto/connection/ssh.rst
@@ -0,0 +1,65 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+SSH Connection
+==============
+The SSH connection type provides connection to use :class:`~airflow.contrib.hooks.ssh_hook.SSHHook` to run commands on a remote server using :class:`~airflow.contrib.operators.ssh_operator.SSHOperator` or transfer file from/to the remote server using :class:`~airflow.contrib.operators.ssh_operator.SFTPOperator`.
+
+Configuring the Connection
+--------------------------
+Host (required)
+    The Remote host to connect.
+
+Username (optional)
+    The Username to connect to the remote_host.
+
+Password (optional)
+    Specify the password of the username to connect to the remote_host.
+
+Port (optional)
+    Port of remote host to connect. Default is 22.
+
+Extra (optional)
+    Specify the extra parameters (as json dictionary) that can be used in ssh
+    connection. The following parameters out of the standard python parameters
+    are supported:
+
+    * **timeout** - An optional timeout (in seconds) for the TCP connect. Default is ``10``.
+    * **compress** - ``true`` to ask the remote client/server to compress traffic; `false` to refuse compression. Default is ``true``.
+    * **no_host_key_check** - Set to ``false`` to restrict connecting to hosts with no entries in ``~/.ssh/known_hosts`` (Hosts file). This provides maximum protection against trojan horse attacks, but can be troublesome when the ``/etc/ssh/ssh_known_hosts`` file is poorly maintained or connections to new hosts are frequently made. This option forces the user to manually add all new hosts. Default is ``true``, ssh will automatically add new host keys to the user known hosts files.
+    * **allow_host_key_change** - Set to ``true`` if you want to allow connecting to hosts that has host key changed or when you get 'REMOTE HOST IDENTIFICATION HAS CHANGED' error.  This wont protect against Man-In-The-Middle attacks. Other possible solution is to remove the host entry from ``~/.ssh/known_hosts`` file. Default is ``false``.
+
+    Example "extras" field:
+
+    .. code-block:: json
+
+       {
+          "timeout": "10",
+          "compress": "false",
+          "no_host_key_check": "false",
+          "allow_host_key_change": "false"
+       }
+
+    When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it
+    following the standard syntax of connections, where extras are passed as parameters
+    of the URI (note that all components of the URI should be URL-encoded).
+
+    For example:
+
+    .. code-block:: bash
+
+        ssh://user:pass@localhost:22?timeout=10&compress=false&no_host_key_check=false&allow_host_key_change=true
diff --git a/docs/howto/executor/use-celery.rst b/docs/howto/executor/use-celery.rst
index fd6db96a95872..ff05334fdc028 100644
--- a/docs/howto/executor/use-celery.rst
+++ b/docs/howto/executor/use-celery.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Scaling Out with Celery
 =======================
 
@@ -40,8 +57,15 @@ Note that you can also run "Celery Flower", a web UI built on top of Celery,
 to monitor your workers. You can use the shortcut command ``airflow flower``
 to start a Flower web server.
 
+Please note that you must have the ``flower`` python library already installed on your system. The recommend way is to install the airflow celery bundle.
+
+.. code-block:: bash
+
+    pip install 'apache-airflow[celery]'
+
+
 Some caveats:
 
 - Make sure to use a database backed result backend
 - Make sure to set a visibility timeout in [celery_broker_transport_options] that exceeds the ETA of your longest running task
-- Tasks can and consume resources, make sure your worker as enough resources to run `worker_concurrency` tasks
+- Tasks can consume resources. Make sure your worker has enough resources to run `worker_concurrency` tasks
diff --git a/docs/howto/executor/use-dask.rst b/docs/howto/executor/use-dask.rst
index 769ce17e01a02..08e923af022a8 100644
--- a/docs/howto/executor/use-dask.rst
+++ b/docs/howto/executor/use-dask.rst
@@ -1,7 +1,24 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Scaling Out with Dask
 =====================
 
-``DaskExecutor`` allows you to run Airflow tasks in a Dask Distributed cluster.
+:class:`airflow.executors.dask_executor.DaskExecutor` allows you to run Airflow tasks in a Dask Distributed cluster.
 
 Dask clusters can be run on a single machine or on remote networks. For complete
 details, consult the `Distributed documentation <https://distributed.readthedocs.io/>`_.
@@ -22,7 +39,7 @@ Next start at least one Worker on any machine that can connect to the host:
 
     dask-worker $DASK_HOST:$DASK_PORT
 
-Edit your ``airflow.cfg`` to set your executor to ``DaskExecutor`` and provide
+Edit your ``airflow.cfg`` to set your executor to :class:`airflow.executors.dask_executor.DaskExecutor` and provide
 the Dask Scheduler address in the ``[dask]`` section.
 
 Please note:
diff --git a/docs/howto/executor/use-mesos.rst b/docs/howto/executor/use-mesos.rst
index c3bf95a63aa36..1cbe268298c93 100644
--- a/docs/howto/executor/use-mesos.rst
+++ b/docs/howto/executor/use-mesos.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Scaling Out with Mesos (community contributed)
 ==============================================
 
@@ -9,7 +26,7 @@ There are two ways you can run airflow as a mesos framework:
 Tasks executed directly on mesos slaves
 ---------------------------------------
 
-``MesosExecutor`` allows you to schedule airflow tasks on a Mesos cluster.
+:class:`airflow.contrib.executors.mesos_executor.MesosExecutor` allows you to schedule airflow tasks on a Mesos cluster.
 For this to work, you need a running mesos cluster and you must perform the following
 steps -
 
@@ -40,7 +57,7 @@ You can now see the airflow framework and corresponding tasks in mesos UI.
 The logs for airflow tasks can be seen in airflow UI as usual.
 
 For more information about mesos, refer to `mesos documentation <http://mesos.apache.org/documentation/latest/>`_.
-For any queries/bugs on `MesosExecutor`, please contact `@kapil-malik <https://github.com/kapil-malik>`_.
+For any queries/bugs on :class:`airflow.contrib.executors.mesos_executor.MesosExecutor`, please contact `@kapil-malik <https://github.com/kapil-malik>`_.
 
 Tasks executed in containers on mesos slaves
 --------------------------------------------
diff --git a/docs/howto/index.rst b/docs/howto/index.rst
index 1342ed85eb757..51746bde9af6e 100644
--- a/docs/howto/index.rst
+++ b/docs/howto/index.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 How-to Guides
 =============
 
@@ -12,12 +29,15 @@ configuring an Airflow environment.
 
     set-config
     initialize-database
-    manage-connections
+    operator/index
+    connection/index
     secure-connections
     write-logs
     executor/use-celery
     executor/use-dask
     executor/use-mesos
+    run-behind-proxy
     run-with-systemd
     run-with-upstart
     use-test-config
+    check-health
diff --git a/docs/howto/initialize-database.rst b/docs/howto/initialize-database.rst
index c85142f35b1c2..fad75be46d25a 100644
--- a/docs/howto/initialize-database.rst
+++ b/docs/howto/initialize-database.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Initializing a Database Backend
 ===============================
 
diff --git a/docs/howto/manage-connections.rst b/docs/howto/manage-connections.rst
deleted file mode 100644
index f5203157aca8d..0000000000000
--- a/docs/howto/manage-connections.rst
+++ /dev/null
@@ -1,135 +0,0 @@
-Managing Connections
-=====================
-
-Airflow needs to know how to connect to your environment. Information
-such as hostname, port, login and passwords to other systems and services is
-handled in the ``Admin->Connection`` section of the UI. The pipeline code you
-will author will reference the 'conn_id' of the Connection objects.
-
-.. image:: ../img/connections.png
-
-Connections can be created and managed using either the UI or environment
-variables.
-
-See the :ref:`Connenctions Concepts <concepts-connections>` documentation for
-more information.
-
-Creating a Connection with the UI
----------------------------------
-
-Open the ``Admin->Connection`` section of the UI. Click the ``Create`` link
-to create a new connection.
-
-.. image:: ../img/connection_create.png
-
-1. Fill in the ``Conn Id`` field with the desired connection ID. It is
-   recommended that you use lower-case characters and separate words with
-   underscores.
-2. Choose the connection type with the ``Conn Type`` field.
-3. Fill in the remaining fields. See
-   :ref:`manage-connections-connection-types` for a description of the fields
-   belonging to the different connection types.
-4. Click the ``Save`` button to create the connection.
-
-Editing a Connection with the UI
---------------------------------
-
-Open the ``Admin->Connection`` section of the UI. Click the pencil icon next
-to the connection you wish to edit in the connection list.
-
-.. image:: ../img/connection_edit.png
-
-Modify the connection properties and click the ``Save`` button to save your
-changes.
-
-Creating a Connection with Environment Variables
-------------------------------------------------
-
-Connections in Airflow pipelines can be created using environment variables.
-The environment variable needs to have a prefix of ``AIRFLOW_CONN_`` for
-Airflow with the value in a URI format to use the connection properly.
-
-When referencing the connection in the Airflow pipeline, the ``conn_id``
-should be the name of the variable without the prefix. For example, if the
-``conn_id`` is named ``postgres_master`` the environment variable should be
-named ``AIRFLOW_CONN_POSTGRES_MASTER`` (note that the environment variable
-must be all uppercase). Airflow assumes the value returned from the
-environment variable to be in a URI format (e.g.
-``postgres://user:password@localhost:5432/master`` or
-``s3://accesskey:secretkey@S3``).
-
-.. _manage-connections-connection-types:
-
-Connection Types
-----------------
-
-.. _connection-type-GCP:
-
-Google Cloud Platform
-~~~~~~~~~~~~~~~~~~~~~
-
-The Google Cloud Platform connection type enables the :ref:`GCP Integrations
-<GCP>`.
-
-Authenticating to GCP
-'''''''''''''''''''''
-
-There are two ways to connect to GCP using Airflow.
-
-1. Use `Application Default Credentials
-   <https://google-auth.readthedocs.io/en/latest/reference/google.auth.html#google.auth.default>`_,
-   such as via the metadata server when running on Google Compute Engine.
-2. Use a `service account
-   <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
-   file (JSON format) on disk.
-
-Default Connection IDs
-''''''''''''''''''''''
-
-The following connection IDs are used by default.
-
-``bigquery_default``
-    Used by the :class:`~airflow.contrib.hooks.bigquery_hook.BigQueryHook`
-    hook.
-
-``google_cloud_datastore_default``
-    Used by the :class:`~airflow.contrib.hooks.datastore_hook.DatastoreHook`
-    hook.
-
-``google_cloud_default``
-    Used by the
-    :class:`~airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook`,
-    :class:`~airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook`,
-    :class:`~airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook`,
-    :class:`~airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook`, and
-    :class:`~airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook` hooks.
-
-Configuring the Connection
-''''''''''''''''''''''''''
-
-Project Id (required)
-    The Google Cloud project ID to connect to.
-
-Keyfile Path
-    Path to a `service account
-    <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
-    file (JSON format) on disk.
-
-    Not required if using application default credentials.
-
-Keyfile JSON
-    Contents of a `service account
-    <https://cloud.google.com/docs/authentication/#service_accounts>`_ key
-    file (JSON format) on disk. It is recommended to :doc:`Secure your connections <secure-connections>` if using this method to authenticate.
-
-    Not required if using application default credentials.
-
-Scopes (comma separated)
-    A list of comma-separated `Google Cloud scopes
-    <https://developers.google.com/identity/protocols/googlescopes>`_ to
-    authenticate with.
-
-    .. note::
-        Scopes are ignored when using application default credentials. See
-        issue `AIRFLOW-2522
-        <https://issues.apache.org/jira/browse/AIRFLOW-2522>`_.
diff --git a/docs/howto/operator/bash.rst b/docs/howto/operator/bash.rst
new file mode 100644
index 0000000000000..b6a075af7edb7
--- /dev/null
+++ b/docs/howto/operator/bash.rst
@@ -0,0 +1,62 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. _howto/operator:BashOperator:
+
+BashOperator
+============
+
+Use the :class:`~airflow.operators.bash_operator.BashOperator` to execute
+commands in a `Bash <https://www.gnu.org/software/bash/>`__ shell.
+
+.. exampleinclude:: ../../../airflow/example_dags/example_bash_operator.py
+    :language: python
+    :start-after: [START howto_operator_bash]
+    :end-before: [END howto_operator_bash]
+
+Templating
+----------
+
+You can use :ref:`Jinja templates <jinja-templating>` to parameterize the
+``bash_command`` argument.
+
+.. exampleinclude:: ../../../airflow/example_dags/example_bash_operator.py
+    :language: python
+    :start-after: [START howto_operator_bash_template]
+    :end-before: [END howto_operator_bash_template]
+
+Troubleshooting
+---------------
+
+Jinja template not found
+""""""""""""""""""""""""
+
+Add a space after the script name when directly calling a Bash script with
+the ``bash_command`` argument. This is because Airflow tries to apply a Jinja
+template to it, which will fail.
+
+.. code-block:: python
+
+    t2 = BashOperator(
+        task_id='bash_example',
+
+        # This fails with `Jinja template not found` error
+        # bash_command="/home/batcher/test.sh",
+
+        # This works (has a space after)
+        bash_command="/home/batcher/test.sh ",
+        dag=dag)
diff --git a/docs/howto/operator/dingding.rst b/docs/howto/operator/dingding.rst
new file mode 100644
index 0000000000000..51bbd95789501
--- /dev/null
+++ b/docs/howto/operator/dingding.rst
@@ -0,0 +1,91 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Dingding Operators
+==================
+
+
+Prerequisite Tasks
+^^^^^^^^^^^^^^^^^^
+
+To use this operators, you must do a few things:
+
+  * Add custom robot to Dingding group which you want to send Dingding message.
+  * Get the webhook token from Dingding custom robot.
+  * Put the Dingding custom robot token in the password field of the ``dingding_default``
+    Connection. Notice that you just need token rather than the whole webhook string.
+
+Basic Usage
+^^^^^^^^^^^
+
+Use the :class:`~airflow/contrib/operators/dingding_operator.DingdingOperator`
+to send Dingding message:
+
+.. literalinclude:: ../../../airflow/contrib/example_dags/example_dingding_operator.py
+    :language: python
+    :start-after: [START howto_operator_dingding]
+    :end-before: [END howto_operator_dingding]
+
+
+Remind users in message
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Use parameters ``at_mobiles`` and ``at_all`` to remind specific users when you send message,
+``at_mobiles`` will be ignored When ``at_all`` is set to ``True``:
+
+.. literalinclude:: ../../../airflow/contrib/example_dags/example_dingding_operator.py
+    :language: python
+    :start-after: [START howto_operator_dingding_remind_users]
+    :end-before: [END howto_operator_dingding_remind_users]
+
+
+Send rich text message
+^^^^^^^^^^^^^^^^^^^^^^
+
+The Dingding operator can send rich text messages including link, markdown, actionCard and feedCard.
+A rich text message can not remind specific users except by using markdown type message:
+
+.. literalinclude:: ../../../airflow/contrib/example_dags/example_dingding_operator.py
+    :language: python
+    :start-after: [START howto_operator_dingding_rich_text]
+    :end-before: [END howto_operator_dingding_rich_text]
+
+
+Sending messages from a Task callback
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Dingding operator could handle task callback by writing a function wrapper dingding operators
+and then pass the function to ``sla_miss_callback``, ``on_success_callback``, ``on_failure_callback``,
+or ``on_retry_callback``. Here we use ``on_failure_callback`` as an example:
+
+.. literalinclude:: ../../../airflow/contrib/example_dags/example_dingding_operator.py
+    :language: python
+    :start-after: [START howto_operator_dingding_failure_callback]
+    :end-before: [END howto_operator_dingding_failure_callback]
+
+
+Changing connection host if you need
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The Dingding operator post http requests using default host ``https://oapi.dingtalk.com``,
+if you need to change the host used you can set the host field of the connection.
+
+
+More information
+^^^^^^^^^^^^^^^^
+
+See Dingding documentation on how to `custom robot
+<https://open-doc.dingtalk.com/microapp/serverapi2/qf2nxq>`_.
diff --git a/docs/howto/operator/gcp/_partials/prerequisite_tasks.rst b/docs/howto/operator/gcp/_partials/prerequisite_tasks.rst
new file mode 100644
index 0000000000000..7af9287da4ac5
--- /dev/null
+++ b/docs/howto/operator/gcp/_partials/prerequisite_tasks.rst
@@ -0,0 +1,31 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+To use these operators, you must do a few things:
+
+  * Select or create a Cloud Platform project using `Cloud Console <https://console.cloud.google.com/project>`__.
+  * Enable billing for your project, as described in `Google Cloud documentation <https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project>`__.
+  * Enable API, as described in `Cloud Console documentation <https://cloud.google.com/apis/docs/enable-disable-apis>`__.
+  * Install API libraries via **pip**.
+
+    .. code-block:: bash
+
+      pip install 'apache-airflow[gcp]'
+
+    Detailed information is available :doc:`../../../../installation`
+
+  * :doc:`Setup Connection <../../connection/gcp>`.
diff --git a/docs/howto/operator/gcp/bigtable.rst b/docs/howto/operator/gcp/bigtable.rst
new file mode 100644
index 0000000000000..3d144a2cec9d6
--- /dev/null
+++ b/docs/howto/operator/gcp/bigtable.rst
@@ -0,0 +1,170 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Bigtable Operators
+===============================
+
+.. contents::
+  :depth: 1
+  :local:
+
+All examples below rely on the following variables, which can be passed via environment variables.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :start-after: [START howto_operator_gcp_bigtable_args]
+    :end-before: [END howto_operator_gcp_bigtable_args]
+
+.. _howto/operator:BigtableInstanceCreateOperator:
+
+BigtableInstanceCreateOperator
+------------------------------
+
+Use the :class:`~airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceCreateOperator`
+to create a Google Cloud Bigtable instance.
+
+If the Cloud Bigtable instance with the given ID exists, the operator does not compare its configuration
+and immediately succeeds. No changes are made to the existing instance.
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_bigtable_instance_create]
+    :end-before: [END howto_operator_gcp_bigtable_instance_create]
+
+.. _howto/operator:BigtableInstanceDeleteOperator:
+
+BigtableInstanceDeleteOperator
+------------------------------
+
+Use the :class:`~airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceDeleteOperator`
+to delete a Google Cloud Bigtable instance.
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_bigtable_instance_delete]
+    :end-before: [END howto_operator_gcp_bigtable_instance_delete]
+
+.. _howto/operator:BigtableClusterUpdateOperator:
+
+BigtableClusterUpdateOperator
+-----------------------------
+
+Use the :class:`~airflow.contrib.operators.gcp_bigtable_operator.BigtableClusterUpdateOperator`
+to modify number of nodes in a Cloud Bigtable cluster.
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_bigtable_cluster_update]
+    :end-before: [END howto_operator_gcp_bigtable_cluster_update]
+
+.. _howto/operator:BigtableTableCreateOperator:
+
+BigtableTableCreateOperator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Creates a table in a Cloud Bigtable instance.
+
+If the table with given ID exists in the Cloud Bigtable instance, the operator compares the Column Families.
+If the Column Families are identical operator succeeds. Otherwise, the operator fails with the appropriate
+error message.
+
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_bigtable_table_create]
+    :end-before: [END howto_operator_gcp_bigtable_table_create]
+
+Advanced
+""""""""
+
+When creating a table, you can specify the optional ``initial_split_keys`` and ``column_families``.
+Please refer to the Python Client for Google Cloud Bigtable documentation
+`for Table <https://googleapis.github.io/google-cloud-python/latest/bigtable/table.html>`_ and `for Column
+Families <https://googleapis.github.io/google-cloud-python/latest/bigtable/column-family.html>`_.
+
+.. _howto/operator:BigtableTableDeleteOperator:
+
+BigtableTableDeleteOperator
+---------------------------
+
+Use the :class:`~airflow.contrib.operators.gcp_bigtable_operator.BigtableTableDeleteOperator`
+to delete a table in Google Cloud Bigtable.
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_bigtable_table_delete]
+    :end-before: [END howto_operator_gcp_bigtable_table_delete]
+
+.. _howto/operator:BigtableTableWaitForReplicationSensor:
+
+BigtableTableWaitForReplicationSensor
+-------------------------------------
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+Use the :class:`~airflow.contrib.operators.gcp_bigtable_operator.BigtableTableWaitForReplicationSensor`
+to wait for the table to replicate fully.
+
+The same arguments apply to this sensor as the BigtableTableCreateOperator_.
+
+**Note:** If the table or the Cloud Bigtable instance does not exist, this sensor waits for the table until
+timeout hits and does not raise any exception.
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_bigtable_operators.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_bigtable_table_wait_for_replication]
+    :end-before: [END howto_operator_gcp_bigtable_table_wait_for_replication]
diff --git a/docs/howto/operator/gcp/compute.rst b/docs/howto/operator/gcp/compute.rst
new file mode 100644
index 0000000000000..1d773ad4ff3c5
--- /dev/null
+++ b/docs/howto/operator/gcp/compute.rst
@@ -0,0 +1,325 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Compute Engine Operators
+===============================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:GceInstanceStartOperator:
+
+GceInstanceStartOperator
+------------------------
+
+Use the
+:class:`~airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator`
+to start an existing Google Compute Engine instance.
+
+
+Arguments
+"""""""""
+
+The following examples of OS environment variables used to pass arguments to the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :start-after: [START howto_operator_gce_args_common]
+    :end-before: [END howto_operator_gce_args_common]
+
+Using the operator
+""""""""""""""""""
+
+The code to create the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_start]
+    :end-before: [END howto_operator_gce_start]
+
+You can also create the operator without project id - project id will be retrieved
+from the GCP connection id used:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_start_no_project_id]
+    :end-before: [END howto_operator_gce_start_no_project_id]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_compute_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gce_instance_start_template_fields]
+    :end-before: [END gce_instance_start_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Compute Engine API documentation to `start an instance
+<https://cloud.google.com/compute/docs/reference/rest/v1/instances/start>`_.
+
+.. _howto/operator:GceInstanceStopOperator:
+
+GceInstanceStopOperator
+-----------------------
+
+Use the operator to stop Google Compute Engine instance.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator`
+
+Arguments
+"""""""""
+
+The following examples of OS environment variables used to pass arguments to the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+   :language: python
+   :start-after: [START howto_operator_gce_args_common]
+   :end-before: [END howto_operator_gce_args_common]
+
+Using the operator
+""""""""""""""""""
+
+The code to create the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_stop]
+    :end-before: [END howto_operator_gce_stop]
+
+You can also create the operator without project id - project id will be retrieved
+from the GCP connection used:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_stop_no_project_id]
+    :end-before: [END howto_operator_gce_stop_no_project_id]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_compute_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gce_instance_stop_template_fields]
+    :end-before: [END gce_instance_stop_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Compute Engine API documentation to `stop an instance
+<https://cloud.google.com/compute/docs/reference/rest/v1/instances/stop>`_.
+
+.. _howto/operator:GceSetMachineTypeOperator:
+
+GceSetMachineTypeOperator
+-------------------------
+
+Use the operator to change machine type of a Google Compute Engine instance.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator`.
+
+Arguments
+"""""""""
+
+The following examples of OS environment variables used to pass arguments to the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :start-after: [START howto_operator_gce_args_common]
+    :end-before: [END howto_operator_gce_args_common]
+
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :start-after: [START howto_operator_gce_args_set_machine_type]
+    :end-before: [END howto_operator_gce_args_set_machine_type]
+
+Using the operator
+""""""""""""""""""
+
+The code to create the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_set_machine_type]
+    :end-before: [END howto_operator_gce_set_machine_type]
+
+You can also create the operator without project id - project id will be retrieved
+from the GCP connection used:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_set_machine_type_no_project_id]
+    :end-before: [END howto_operator_gce_set_machine_type_no_project_id]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_compute_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gce_instance_set_machine_type_template_fields]
+    :end-before: [END gce_instance_set_machine_type_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Compute Engine API documentation to `set the machine type
+<https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType>`_.
+
+.. _howto/operator:GceInstanceTemplateCopyOperator:
+
+GceInstanceTemplateCopyOperator
+-------------------------------
+
+Use the operator to copy an existing Google Compute Engine instance template
+applying a patch to it.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_compute_operator.GceInstanceTemplateCopyOperator`.
+
+Arguments
+"""""""""
+
+The following examples of OS environment variables used to pass arguments to the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :start-after: [START howto_operator_compute_igm_common_args]
+    :end-before: [END howto_operator_compute_igm_common_args]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :start-after: [START howto_operator_compute_template_copy_args]
+    :end-before: [END howto_operator_compute_template_copy_args]
+
+Using the operator
+""""""""""""""""""
+
+The code to create the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_igm_copy_template]
+    :end-before: [END howto_operator_gce_igm_copy_template]
+
+You can also create the operator without project id - project id will be retrieved
+from the GCP connection used:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_igm_copy_template_no_project_id]
+    :end-before: [END howto_operator_gce_igm_copy_template_no_project_id]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_compute_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gce_instance_template_copy_operator_template_fields]
+    :end-before: [END gce_instance_template_copy_operator_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Compute Engine API documentation to `create a new instance with an existing template
+<https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates>`_.
+
+.. _howto/operator:GceInstanceGroupManagerUpdateTemplateOperator:
+
+GceInstanceGroupManagerUpdateTemplateOperator
+---------------------------------------------
+
+Use the operator to update a template in Google Compute Engine Instance Group Manager.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_compute_operator.GceInstanceGroupManagerUpdateTemplateOperator`.
+
+Arguments
+"""""""""
+
+The following examples of OS environment variables used to pass arguments to the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :start-after: [START howto_operator_compute_igm_common_args]
+    :end-before: [END howto_operator_compute_igm_common_args]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :start-after: [START howto_operator_compute_igm_update_template_args]
+    :end-before: [END howto_operator_compute_igm_update_template_args]
+
+Using the operator
+""""""""""""""""""
+
+The code to create the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_igm_update_template]
+    :end-before: [END howto_operator_gce_igm_update_template]
+
+You can also create the operator without project id - project id will be retrieved
+from the GCP connection used:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_compute_igm.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gce_igm_update_template_no_project_id]
+    :end-before: [END howto_operator_gce_igm_update_template_no_project_id]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_compute_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gce_igm_update_template_operator_template_fields]
+    :end-before: [END gce_igm_update_template_operator_template_fields]
+
+Troubleshooting
+"""""""""""""""
+
+You might find that your GceInstanceGroupManagerUpdateTemplateOperator fails with
+missing permissions. To execute the operation, the service account requires
+the permissions that theService Account User role provides
+(assigned via Google Cloud IAM).
+
+More information
+""""""""""""""""
+
+See Google Compute Engine API documentation to `manage a group instance
+<https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers>`_.
diff --git a/docs/howto/operator/gcp/function.rst b/docs/howto/operator/gcp/function.rst
new file mode 100644
index 0000000000000..2d2af271e869b
--- /dev/null
+++ b/docs/howto/operator/gcp/function.rst
@@ -0,0 +1,202 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Functions Operators
+================================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:GcfFunctionDeleteOperator:
+
+GcfFunctionDeleteOperator
+-------------------------
+
+Use the operator to delete a function from Google Cloud Functions.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator`.
+
+Arguments
+"""""""""
+
+The following examples of OS environment variables show how you can build function name
+to use in the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :start-after: [START howto_operator_gcf_common_variables]
+    :end-before: [END howto_operator_gcf_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcf_delete]
+    :end-before: [END howto_operator_gcf_delete]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_function_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcf_function_delete_template_fields]
+    :end-before: [END gcf_function_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Functions API documentation to `delete a function
+<https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/delete>`_.
+
+.. _howto/operator:GcfFunctionDeployOperator:
+
+GcfFunctionDeployOperator
+-------------------------
+
+Use the operator to deploy a function to Google Cloud Functions.
+If a function with this name already exists, it will be updated.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator`.
+
+
+Arguments
+"""""""""
+
+In the example DAG the following environment variables are used to parameterize the
+operator's definition:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :start-after: [START howto_operator_gcf_common_variables]
+    :end-before: [END howto_operator_gcf_common_variables]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :start-after: [START howto_operator_gcf_deploy_variables]
+    :end-before: [END howto_operator_gcf_deploy_variables]
+
+Some of those variables are used to create the request's body:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :start-after: [START howto_operator_gcf_deploy_body]
+    :end-before: [END howto_operator_gcf_deploy_body]
+
+When a DAG is created, the default_args dictionary can be used to pass
+arguments common with other tasks:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :start-after: [START howto_operator_gcf_default_args]
+    :end-before: [END howto_operator_gcf_default_args]
+
+Note that the neither the body nor the default args are complete in the above examples.
+Depending on the variables set, there might be different variants on how to pass source
+code related fields. Currently, you can pass either ``sourceArchiveUrl``,
+``sourceRepository`` or ``sourceUploadUrl`` as described in the
+`Cloud Functions API specification
+<https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#CloudFunction>`_.
+
+Additionally, ``default_args`` or direct operator args might contain ``zip_path``
+parameter
+to run the extra step of uploading the source code before deploying it.
+In this case, you also need to provide an empty ``sourceUploadUrl``
+parameter in the body.
+
+Using the operator
+""""""""""""""""""
+
+Depending on the combination of parameters, the Function's source code can be obtained
+from different sources:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :start-after: [START howto_operator_gcf_deploy_variants]
+    :end-before: [END howto_operator_gcf_deploy_variants]
+
+The code to create the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcf_deploy]
+    :end-before: [END howto_operator_gcf_deploy]
+
+You can also create the operator without project id - project id will be retrieved
+from the GCP connection used:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_function.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcf_deploy_no_project_id]
+    :end-before: [END howto_operator_gcf_deploy_no_project_id]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_function_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcf_function_deploy_template_fields]
+    :end-before: [END gcf_function_deploy_template_fields]
+
+
+Troubleshooting
+"""""""""""""""
+
+If during the deploy you see an error similar to:
+
+`"HttpError 403: Missing necessary permission iam.serviceAccounts.actAs for on resource
+project-name@appspot.gserviceaccount.com. Please grant the
+roles/iam.serviceAccountUser role."`
+
+it means that your service account does not have the correct Cloud IAM permissions.
+
+1. Assign your Service Account the Cloud Functions Developer role.
+2. Grant the user the Cloud IAM Service Account User role on the Cloud Functions runtime
+   service account.
+
+The typical way of assigning Cloud IAM permissions with `gcloud` is
+shown below. Just replace PROJECT_ID with ID of your Google Cloud Platform project
+and SERVICE_ACCOUNT_EMAIL with the email ID of your service account.
+
+.. code-block:: bash
+
+  gcloud iam service-accounts add-iam-policy-binding \
+    PROJECT_ID@appspot.gserviceaccount.com \
+    --member="serviceAccount:[SERVICE_ACCOUNT_EMAIL]" \
+    --role="roles/iam.serviceAccountUser"
+
+You can also do that via the GCP Web console.
+
+See `Adding the IAM service agent user role to the runtime service <https://cloud.google.com/functions/docs/reference/iam/roles#adding_the_iam_service_agent_user_role_to_the_runtime_service_account>`_  for details.
+
+If the source code for your function is in Google Source Repository, make sure that
+your service account has the Source Repository Viewer role so that the source code
+can be downloaded if necessary.
+
+More information
+""""""""""""""""
+
+See Google Cloud API documentation `to create a function
+<https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/create>`_.
diff --git a/docs/howto/operator/gcp/gcs.rst b/docs/howto/operator/gcp/gcs.rst
new file mode 100644
index 0000000000000..a5a41c9f6545c
--- /dev/null
+++ b/docs/howto/operator/gcp/gcs.rst
@@ -0,0 +1,127 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Storage Operators
+==============================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:GoogleCloudStorageToBigQueryOperator:
+
+GoogleCloudStorageToBigQueryOperator
+------------------------------------
+
+Use the
+:class:`~airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator`
+to execute a BigQuery load job.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcs_to_bq_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcs_to_bq]
+    :end-before: [END howto_operator_gcs_to_bq]
+
+
+.. _howto/operator:GoogleCloudStorageBucketCreateAclEntryOperator:
+
+GoogleCloudStorageBucketCreateAclEntryOperator
+----------------------------------------------
+
+Creates a new ACL entry on the specified bucket.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcs_acl.py
+    :language: python
+    :start-after: [START howto_operator_gcs_acl_args_common]
+    :end-before: [END howto_operator_gcs_acl_args_common]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcs_acl.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcs_bucket_create_acl_entry_task]
+    :end-before: [END howto_operator_gcs_bucket_create_acl_entry_task]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcs_acl_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcs_bucket_create_acl_template_fields]
+    :end-before: [END gcs_bucket_create_acl_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Storage Documentation to `create a new ACL entry for a bucket
+<https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert>`_.
+
+.. _howto/operator:GoogleCloudStorageObjectCreateAclEntryOperator:
+
+GoogleCloudStorageObjectCreateAclEntryOperator
+----------------------------------------------
+
+Creates a new ACL entry on the specified object.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcs_acl.py
+    :language: python
+    :start-after: [START howto_operator_gcs_acl_args_common]
+    :end-before: [END howto_operator_gcs_acl_args_common]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcs_acl.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcs_object_create_acl_entry_task]
+    :end-before: [END howto_operator_gcs_object_create_acl_entry_task]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcs_acl_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcs_object_create_acl_template_fields]
+    :end-before: [END gcs_object_create_acl_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Storage insert documentation to `create a ACL entry for ObjectAccess
+<https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert>`_.
diff --git a/docs/howto/operator/gcp/index.rst b/docs/howto/operator/gcp/index.rst
new file mode 100644
index 0000000000000..bee1c8de0227b
--- /dev/null
+++ b/docs/howto/operator/gcp/index.rst
@@ -0,0 +1,26 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Operators
+======================
+
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    *
diff --git a/docs/howto/operator/gcp/natural_language.rst b/docs/howto/operator/gcp/natural_language.rst
new file mode 100644
index 0000000000000..06ccfadf78203
--- /dev/null
+++ b/docs/howto/operator/gcp/natural_language.rst
@@ -0,0 +1,180 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Natural Language Operators
+=======================================
+
+The `Google Cloud Natural Language <https://cloud.google.com/natural-language/>`__
+can be used to reveal the structure and meaning of text via powerful machine
+learning models. You can use it to extract information about people, places,
+events and much more, mentioned in text documents, news articles or blog posts.
+You can use it to understand sentiment about your product on social media or
+parse intent from customer conversations happening in a call center or a
+messaging app.
+
+.. contents::
+  :depth: 1
+  :local:
+
+Prerequisite Tasks
+^^^^^^^^^^^^^^^^^^
+
+.. include:: _partials/prerequisite_tasks.rst
+
+
+.. _howto/operator:CloudNaturalLanguageDocuments:
+
+Documents
+^^^^^^^^^
+
+Each operator uses a :class:`~google.cloud.language_v1.types.Document` for
+representing text.
+
+Here is an example of document with text provided as a string:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :start-after: [START howto_operator_gcp_natural_language_document_text]
+    :end-before: [END howto_operator_gcp_natural_language_document_text]
+
+In addition to supplying string, a document can refer to content stored in Google Cloud Storage.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :start-after: [START howto_operator_gcp_natural_language_document_gcs]
+    :end-before: [END howto_operator_gcp_natural_language_document_gcs]
+
+.. _howto/operator:CloudLanguageAnalyzeEntitiesOperator:
+
+Analyzing Entities
+^^^^^^^^^^^^^^^^^^
+
+Entity Analysis inspects the given text for known entities (proper nouns such as
+public figures, landmarks, etc.), and returns information about those entities.
+Entity analysis is performed with the
+:class:`~airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitiesOperator` operator.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_entities]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_entities]
+
+You can use :ref:`Jinja templating <jinja-templating>` with
+:template-fields:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitiesOperator`
+parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
+to be used by other operators.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_entities_result]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_entities_result]
+
+.. _howto/operator:CloudLanguageAnalyzeEntitySentimentOperator:
+
+Analyzing Entity Sentiment
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sentiment Analysis inspects the given text and identifies the prevailing
+emotional opinion within the text, especially to determine a writer's attitude
+as positive, negative, or neutral. Sentiment analysis is performed through
+the :class:`~airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitySentimentOperator`
+operator.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_entity_sentiment]
+
+You can use :ref:`Jinja templating <jinja-templating>` with
+:template-fields:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitiesOperator`
+parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
+to be used by other operators.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment_result]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_entity_sentiment_result]
+
+.. _howto/operator:CloudLanguageAnalyzeSentimentOperator:
+
+Analyzing Sentiment
+^^^^^^^^^^^^^^^^^^^
+
+Sentiment Analysis inspects the given text and identifies the prevailing
+emotional opinion within the text, especially to determine a writer's
+attitude as positive, negative, or neutral. Sentiment analysis is performed
+through the
+:class:`~airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentimentOperator`
+operator.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_sentiment]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_sentiment]
+
+You can use :ref:`Jinja templating <jinja-templating>` with
+:template-fields:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentimentOperator`
+parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
+to be used by other operators.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_sentiment_result]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_sentiment_result]
+
+.. _howto/operator:CloudLanguageClassifyTextOperator:
+
+Classifying Content
+^^^^^^^^^^^^^^^^^^^
+
+Content Classification analyzes a document and returns a list of content
+categories that apply to the text found in the document. To classify the
+content in a document, use the
+:class:`~airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageClassifyTextOperator`
+operator.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_classify_text]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_classify_text]
+
+You can use :ref:`Jinja templating <jinja-templating>` with
+:template-fields:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageClassifyTextOperator`
+parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
+to be used by other operators.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_natural_language.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_gcp_natural_language_analyze_classify_text_result]
+    :end-before: [END howto_operator_gcp_natural_language_analyze_classify_text_result]
+
+
+Reference
+^^^^^^^^^
+
+For further information, look at:
+
+* `Client Library Documentation <https://googleapis.github.io/google-cloud-python/latest/language/index.html>`__
+* `Product Documentation <https://cloud.google.com/natural-language/docs/>`__
diff --git a/docs/howto/operator/gcp/spanner.rst b/docs/howto/operator/gcp/spanner.rst
new file mode 100644
index 0000000000000..0d4c5a4e3d727
--- /dev/null
+++ b/docs/howto/operator/gcp/spanner.rst
@@ -0,0 +1,276 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Spanner Operators
+==============================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:CloudSpannerInstanceDatabaseDeleteOperator:
+
+CloudSpannerInstanceDatabaseDeleteOperator
+------------------------------------------
+
+Deletes a database from the specified Cloud Spanner instance. If the database does not
+exist, no action is taken, and the operator succeeds.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeleteOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :start-after: [START howto_operator_spanner_arguments]
+    :end-before: [END howto_operator_spanner_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_spanner_database_delete]
+    :end-before: [END howto_operator_spanner_database_delete]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_spanner_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_spanner_delete_template_fields]
+    :end-before: [END gcp_spanner_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Spanner API documentation to drop an instance of a database
+<https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/dropDatabase>`_.
+
+.. _howto/operator:CloudSpannerInstanceDatabaseDeployOperator:
+
+CloudSpannerInstanceDatabaseDeployOperator
+------------------------------------------
+
+Creates a new Cloud Spanner database in the specified instance, or if the
+desired database exists, assumes success with no changes applied to database
+configuration. No structure of the database is verified - it's enough if the database exists
+with the same name.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeployOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :start-after: [START howto_operator_spanner_arguments]
+    :end-before: [END howto_operator_spanner_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_spanner_database_deploy]
+    :end-before: [END howto_operator_spanner_database_deploy]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_spanner_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_spanner_database_deploy_template_fields]
+    :end-before: [END gcp_spanner_database_deploy_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Spanner API documentation `to create a new database
+<https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/create>`_.
+
+.. _howto/operator:CloudSpannerInstanceDatabaseUpdateOperator:
+
+CloudSpannerInstanceDatabaseUpdateOperator
+------------------------------------------
+
+Runs a DDL query in a Cloud Spanner database and allows you to modify the structure of an
+existing database.
+
+You can optionally specify an operation_id parameter which simplifies determining whether
+the statements were executed in case the update_database call is replayed
+(idempotency check). The operation_id should be unique within the database, and must be
+a valid identifier: `[a-z][a-z0-9_]*`. More information can be found in
+`the documentation of updateDdl API <https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/updateDdl>`_
+
+For parameter definition take a look at
+:class:`~airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseUpdateOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :start-after: [START howto_operator_spanner_arguments]
+    :end-before: [END howto_operator_spanner_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_spanner_database_update]
+    :end-before: [END howto_operator_spanner_database_update]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_spanner_database_update_idempotent]
+    :end-before: [END howto_operator_spanner_database_update_idempotent]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_spanner_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_spanner_database_update_template_fields]
+    :end-before: [END gcp_spanner_database_update_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Spanner API documentation for `database update_ddl
+<https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/updateDdl>`_.
+
+.. _howto/operator:CloudSpannerInstanceDatabaseQueryOperator:
+
+CloudSpannerInstanceDatabaseQueryOperator
+-----------------------------------------
+
+Executes an arbitrary DML query (INSERT, UPDATE, DELETE).
+
+For parameter definition take a look at
+:class:`~airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseQueryOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables.
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :start-after: [START howto_operator_spanner_arguments]
+    :end-before: [END howto_operator_spanner_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_spanner_query]
+    :end-before: [END howto_operator_spanner_query]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_spanner_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_spanner_query_template_fields]
+    :end-before: [END gcp_spanner_query_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Spanner API documentation for more information about `DML syntax
+<https://cloud.google.com/spanner/docs/dml-syntax>`_.
+
+.. _howto/operator:CloudSpannerInstanceDeleteOperator:
+
+CloudSpannerInstanceDeleteOperator
+----------------------------------
+
+Deletes a Cloud Spanner instance. If an instance does not exist, no action is taken,
+and the operator succeeds.
+
+For parameter definition take a look at
+:class:`~airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDeleteOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :start-after: [START howto_operator_spanner_arguments]
+    :end-before: [END howto_operator_spanner_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_spanner.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_spanner_delete]
+    :end-before: [END howto_operator_spanner_delete]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_spanner_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_spanner_delete_template_fields]
+    :end-before: [END gcp_spanner_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud Spanner API documentation to `delete an instance
+<https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances/delete>`_.
diff --git a/docs/howto/operator/gcp/sql.rst b/docs/howto/operator/gcp/sql.rst
new file mode 100644
index 0000000000000..a18a95ef8ebd8
--- /dev/null
+++ b/docs/howto/operator/gcp/sql.rst
@@ -0,0 +1,644 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Sql Operators
+==========================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:CloudSqlInstanceDatabaseCreateOperator:
+
+CloudSqlInstanceDatabaseCreateOperator
+--------------------------------------
+
+Creates a new database inside a Cloud SQL instance.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_db_create]
+    :end-before: [END howto_operator_cloudsql_db_create]
+
+Example request body:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_db_create_body]
+    :end-before: [END howto_operator_cloudsql_db_create_body]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_db_create_template_fields]
+    :end-before: [END gcp_sql_db_create_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation for `to create a new database inside the instance
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert>`_.
+
+.. _howto/operator:CloudSqlInstanceDatabaseDeleteOperator:
+
+CloudSqlInstanceDatabaseDeleteOperator
+--------------------------------------
+
+Deletes a database from a Cloud SQL instance.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_db_delete]
+    :end-before: [END howto_operator_cloudsql_db_delete]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_db_delete_template_fields]
+    :end-before: [END gcp_sql_db_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `delete a database
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/delete>`_.
+
+.. _howto/operator:CloudSqlInstanceDatabasePatchOperator:
+
+CloudSqlInstanceDatabasePatchOperator
+-------------------------------------
+
+Updates a resource containing information about a database inside a Cloud SQL instance
+using patch semantics.
+See: https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_db_patch]
+    :end-before: [END howto_operator_cloudsql_db_patch]
+
+Example request body:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_db_patch_body]
+    :end-before: [END howto_operator_cloudsql_db_patch_body]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_db_patch_template_fields]
+    :end-before: [END gcp_sql_db_patch_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `update a database
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/patch>`_.
+
+.. _howto/operator:CloudSqlInstanceDeleteOperator:
+
+CloudSqlInstanceDeleteOperator
+------------------------------
+
+Deletes a Cloud SQL instance in Google Cloud Platform.
+
+It is also used for deleting read and failover replicas.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_delete]
+    :end-before: [END howto_operator_cloudsql_delete]
+
+Note: If the instance has read or failover replicas you need to delete them before you delete the primary instance.
+Replicas are deleted the same way as primary instances:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_replicas_delete]
+    :end-before: [END howto_operator_cloudsql_replicas_delete]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_delete_template_fields]
+    :end-before: [END gcp_sql_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `delete a SQL instance
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/delete>`_.
+
+.. _howto/operator:CloudSqlInstanceExportOperator:
+
+CloudSqlInstanceExportOperator
+------------------------------
+
+Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump
+or CSV file.
+
+.. note::
+    This operator is idempotent. If executed multiple times with the same
+    export file URI, the export file in GCS will simply be overridden.
+
+For parameter definition take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceExportOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from Airflow variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_export_import_arguments]
+    :end-before: [END howto_operator_cloudsql_export_import_arguments]
+
+Example body defining the export operation:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_export_body]
+    :end-before: [END howto_operator_cloudsql_export_body]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_export]
+    :end-before: [END howto_operator_cloudsql_export]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_export_template_fields]
+    :end-before: [END gcp_sql_export_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `export data
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/export>`_.
+
+Troubleshooting
+"""""""""""""""
+
+If you receive an "Unauthorized" error in GCP, make sure that the service account
+of the Cloud SQL instance is authorized to write to the selected GCS bucket.
+
+It is not the service account configured in Airflow that communicates with GCS,
+but rather the service account of the particular Cloud SQL instance.
+
+To grant the service account with the appropriate WRITE permissions for the GCS bucket
+you can use the :class:`~airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator`,
+as shown in the example:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_export_gcs_permissions]
+    :end-before: [END howto_operator_cloudsql_export_gcs_permissions]
+
+
+.. _howto/operator:CloudSqlInstanceImportOperator:
+
+CloudSqlInstanceImportOperator
+------------------------------
+
+Imports data into a Cloud SQL instance from a SQL dump or CSV file in Cloud Storage.
+
+CSV import:
+"""""""""""
+
+This operator is NOT idempotent for a CSV import. If the same file is imported
+multiple times, the imported data will be duplicated in the database.
+Moreover, if there are any unique constraints the duplicate import may result in an
+error.
+
+SQL import:
+"""""""""""
+
+This operator is idempotent for a SQL import if it was also exported by Cloud SQL.
+The exported SQL contains 'DROP TABLE IF EXISTS' statements for all tables
+to be imported.
+
+If the import file was generated in a different way, idempotence is not guaranteed.
+It has to be ensured on the SQL file level.
+
+For parameter definition take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceImportOperator`.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from Airflow variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_export_import_arguments]
+    :end-before: [END howto_operator_cloudsql_export_import_arguments]
+
+Example body defining the import operation:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_import_body]
+    :end-before: [END howto_operator_cloudsql_import_body]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_import]
+    :end-before: [END howto_operator_cloudsql_import]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_import_template_fields]
+    :end-before: [END gcp_sql_import_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `import data
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/import>`_.
+
+Troubleshooting
+"""""""""""""""
+
+If you receive an "Unauthorized" error in GCP, make sure that the service account
+of the Cloud SQL instance is authorized to read from the selected GCS object.
+
+It is not the service account configured in Airflow that communicates with GCS,
+but rather the service account of the particular Cloud SQL instance.
+
+To grant the service account with the appropriate READ permissions for the GCS object
+you can use the :class:`~airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator`,
+as shown in the example:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_import_gcs_permissions]
+    :end-before: [END howto_operator_cloudsql_import_gcs_permissions]
+
+.. _howto/operator:CloudSqlInstanceCreateOperator:
+
+CloudSqlInstanceCreateOperator
+------------------------------
+
+Creates a new Cloud SQL instance in Google Cloud Platform.
+
+It is also used for creating read replicas.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator`.
+
+If an instance with the same name exists, no action will be taken and the operator
+will succeed.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+Some other arguments are created based on the arguments above:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_create_arguments]
+    :end-before: [END howto_operator_cloudsql_create_arguments]
+
+Example body defining the instance with failover replica:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_create_body]
+    :end-before: [END howto_operator_cloudsql_create_body]
+
+Example body defining read replica for the instance above:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_create_replica]
+    :end-before: [END howto_operator_cloudsql_create_replica]
+
+Note: Failover replicas are created together with the instance in a single task.
+Read replicas need to be created in separate tasks.
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_create]
+    :end-before: [END howto_operator_cloudsql_create]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_create_template_fields]
+    :end-before: [END gcp_sql_create_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `create an instance
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert>`_.
+
+.. _howto/operator:CloudSqlInstancePatchOperator:
+
+CloudSqlInstancePatchOperator
+-----------------------------
+
+Updates settings of a Cloud SQL instance in Google Cloud Platform (partial update).
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator`.
+
+This is a partial update, so only values for the settings specified in the body
+will be set / updated. The rest of the existing instance's configuration will remain
+unchanged.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_arguments]
+    :end-before: [END howto_operator_cloudsql_arguments]
+
+Example body defining the instance:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_patch_body]
+    :end-before: [END howto_operator_cloudsql_patch_body]
+
+Using the operator
+""""""""""""""""""
+
+You can create the operator with or without project id. If project id is missing
+it will be retrieved from the GCP connection used. Both variants are shown:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_cloudsql_patch]
+    :end-before: [END howto_operator_cloudsql_patch]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_patch_template_fields]
+    :end-before: [END gcp_sql_patch_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL API documentation to `patch an instance
+<https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch>`_.
+
+.. _howto/operator:CloudSqlQueryOperator:
+
+CloudSqlQueryOperator
+---------------------
+
+Performs DDL or DML SQL queries in Google Cloud SQL instance. The DQL
+(retrieving data from Google Cloud SQL) is not supported. You might run the SELECT
+queries, but the results of those queries are discarded.
+
+You can specify various connectivity methods to connect to running instance,
+starting from public IP plain connection through public IP with SSL or both TCP and
+socket connection via Cloud SQL Proxy. The proxy is downloaded and started/stopped
+dynamically as needed by the operator.
+
+There is a *gcpcloudsql://* connection type that you should use to define what
+kind of connectivity you want the operator to use. The connection is a "meta"
+type of connection. It is not used to make an actual connectivity on its own, but it
+determines whether Cloud SQL Proxy should be started by `CloudSqlDatabaseHook`
+and what kind of database connection (Postgres or MySQL) should be created
+dynamically to connect to Cloud SQL via public IP address or via the proxy.
+The 'CloudSqlDatabaseHook` uses
+:class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlProxyRunner` to manage Cloud SQL
+Proxy lifecycle (each task has its own Cloud SQL Proxy)
+
+When you build connection, you should use connection parameters as described in
+:class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook`. You can see
+examples of connections below for all the possible types of connectivity. Such connection
+can be reused between different tasks (instances of `CloudSqlQueryOperator`). Each
+task will get their own proxy started if needed with their own TCP or UNIX socket.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator`.
+
+Since query operator can run arbitrary query, it cannot be guaranteed to be
+idempotent. SQL query designer should design the queries to be idempotent. For example,
+both Postgres and MySQL support CREATE TABLE IF NOT EXISTS statements that can be
+used to create tables in an idempotent way.
+
+Arguments
+"""""""""
+
+If you define connection via `AIRFLOW_CONN_*` URL defined in an environment
+variable, make sure the URL components in the URL are URL-encoded.
+See examples below for details.
+
+Note that in case of SSL connections you need to have a mechanism to make the
+certificate/key files available in predefined locations for all the workers on
+which the operator can run. This can be provided for example by mounting
+NFS-like volumes in the same path for all the workers.
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql_query.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_query_arguments]
+    :end-before: [END howto_operator_cloudsql_query_arguments]
+
+Example connection definitions for all connectivity cases. Note that all the components
+of the connection URI should be URL-encoded:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql_query.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_query_connections]
+    :end-before: [END howto_operator_cloudsql_query_connections]
+
+Using the operator
+""""""""""""""""""
+
+Example operators below are using all connectivity options. Note connection id
+from the operator matches the `AIRFLOW_CONN_*` postfix uppercase. This is
+standard AIRFLOW notation for defining connection via environment variables):
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_sql_query.py
+    :language: python
+    :start-after: [START howto_operator_cloudsql_query_operators]
+    :end-before: [END howto_operator_cloudsql_query_operators]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_sql_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_sql_query_template_fields]
+    :end-before: [END gcp_sql_query_template_fields]
+
+More information
+""""""""""""""""
+
+See Google Cloud SQL documentation for `MySQL <https://cloud.google.com/sql/docs/mysql/sql-proxy>`_ and
+`PostgreSQL <https://cloud.google.com/sql/docs/postgres/sql-proxy>`_ related proxies.
diff --git a/docs/howto/operator/gcp/transfer.rst b/docs/howto/operator/gcp/transfer.rst
new file mode 100644
index 0000000000000..c41313669cf4c
--- /dev/null
+++ b/docs/howto/operator/gcp/transfer.rst
@@ -0,0 +1,420 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Transfer Service Operators
+=======================================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:GcpTransferServiceJobCreateOperator:
+
+GcpTransferServiceJobCreateOperator
+-----------------------------------
+
+Create a transfer job.
+
+The function accepts dates in two formats:
+
+- consistent with `Google API <https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#TimeOfDay>`_ ::
+
+    { "year": 2019, "month": 2, "day": 11 }
+
+- as an :class:`~datetime.datetime` object
+
+The function accepts time in two formats:
+
+- consistent with `Google API <https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#TimeOfDay>`_ ::
+
+    { "hours": 12, "minutes": 30, "seconds": 0 }
+
+- as an :class:`~datetime.time` object
+
+If you want to create a job transfer that copies data from AWS S3 then you must have a connection configured. Information about configuration for AWS is available: :doc:`../../connection/aws`
+The selected connection for AWS can be indicated by the parameter ``aws_conn_id``.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_create_job_body_gcp]
+      :end-before: [END howto_operator_gcp_transfer_create_job_body_gcp]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_create_job_body_aws]
+      :end-before: [END howto_operator_gcp_transfer_create_job_body_aws]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_create_job]
+      :end-before: [END howto_operator_gcp_transfer_create_job]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_job_create_template_fields]
+    :end-before: [END gcp_transfer_job_create_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferJobs.create
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/create>`_.
+
+.. _howto/operator:GcpTransferServiceJobDeleteOperator:
+
+GcpTransferServiceJobDeleteOperator
+-----------------------------------
+
+Deletes a transfer job.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_delete_job]
+      :end-before: [END howto_operator_gcp_transfer_delete_job]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_job_delete_template_fields]
+    :end-before: [END gcp_transfer_job_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - REST Resource: transferJobs - Status
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#Status>`_
+
+.. _howto/operator:GcpTransferServiceJobUpdateOperator:
+
+GcpTransferServiceJobUpdateOperator
+-----------------------------------
+
+Updates a transfer job.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_update_job_body]
+      :end-before: [END howto_operator_gcp_transfer_update_job_body]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_update_job]
+      :end-before: [END howto_operator_gcp_transfer_update_job]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_job_update_template_fields]
+    :end-before: [END gcp_transfer_job_update_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferJobs.patch
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/patch>`_
+
+.. _howto/operator:GcpTransferServiceOperationCancelOperator:
+
+GcpTransferServiceOperationCancelOperator
+-----------------------------------------
+
+Gets a transfer operation. The result is returned to XCOM.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_cancel_operation]
+      :end-before: [END howto_operator_gcp_transfer_cancel_operation]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_operation_cancel_template_fields]
+    :end-before: [END gcp_transfer_operation_cancel_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferOperations.cancel
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations/cancel>`_
+
+
+.. _howto/operator:GcpTransferServiceOperationGetOperator:
+
+GcpTransferServiceOperationGetOperator
+--------------------------------------
+
+Gets a transfer operation. The result is returned to XCOM.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_get_operation]
+      :end-before: [END howto_operator_gcp_transfer_get_operation]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_operation_get_template_fields]
+    :end-before: [END gcp_transfer_operation_get_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferOperations.get
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations/get>`_
+
+.. _howto/operator:GcpTransferServiceOperationsListOperator:
+
+GcpTransferServiceOperationsListOperator
+----------------------------------------
+
+List a transfer operations. The result is returned to XCOM.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_list_operations]
+      :end-before: [END howto_operator_gcp_transfer_list_operations]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_operations_list_template_fields]
+    :end-before: [END gcp_transfer_operations_list_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferOperations.list
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations/list>`_
+
+.. _howto/operator:GcpTransferServiceOperationPauseOperator:
+
+GcpTransferServiceOperationPauseOperator
+----------------------------------------
+
+Pauses a transfer operations.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_pause_operation]
+      :end-before: [END howto_operator_gcp_transfer_pause_operation]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_operation_pause_template_fields]
+    :end-before: [END gcp_transfer_operation_pause_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferOperations.pause
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations/pause>`_
+
+.. _howto/operator:GcpTransferServiceOperationResumeOperator:
+
+GcpTransferServiceOperationResumeOperator
+-----------------------------------------
+
+Resumes a transfer operations.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_resume_operation]
+      :end-before: [END howto_operator_gcp_transfer_resume_operation]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_transfer_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_operation_resume_template_fields]
+    :end-before: [END gcp_transfer_operation_resume_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - Method: transferOperations.resume
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferOperations/resume>`_
+
+
+GCPTransferServiceWaitForJobStatusSensor
+----------------------------------------
+
+Waits for at least one operation belonging to the job to have the expected status.
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :start-after: [START howto_operator_gcp_transfer_common_variables]
+      :end-before: [END howto_operator_gcp_transfer_common_variables]
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_transfer.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_wait_operation]
+      :end-before: [END howto_operator_gcp_transfer_wait_operation]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/sensors/gcp_transfer_sensor.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_job_sensor_template_fields]
+    :end-before: [END gcp_transfer_job_sensor_template_fields]
diff --git a/docs/howto/operator/gcp/translate.rst b/docs/howto/operator/gcp/translate.rst
new file mode 100644
index 0000000000000..32ad7bfdb688a
--- /dev/null
+++ b/docs/howto/operator/gcp/translate.rst
@@ -0,0 +1,68 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Translate Operators
+--------------------------------
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:CloudTranslateTextOperator:
+
+CloudTranslateTextOperator
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Translate a string or list of strings.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_translate_operator.CloudTranslateTextOperator`
+
+Using the operator
+""""""""""""""""""
+
+Basic usage of the operator:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_translate.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_translate_text]
+      :end-before: [END howto_operator_translate_text]
+
+The result of translation is available as dictionary or array of dictionaries accessible via the usual
+XCom mechanisms of Airflow:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_translate.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_translate_access]
+      :end-before: [END howto_operator_translate_access]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_translate_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START translate_template_fields]
+    :end-before: [END translate_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Translate documentation <https://cloud.google.com/translate/docs/translating-text>`_.
diff --git a/docs/howto/operator/gcp/vision.rst b/docs/howto/operator/gcp/vision.rst
new file mode 100644
index 0000000000000..67ed7cc44899e
--- /dev/null
+++ b/docs/howto/operator/gcp/vision.rst
@@ -0,0 +1,933 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Google Cloud Vision Operators
+=============================
+
+.. contents::
+  :depth: 1
+  :local:
+
+.. _howto/operator:CloudVisionAddProductToProductSetOperator:
+
+CloudVisionAddProductToProductSetOperator
+-----------------------------------------
+
+Creates a new :code:`ReferenceImage` resource.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionAddProductToProductSetOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_args_common]
+      :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_explicit_id]
+      :end-before: [END howto_operator_vision_product_explicit_id]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_set_explicit_id]
+      :end-before: [END howto_operator_vision_product_set_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+We are using the :class:`~google.cloud.vision_v1.types.Product`,
+:class:`~google.cloud.vision_v1.types.ProductSet` and ``Retry`` objects from
+Google libraries:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_retry_import]
+      :end-before: [END howto_operator_vision_retry_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_set_import]
+      :end-before: [END howto_operator_vision_product_set_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_import]
+      :end-before: [END howto_operator_vision_product_import]
+
+
+If ``product_set_id`` and ``product_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_add_product_to_product_set]
+      :end-before: [END howto_operator_vision_add_product_to_product_set]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_add_product_to_product_set_2]
+      :end-before: [END howto_operator_vision_add_product_to_product_set_2]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_add_product_to_product_set_template_fields]
+    :end-before: [END vision_add_product_to_product_set_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Add Product To Product Set documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.add_product_to_product_set>`_.
+
+
+.. _howto/operator:CloudVisionAnnotateImageOperator:
+
+CloudVisionAnnotateImageOperator
+--------------------------------
+
+Run image detection and annotation for an image.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionAnnotateImageOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_args_common]
+      :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_annotate_image_url]
+      :end-before: [END howto_operator_vision_annotate_image_url]
+
+
+Using the operator
+""""""""""""""""""
+
+We are using the :class:`~google.cloud.enums` and ``Retry`` objects from
+Google libraries:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_retry_import]
+      :end-before: [END howto_operator_vision_retry_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_enums_import]
+      :end-before: [END howto_operator_vision_enums_import]
+
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_annotate_image]
+      :end-before: [END howto_operator_vision_annotate_image]
+
+The result can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_annotate_image_result]
+      :end-before: [END howto_operator_vision_annotate_image_result]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_annotate_image_template_fields]
+    :end-before: [END vision_annotate_image_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Annotate Image documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ImageAnnotatorClient.annotate_image>`_.
+
+.. _howto/operator:CloudVisionProductCreateOperator:
+
+CloudVisionProductCreateOperator
+--------------------------------
+
+Creates and returns a new product resource.
+
+Possible errors regarding the :code:`Product` object provided:
+
+- Returns INVALID_ARGUMENT if `display_name` is missing or longer than 4096 characters.
+- Returns INVALID_ARGUMENT if `description` is longer than 4096 characters.
+- Returns INVALID_ARGUMENT if `product_category` is missing or invalid.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductCreateOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_explicit_id]
+    :end-before: [END howto_operator_vision_product_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+We are using the ``Product`` and ``Retry`` objects from Google libraries:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_import]
+    :end-before: [END howto_operator_vision_product_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_retry_import]
+    :end-before: [END howto_operator_vision_retry_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product]
+    :end-before: [END howto_operator_vision_product]
+
+The ``product_id`` argument can be omitted (it will be generated by the API):
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_create]
+    :end-before: [END howto_operator_vision_product_create]
+
+Or it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_create_2]
+    :end-before: [END howto_operator_vision_product_create_2]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_product_create_template_fields]
+    :end-before: [END vision_product_create_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Product create documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.create_product>`_.
+
+.. _howto/operator:CloudVisionProductDeleteOperator:
+
+CloudVisionProductDeleteOperator
+--------------------------------
+
+Permanently deletes a product and its reference images.
+
+Metadata of the product and all its images will be deleted right away, but search queries
+against :code:`ProductSets` containing the product may still work until all related
+caches are refreshed.
+
+Possible errors:
+
+- Returns NOT_FOUND if the product does not exist.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductDeleteOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_explicit_id]
+    :end-before: [END howto_operator_vision_product_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+If ``product_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_delete]
+    :end-before: [END howto_operator_vision_product_delete]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_delete_2]
+    :end-before: [END howto_operator_vision_product_delete_2]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_product_delete_template_fields]
+    :end-before: [END vision_product_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Product delete documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.delete_product>`_.
+
+.. _howto/operator:CloudVisionProductGetOperator:
+
+CloudVisionProductGetOperator
+-----------------------------
+
+Gets information associated with a :code:`Product`.
+
+Possible errors:
+
+- Returns NOT_FOUND if the `Product` does not exist.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductGetOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_explicit_id]
+    :end-before: [END howto_operator_vision_product_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+If ``product_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_get]
+    :end-before: [END howto_operator_vision_product_get]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_get_2]
+    :end-before: [END howto_operator_vision_product_get_2]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_product_get_template_fields]
+    :end-before: [END vision_product_get_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Product get documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.get_product>`_.
+
+.. _howto/operator:CloudVisionProductSetCreateOperator:
+
+CloudVisionProductSetCreateOperator
+-----------------------------------
+
+Creates a new :code:`ProductSet` resource.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set_explicit_id
+    :end-before: [END howto_operator_vision_product_set_explicit_id
+
+Using the operator
+""""""""""""""""""
+
+We are using the ``ProductSet`` and ``Retry`` objects from Google libraries:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set_import]
+    :end-before: [END howto_operator_vision_product_set_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_retry_import]
+    :end-before: [END howto_operator_vision_retry_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set]
+    :end-before: [END howto_operator_vision_product_set]
+
+The ``product_set_id`` argument can be omitted (it will be generated by the API):
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_create]
+    :end-before: [END howto_operator_vision_product_set_create]
+
+Or it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_create_2]
+    :end-before: [END howto_operator_vision_product_set_create_2]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_productset_create_template_fields]
+    :end-before: [END vision_productset_create_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision ProductSet create documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.create_product_set>`_.
+
+.. _howto/operator:CloudVisionProductSetDeleteOperator:
+
+CloudVisionProductSetDeleteOperator
+-----------------------------------
+
+Permanently deletes a :code:`ProductSet`. :code:`Products` and :code:`ReferenceImages` in
+the :code:`ProductSet` are not deleted. The actual image files are not deleted from
+Google Cloud Storage.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetDeleteOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set_explicit_id]
+    :end-before: [END howto_operator_vision_product_set_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+If ``product_set_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_delete]
+    :end-before: [END howto_operator_vision_product_set_delete]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_delete_2]
+    :end-before: [END howto_operator_vision_product_set_delete_2]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_productset_delete_template_fields]
+    :end-before: [END vision_productset_delete_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision ProductSet delete documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.delete_product_set>`_.
+
+.. _howto/operator:CloudVisionProductSetGetOperator:
+
+CloudVisionProductSetGetOperator
+--------------------------------
+
+Gets information associated with a :code:`ProductSet`.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetGetOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set_explicit_id]
+    :end-before: [END howto_operator_vision_product_set_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+If ``product_set_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_get]
+    :end-before: [END howto_operator_vision_product_set_get]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_get_2]
+    :end-before: [END howto_operator_vision_product_set_get_2]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_productset_get_template_fields]
+    :end-before: [END vision_productset_get_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision ProductSet get documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.get_product_set>`_.
+
+.. _howto/operator:CloudVisionProductSetUpdateOperator:
+
+CloudVisionProductSetUpdateOperator
+-----------------------------------
+
+Makes changes to a :code:`ProductSet` resource. Only :code:`display_name` can be updated
+currently.
+
+.. note:: To locate the `ProductSet` resource, its `name` in the form
+  ``projects/PROJECT_ID/locations/LOC_ID/productSets/PRODUCT_SET_ID`` is necessary.
+
+You can provide the `name` directly as an attribute of the `product_set` object.
+However, you can leave it blank and provide `location` and `product_set_id` instead (and
+optionally `project_id` - if not present, the connection default will be used) and the
+`name` will be created by the operator itself.
+
+This mechanism exists for your convenience, to allow leaving the `project_id` empty and
+having Airflow use the connection default `project_id`.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetUpdateOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set_explicit_id]
+    :end-before: [END howto_operator_vision_product_set_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+We are using the ``ProductSet`` object from the Google Cloud Vision library:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set_import]
+    :end-before: [END howto_operator_vision_product_set_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_set]
+    :end-before: [END howto_operator_vision_product_set]
+
+Initialization of the task:
+
+If ``product_set_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_update]
+    :end-before: [END howto_operator_vision_product_set_update]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_set_update_2]
+    :end-before: [END howto_operator_vision_product_set_update_2]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_productset_update_template_fields]
+    :end-before: [END vision_productset_update_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision ProductSet update documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.update_product_set>`_.
+
+.. _howto/operator:CloudVisionProductUpdateOperator:
+
+CloudVisionProductUpdateOperator
+--------------------------------
+
+Makes changes to a :code:`Product` resource. Only the :code:`display_name`,
+:code:`description`, and :code:`labels` fields can be updated right now.
+If labels are updated, the change will not be reflected in queries until the next index
+time.
+
+.. note:: To locate the `Product` resource, its `name` in the form
+  ``projects/PROJECT_ID/locations/LOC_ID/products/PRODUCT_ID`` is necessary.
+
+You can provide the `name` directly as an attribute of the `product` object. However, you
+can leave it blank and provide `location` and `product_id` instead (and optionally
+`project_id` - if not present, the connection default will be used) and the `name` will
+be created by the operator itself.
+
+This mechanism exists for your convenience, to allow leaving the `project_id` empty and
+having Airflow use the connection default `project_id`.
+
+Possible errors:
+
+- Returns NOT_FOUND if the `Product` does not exist.
+- Returns INVALID_ARGUMENT if `display_name` is present in `update_mask` but is missing
+  from the request or longer than 4096 characters.
+- Returns INVALID_ARGUMENT if `description` is present in `update_mask` but is longer than
+  4096 characters.
+- Returns INVALID_ARGUMENT if `product_category` is present in `update_mask`.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductUpdateOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_args_common]
+    :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_explicit_id]
+    :end-before: [END howto_operator_vision_product_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+We are using the ``Product`` object from the Google Cloud Vision library:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product_import]
+    :end-before: [END howto_operator_vision_product_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :start-after: [START howto_operator_vision_product]
+    :end-before: [END howto_operator_vision_product]
+
+If ``product_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_update]
+    :end-before: [END howto_operator_vision_product_update]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_vision_product_update_2]
+    :end-before: [END howto_operator_vision_product_update_2]
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_product_update_template_fields]
+    :end-before: [END vision_product_update_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Product update documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.update_product>`_.
+
+.. _howto/operator:CloudVisionReferenceImageCreateOperator:
+
+CloudVisionReferenceImageCreateOperator
+---------------------------------------
+
+Creates a new :code:`ReferenceImage` resource.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionReferenceImageCreateOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_args_common]
+      :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_reference_image_args]
+      :end-before: [END howto_operator_vision_reference_image_args]
+
+Using the operator
+""""""""""""""""""
+
+We are using the :class:`~google.cloud.vision_v1.types.ReferenceImage` and ``Retry`` objects from Google libraries:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_reference_image_import]
+      :end-before: [END howto_operator_vision_reference_image_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_retry_import]
+      :end-before: [END howto_operator_vision_retry_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_reference_image]
+      :end-before: [END howto_operator_vision_reference_image]
+
+The ``product_set_id`` argument can be omitted (it will be generated by the API):
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_reference_image_create]
+      :end-before: [END howto_operator_vision_reference_image_create]
+
+Or it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_reference_image_create_2]
+      :end-before: [END howto_operator_vision_reference_image_create_2]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_reference_image_create_template_fields]
+    :end-before: [END vision_reference_image_create_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision ReferenceImage create documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.create_reference_image>`_.
+
+.. _howto/operator:CloudVisionRemoveProductFromProductSetOperator:
+
+CloudVisionRemoveProductFromProductSetOperator
+----------------------------------------------
+
+Creates a new :code:`ReferenceImage` resource.
+
+For parameter definition, take a look at
+:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionRemoveProductFromProductSetOperator`
+
+Arguments
+"""""""""
+
+Some arguments in the example DAG are taken from the OS environment variables:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_args_common]
+      :end-before: [END howto_operator_vision_args_common]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_explicit_id]
+      :end-before: [END howto_operator_vision_product_explicit_id]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_set_explicit_id]
+      :end-before: [END howto_operator_vision_product_set_explicit_id]
+
+Using the operator
+""""""""""""""""""
+
+We are using the :class:`~google.cloud.vision_v1.types.Product`,
+:class:`~google.cloud.vision_v1.types.ProductSet` and ``Retry`` objects from
+Google libraries:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_retry_import]
+      :end-before: [END howto_operator_vision_retry_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_set_import]
+      :end-before: [END howto_operator_vision_product_set_import]
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :start-after: [START howto_operator_vision_product_import]
+      :end-before: [END howto_operator_vision_product_import]
+
+
+If ``product_set_id`` and ``product_id`` was generated by the API it can be extracted from XCOM:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_remove_product_from_product_set]
+      :end-before: [END howto_operator_vision_remove_product_from_product_set]
+
+Otherwise it can be specified explicitly:
+
+.. exampleinclude:: ../../../../airflow/contrib/example_dags/example_gcp_vision.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_vision_remove_product_from_product_set_2]
+      :end-before: [END howto_operator_vision_remove_product_from_product_set_2]
+
+
+Templating
+""""""""""
+
+.. exampleinclude:: ../../../../airflow/contrib/operators/gcp_vision_operator.py
+    :language: python
+    :dedent: 4
+    :start-after: [START vision_remove_product_from_product_set_template_fields]
+    :end-before: [END vision_remove_product_from_product_set_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Vision Remove Product From Product Set documentation
+<https://googleapis.github.io/google-cloud-python/latest/vision/gapic/v1/api.html#google.cloud.vision_v1.ProductSearchClient.remove_product_from_product_set>`_.
diff --git a/docs/howto/operator/index.rst b/docs/howto/operator/index.rst
new file mode 100644
index 0000000000000..3ed731d020d46
--- /dev/null
+++ b/docs/howto/operator/index.rst
@@ -0,0 +1,34 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Using Operators
+===============
+
+An operator represents a single, ideally idempotent, task. Operators
+determine what actually executes when your DAG runs.
+
+See the :ref:`Operators Concepts <concepts-operators>` documentation and the
+:doc:`Operators API Reference <../../_api/index>` for more
+information.
+
+.. toctree::
+    :maxdepth: 2
+
+    bash
+    dingding
+    gcp/index
+    python
diff --git a/docs/howto/operator/python.rst b/docs/howto/operator/python.rst
new file mode 100644
index 0000000000000..da2180138e069
--- /dev/null
+++ b/docs/howto/operator/python.rst
@@ -0,0 +1,50 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. _howto/operator:PythonOperator:
+
+PythonOperator
+==============
+
+Use the :class:`~airflow.operators.python_operator.PythonOperator` to execute
+Python callables.
+
+.. exampleinclude:: ../../../airflow/example_dags/example_python_operator.py
+    :language: python
+    :start-after: [START howto_operator_python]
+    :end-before: [END howto_operator_python]
+
+Passing in arguments
+^^^^^^^^^^^^^^^^^^^^
+
+Use the ``op_args`` and ``op_kwargs`` arguments to pass additional arguments
+to the Python callable.
+
+.. exampleinclude:: ../../../airflow/example_dags/example_python_operator.py
+    :language: python
+    :start-after: [START howto_operator_python_kwargs]
+    :end-before: [END howto_operator_python_kwargs]
+
+Templating
+^^^^^^^^^^
+
+When you set the ``provide_context`` argument to ``True``, Airflow passes in
+an additional set of keyword arguments: one for each of the :doc:`Jinja
+template variables <../../macros>` and a ``templates_dict`` argument.
+
+The ``templates_dict`` argument is templated, so each value in the dictionary
+is evaluated as a :ref:`Jinja template <jinja-templating>`.
diff --git a/docs/howto/run-behind-proxy.rst b/docs/howto/run-behind-proxy.rst
new file mode 100644
index 0000000000000..5db4ab0bf155e
--- /dev/null
+++ b/docs/howto/run-behind-proxy.rst
@@ -0,0 +1,82 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Running Airflow behind a reverse proxy
+======================================
+
+Airflow can be set up behind a reverse proxy, with the ability to set its endpoint with great
+flexibility.
+
+For example, you can configure your reverse proxy to get:
+
+::
+
+    https://lab.mycompany.com/myorg/airflow/
+
+To do so, you need to set the following setting in your `airflow.cfg`::
+
+    base_url = http://my_host/myorg/airflow
+
+Additionally if you use Celery Executor, you can get Flower in `/myorg/flower` with::
+
+    flower_url_prefix = /myorg/flower
+
+Your reverse proxy (ex: nginx) should be configured as follow:
+
+- pass the url and http header as it for the Airflow webserver, without any rewrite, for example::
+
+      server {
+        listen 80;
+        server_name lab.mycompany.com;
+
+        location /myorg/airflow/ {
+            proxy_pass http://localhost:8080;
+            proxy_set_header Host $host;
+            proxy_redirect off;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+        }
+      }
+
+- rewrite the url for the flower endpoint::
+
+      server {
+          listen 80;
+          server_name lab.mycompany.com;
+
+          location /myorg/flower/ {
+              rewrite ^/myorg/flower/(.*)$ /$1 break;  # remove prefix from http header
+              proxy_pass http://localhost:5555;
+              proxy_set_header Host $host;
+              proxy_redirect off;
+              proxy_http_version 1.1;
+              proxy_set_header Upgrade $http_upgrade;
+              proxy_set_header Connection "upgrade";
+          }
+      }
+
+To ensure that Airflow generates URLs with the correct scheme when
+running behind a TLS-terminating proxy, you should configure the proxy
+to set the `X-Forwarded-Proto` header, and enable the `ProxyFix`
+middleware in your `airflow.cfg`::
+
+    enable_proxy_fix = True
+
+.. note::
+    You should only enable the `ProxyFix` middleware when running
+    Airflow behind a trusted proxy (AWS ELB, nginx, etc.).
diff --git a/docs/howto/run-with-systemd.rst b/docs/howto/run-with-systemd.rst
index 1dc138347c37a..e225c4a35ecfd 100644
--- a/docs/howto/run-with-systemd.rst
+++ b/docs/howto/run-with-systemd.rst
@@ -1,15 +1,38 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Running Airflow with systemd
 ============================
 
 Airflow can integrate with systemd based systems. This makes watching your
-daemons easy as systemd can take care of restarting a daemon on failure.
-In the ``scripts/systemd`` directory you can find unit files that
-have been tested on Redhat based systems. You can copy those to
-``/usr/lib/systemd/system``. It is assumed that Airflow will run under
-``airflow:airflow``. If not (or if you are running on a non Redhat
-based system) you probably need to adjust the unit files.
-
-Environment configuration is picked up from ``/etc/sysconfig/airflow``.
-An example file is supplied. Make sure to specify the ``SCHEDULER_RUNS``
-variable in this file when you run the scheduler. You
-can also define here, for example, ``AIRFLOW_HOME`` or ``AIRFLOW_CONFIG``.
+daemons easy as `systemd` can take care of restarting a daemon on failures.
+
+In the ``scripts/systemd`` directory, you can find unit files that
+have been tested on Redhat based systems. These files can be used as-is by copying them over to
+``/usr/lib/systemd/system``.
+
+The following **assumptions** have been made while creating these unit files:
+
+#. Airflow runs as the following `user:group` ``airflow:airflow``.
+#. Airflow runs on a Redhat based system.
+
+If this is not the case, appropriate changes will need to be made.
+
+Please **note** that environment configuration is picked up from ``/etc/sysconfig/airflow``.
+
+An example file is supplied within ``scripts/systemd``.
+You can also define configuration at ``AIRFLOW_HOME`` or ``AIRFLOW_CONFIG``.
diff --git a/docs/howto/run-with-upstart.rst b/docs/howto/run-with-upstart.rst
index c18a203f17b31..a45a07de8f3c0 100644
--- a/docs/howto/run-with-upstart.rst
+++ b/docs/howto/run-with-upstart.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Running Airflow with upstart
 ============================
 
@@ -5,12 +22,17 @@ Airflow can integrate with upstart based systems. Upstart automatically starts a
 have a corresponding ``*.conf`` file in ``/etc/init`` upon system boot. On failure, upstart automatically restarts
 the process (until it reaches re-spawn limit set in a ``*.conf`` file).
 
-You can find sample upstart job files in the ``scripts/upstart`` directory. These files have been tested on
-Ubuntu 14.04 LTS. You may have to adjust ``start on`` and ``stop on`` stanzas to make it work on other upstart
-systems. Some of the possible options are listed in ``scripts/upstart/README``.
+You can find sample upstart job files in the ``scripts/upstart`` directory.
+
+The following assumptions have been used while creating these unit files:
+
+1. Airflow will run as the following `user:group` ``airflow:airflow``.
+    Change ``setuid`` and ``setgid`` appropriately in ``*.conf`` if airflow runs as a different user or group
+2. These files have been tested on **Ubuntu 14.04 LTS**
+    You may have to adjust ``start on`` and ``stop on`` stanzas to make it work on other upstart systems.
+    Some of the possible options are listed in ``scripts/upstart/README``
 
-Modify ``*.conf`` files as needed and copy to ``/etc/init`` directory. It is assumed that airflow will run
-under ``airflow:airflow``. Change ``setuid`` and ``setgid`` in ``*.conf`` files if you use other user/group
+Modify ``*.conf`` files as needed and copy to ``/etc/init`` directory.
 
 You can use ``initctl`` to manually start, stop, view status of the airflow process that has been
 integrated with upstart
diff --git a/docs/howto/secure-connections.rst b/docs/howto/secure-connections.rst
index b9c1fa10048f5..a9240e1e20426 100644
--- a/docs/howto/secure-connections.rst
+++ b/docs/howto/secure-connections.rst
@@ -1,32 +1,61 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Securing Connections
 ====================
 
 By default, Airflow will save the passwords for the connection in plain text
 within the metadata database. The ``crypto`` package is highly recommended
 during installation. The ``crypto`` package does require that your operating
-system have libffi-dev installed.
+system has ``libffi-dev`` installed.
+
+If ``crypto`` package was not installed initially, it means that your Fernet key in ``airflow.cfg`` is empty.
+
+You can still enable encryption for passwords within connections by following below steps:
+
+#. Install crypto package ``pip install 'apache-airflow[crypto]'``
+#. Generate fernet_key, using this code snippet below. ``fernet_key`` must be a base64-encoded 32-byte key:
+
+    .. code:: python
 
-If ``crypto`` package was not installed initially, you can still enable encryption for
-connections by following steps below:
+      from cryptography.fernet import Fernet
+      fernet_key= Fernet.generate_key()
+      print(fernet_key.decode()) # your fernet_key, keep it in secured place!
 
-1. Install crypto package ``pip install apache-airflow[crypto]``
-2. Generate fernet_key, using this code snippet below. fernet_key must be a base64-encoded 32-byte key.
+#. Replace ``airflow.cfg`` fernet_key value with the one from `Step 2`. *Alternatively,* you can store your fernet_key in OS environment variable - You do not need to change ``airflow.cfg`` in this case as Airflow will use environment variable over the value in ``airflow.cfg``:
 
-.. code:: python
+    .. code-block:: bash
 
-    from cryptography.fernet import Fernet
-    fernet_key= Fernet.generate_key()
-    print(fernet_key) # your fernet_key, keep it in secured place!
+      # Note the double underscores
+      export AIRFLOW__CORE__FERNET_KEY=your_fernet_key
 
-3. Replace ``airflow.cfg`` fernet_key value with the one from step 2.
-Alternatively, you can store your fernet_key in OS environment variable. You
-do not need to change ``airflow.cfg`` in this case as Airflow will use environment
-variable over the value in ``airflow.cfg``:
+#. Restart the webserver
+#. For existing connections (the ones that you had defined before installing ``airflow[crypto]`` and creating a Fernet key), you need to open each connection in the connection admin UI, re-type the password, and save the change
 
-.. code-block:: bash
+Rotating encryption keys
+========================
 
-  # Note the double underscores
-  EXPORT AIRFLOW__CORE__FERNET_KEY = your_fernet_key
+Once connection credentials and variables have been encrypted using a fernet
+key, changing the key will cause decryption of existing credentials to fail. To
+rotate the fernet key without invalidating existing encrypted values, prepend
+the new key to the ``fernet_key`` setting, run
+``airflow rotate_fernet_key``, and then drop the original key from
+``fernet_keys``:
 
-4. Restart Airflow webserver.
-5. For existing connections (the ones that you had defined before installing ``airflow[crypto]`` and creating a Fernet key), you need to open each connection in the connection admin UI, re-type the password, and save it.
+#. Set ``fernet_key`` to ``new_fernet_key,old_fernet_key``
+#. Run ``airflow rotate_fernet_key`` to re-encrypt existing credentials with the new fernet key
+#. Set ``fernet_key`` to ``new_fernet_key``
diff --git a/docs/howto/set-config.rst b/docs/howto/set-config.rst
index 2caf5d5eb7275..dbbfe6198d792 100644
--- a/docs/howto/set-config.rst
+++ b/docs/howto/set-config.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Setting Configuration Options
 =============================
 
@@ -20,16 +37,30 @@ or by creating a corresponding environment variable:
 
     AIRFLOW__CORE__SQL_ALCHEMY_CONN=my_conn_string
 
-You can also derive the connection string at run time by appending ``_cmd`` to the key like this:
+You can also derive the connection string at run time by appending ``_cmd`` to
+the key like this:
 
 .. code-block:: bash
 
     [core]
     sql_alchemy_conn_cmd = bash_command_to_run
 
--But only three such configuration elements namely sql_alchemy_conn, broker_url and result_backend can be fetched as a command. The idea behind this is to not store passwords on boxes in plain text files. The order of precedence is as follows -
+The following config options support this ``_cmd`` version:
+
+* ``sql_alchemy_conn`` in ``[core]`` section
+* ``fernet_key`` in ``[core]`` section
+* ``broker_url`` in ``[celery]`` section
+* ``result_backend`` in ``[celery]`` section
+* ``password`` in ``[atlas]`` section
+* ``smtp_password`` in ``[smtp]`` section
+* ``bind_password`` in ``[ldap]`` section
+* ``git_password`` in ``[kubernetes]`` section
+
+The idea behind this is to not store passwords on boxes in plain text files.
+
+The universal order of precedence for all configuration options is as follows:
 
-1. environment variable
-2. configuration in airflow.cfg
-3. command in airflow.cfg
-4. default
+#. set as an environment variable
+#. set in ``airflow.cfg``
+#. command in ``airflow.cfg``
+#. Airflow's built in defaults
diff --git a/docs/howto/use-test-config.rst b/docs/howto/use-test-config.rst
index 5cb4790287992..e871d87b41fc2 100644
--- a/docs/howto/use-test-config.rst
+++ b/docs/howto/use-test-config.rst
@@ -1,17 +1,32 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Using the Test Mode Configuration
 =================================
 
 Airflow has a fixed set of "test mode" configuration options. You can load these
-at any time by calling ``airflow.configuration.load_test_config()`` (note this
-operation is not reversible!). However, some options (like the DAG_FOLDER) are
-loaded before you have a chance to call load_test_config(). In order to eagerly load
-the test configuration, set test_mode in airflow.cfg:
+at any time by calling ``airflow.configuration.load_test_config()``. Please **note** that this operation is **not reversible**.
+
+Some options for example, DAG_FOLDER, are loaded before you have a chance to call load_test_config().
+In order to eagerly load the test configuration, set test_mode in airflow.cfg:
 
 .. code-block:: bash
 
   [tests]
   unit_test_mode = True
 
-Due to Airflow's automatic environment variable expansion (see :doc:`set-config`),
-you can also set the env var ``AIRFLOW__CORE__UNIT_TEST_MODE`` to temporarily overwrite
-airflow.cfg.
+Due to Airflow's automatic environment variable expansion (see :doc:`set-config`), you can also set the environment variable ``AIRFLOW__CORE__UNIT_TEST_MODE`` to temporarily overwrite airflow.cfg.
diff --git a/docs/howto/write-logs.rst b/docs/howto/write-logs.rst
index a29f6069ca089..b069c5a50a438 100644
--- a/docs/howto/write-logs.rst
+++ b/docs/howto/write-logs.rst
@@ -1,38 +1,55 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Writing Logs
 ============
 
 Writing Logs Locally
 --------------------
 
-Users can specify a logs folder in ``airflow.cfg`` using the
-``base_log_folder`` setting. By default, it is in the ``AIRFLOW_HOME``
+Users can specify the directory to place log files in ``airflow.cfg`` using
+``base_log_folder``. By default, logs are placed in the ``AIRFLOW_HOME``
 directory.
 
-In addition, users can supply a remote location for storing logs and log
-backups in cloud storage.
+The following convention is followed while naming logs: ``{dag_id}/{task_id}/{execution_date}/{try_number}.log``
+
+In addition, users can supply a remote location to store current logs and backups.
 
-In the Airflow Web UI, local logs take precedance over remote logs. If local logs
+In the Airflow Web UI, local logs take precedence over remote logs. If local logs
 can not be found or accessed, the remote logs will be displayed. Note that logs
-are only sent to remote storage once a task completes (including failure). In other
-words, remote logs for running tasks are unavailable. Logs are stored in the log
-folder as ``{dag_id}/{task_id}/{execution_date}/{try_number}.log``.
+are only sent to remote storage once a task is complete (including failure); In other words, remote logs for
+running tasks are unavailable.
+
+Before you begin
+''''''''''''''''
+
+Remote logging uses an existing Airflow connection to read or write logs. If you
+don't have a connection properly setup, this process will fail.
 
 .. _write-logs-amazon:
 
 Writing Logs to Amazon S3
 -------------------------
 
-Before you begin
-''''''''''''''''
-
-Remote logging uses an existing Airflow connection to read/write logs. If you
-don't have a connection properly setup, this will fail.
 
 Enabling remote logging
 '''''''''''''''''''''''
 
-To enable this feature, ``airflow.cfg`` must be configured as in this
-example:
+To enable this feature, ``airflow.cfg`` must be configured as follows:
 
 .. code-block:: bash
 
@@ -40,6 +57,7 @@ example:
     # Airflow can store logs remotely in AWS S3. Users must supply a remote
     # location URL (starting with either 's3://...') and an Airflow connection
     # id that provides access to the storage location.
+    remote_logging = True
     remote_base_log_folder = s3://my-bucket/path/to/logs
     remote_log_conn_id = MyS3Conn
     # Use server-side encryption for logs stored in S3
@@ -53,11 +71,12 @@ Writing Logs to Azure Blob Storage
 ----------------------------------
 
 Airflow can be configured to read and write task logs in Azure Blob Storage.
-Follow the steps below to enable Azure Blob Storage logging.
 
-#. Airflow's logging system requires a custom .py file to be located in the ``PYTHONPATH``, so that it's importable from Airflow. Start by creating a directory to store the config file. ``$AIRFLOW_HOME/config`` is recommended.
+Follow the steps below to enable Azure Blob Storage logging:
+
+#. Airflow's logging system requires a custom `.py` file to be located in the ``PYTHONPATH``, so that it's importable from Airflow. Start by creating a directory to store the config file, ``$AIRFLOW_HOME/config`` is recommended.
 #. Create empty files called ``$AIRFLOW_HOME/config/log_config.py`` and ``$AIRFLOW_HOME/config/__init__.py``.
-#. Copy the contents of ``airflow/config_templates/airflow_local_settings.py`` into the ``log_config.py`` file that was just created in the step above.
+#. Copy the contents of ``airflow/config_templates/airflow_local_settings.py`` into the ``log_config.py`` file created in `Step 2`.
 #. Customize the following portions of the template:
 
     .. code-block:: bash
@@ -89,71 +108,32 @@ Writing Logs to Google Cloud Storage
 
 Follow the steps below to enable Google Cloud Storage logging.
 
-#. Airflow's logging system requires a custom .py file to be located in the ``PYTHONPATH``, so that it's importable from Airflow. Start by creating a directory to store the config file. ``$AIRFLOW_HOME/config`` is recommended.
-#. Create empty files called ``$AIRFLOW_HOME/config/log_config.py`` and ``$AIRFLOW_HOME/config/__init__.py``.
-#. Copy the contents of ``airflow/config_templates/airflow_local_settings.py`` into the ``log_config.py`` file that was just created in the step above.
-#. Customize the following portions of the template:
-
-    .. code-block:: bash
-
-        # Add this variable to the top of the file. Note the trailing slash.
-        GCS_LOG_FOLDER = 'gs://<bucket where logs should be persisted>/'
-
-        # Rename DEFAULT_LOGGING_CONFIG to LOGGING CONFIG
-        LOGGING_CONFIG = ...
-
-        # Add a GCSTaskHandler to the 'handlers' block of the LOGGING_CONFIG variable
-        'gcs.task': {
-            'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler',
-            'formatter': 'airflow.task',
-            'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
-            'gcs_log_folder': GCS_LOG_FOLDER,
-            'filename_template': FILENAME_TEMPLATE,
-        },
-
-        # Update the airflow.task and airflow.tas_runner blocks to be 'gcs.task' instead of 'file.task'.
-        'loggers': {
-            'airflow.task': {
-                'handlers': ['gcs.task'],
-                ...
-            },
-            'airflow.task_runner': {
-                'handlers': ['gcs.task'],
-                ...
-            },
-            'airflow': {
-                'handlers': ['console'],
-                ...
-            },
-        }
-
-#. Make sure a Google Cloud Platform connection hook has been defined in Airflow. The hook should have read and write access to the Google Cloud Storage bucket defined above in ``GCS_LOG_FOLDER``.
-
-#. Update ``$AIRFLOW_HOME/airflow.cfg`` to contain:
-
-    .. code-block:: bash
+To enable this feature, ``airflow.cfg`` must be configured as in this
+example:
 
-        task_log_reader = gcs.task
-        logging_config_class = log_config.LOGGING_CONFIG
-        remote_log_conn_id = <name of the Google cloud platform hook>
+.. code-block:: bash
 
+    [core]
+    # Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
+    # Users must supply an Airflow connection id that provides access to the storage
+    # location. If remote_logging is set to true, see UPDATING.md for additional
+    # configuration requirements.
+    remote_logging = True
+    remote_base_log_folder = gs://my-bucket/path/to/logs
+    remote_log_conn_id = MyGCSConn
+
+#. Install the ``gcp_api`` package first, like so: ``pip install 'apache-airflow[gcp_api]'``.
+#. Make sure a Google Cloud Platform connection hook has been defined in Airflow. The hook should have read and write access to the Google Cloud Storage bucket defined above in ``remote_base_log_folder``.
 #. Restart the Airflow webserver and scheduler, and trigger (or wait for) a new task execution.
 #. Verify that logs are showing up for newly executed tasks in the bucket you've defined.
 #. Verify that the Google Cloud Storage viewer is working in the UI. Pull up a newly executed task, and verify that you see something like:
 
-    .. code-block:: bash
-
-        *** Reading remote log from gs://<bucket where logs should be persisted>/example_bash_operator/run_this_last/2017-10-03T00:00:00/16.log.
-        [2017-10-03 21:57:50,056] {cli.py:377} INFO - Running on host chrisr-00532
-        [2017-10-03 21:57:50,093] {base_task_runner.py:115} INFO - Running: ['bash', '-c', u'airflow run example_bash_operator run_this_last 2017-10-03T00:00:00 --job_id 47 --raw -sd DAGS_FOLDER/example_dags/example_bash_operator.py']
-        [2017-10-03 21:57:51,264] {base_task_runner.py:98} INFO - Subtask: [2017-10-03 21:57:51,263] {__init__.py:45} INFO - Using executor SequentialExecutor
-        [2017-10-03 21:57:51,306] {base_task_runner.py:98} INFO - Subtask: [2017-10-03 21:57:51,306] {models.py:186} INFO - Filling up the DagBag from /airflow/dags/example_dags/example_bash_operator.py
+.. code-block:: bash
 
-Note the top line that says it's reading from the remote log file.
+  *** Reading remote log from gs://<bucket where logs should be persisted>/example_bash_operator/run_this_last/2017-10-03T00:00:00/16.log.
+  [2017-10-03 21:57:50,056] {cli.py:377} INFO - Running on host chrisr-00532
+  [2017-10-03 21:57:50,093] {base_task_runner.py:115} INFO - Running: ['bash', '-c', u'airflow run example_bash_operator run_this_last 2017-10-03T00:00:00 --job_id 47 --raw -sd DAGS_FOLDER/example_dags/example_bash_operator.py']
+  [2017-10-03 21:57:51,264] {base_task_runner.py:98} INFO - Subtask: [2017-10-03 21:57:51,263] {__init__.py:45} INFO - Using executor SequentialExecutor
+  [2017-10-03 21:57:51,306] {base_task_runner.py:98} INFO - Subtask: [2017-10-03 21:57:51,306] {models.py:186} INFO - Filling up the DagBag from /airflow/dags/example_dags/example_bash_operator.py
 
-Please be aware that if you were persisting logs to Google Cloud Storage
-using the old-style airflow.cfg configuration method, the old logs will no
-longer be visible in the Airflow UI, though they'll still exist in Google
-Cloud Storage. This is a backwards incompatbile change. If you are unhappy
-with it, you can change the ``FILENAME_TEMPLATE`` to reflect the old-style
-log filename format.
+**Note** that the path to the remote log file is listed on the first line.
diff --git a/docs/img/airflow.gif b/docs/img/airflow.gif
index 1889b868f708a..7646e37f35ed5 100644
Binary files a/docs/img/airflow.gif and b/docs/img/airflow.gif differ
diff --git a/docs/img/branch_with_trigger.png b/docs/img/branch_with_trigger.png
new file mode 100644
index 0000000000000..0aab8a1b3a5c8
Binary files /dev/null and b/docs/img/branch_with_trigger.png differ
diff --git a/docs/img/branch_without_trigger.png b/docs/img/branch_without_trigger.png
new file mode 100644
index 0000000000000..748c796a4981f
Binary files /dev/null and b/docs/img/branch_without_trigger.png differ
diff --git a/docs/img/code.png b/docs/img/code.png
index ac49291f1c333..60e8f58a33821 100644
Binary files a/docs/img/code.png and b/docs/img/code.png differ
diff --git a/docs/img/context.png b/docs/img/context.png
index de75e487c01f4..bac718f1d1a26 100644
Binary files a/docs/img/context.png and b/docs/img/context.png differ
diff --git a/docs/img/dags.png b/docs/img/dags.png
index a551f02b1d2c2..04c82131f945c 100644
Binary files a/docs/img/dags.png and b/docs/img/dags.png differ
diff --git a/docs/img/duration.png b/docs/img/duration.png
index 18d723c005a48..b59fc092573a6 100644
Binary files a/docs/img/duration.png and b/docs/img/duration.png differ
diff --git a/docs/img/gantt.png b/docs/img/gantt.png
index c462adbdf4bdf..d74e1e5c3e5dd 100644
Binary files a/docs/img/gantt.png and b/docs/img/gantt.png differ
diff --git a/docs/img/graph.png b/docs/img/graph.png
index cbc58e6d8f146..ea2b7054e22e9 100644
Binary files a/docs/img/graph.png and b/docs/img/graph.png differ
diff --git a/docs/img/incubator.jpg b/docs/img/incubator.jpg
deleted file mode 100644
index 6f34a85e81c74..0000000000000
Binary files a/docs/img/incubator.jpg and /dev/null differ
diff --git a/docs/img/latest_only_with_trigger.png b/docs/img/latest_only_with_trigger.png
index 629adfa907964..623f8ee177ff0 100644
Binary files a/docs/img/latest_only_with_trigger.png and b/docs/img/latest_only_with_trigger.png differ
diff --git a/docs/img/scheduler_loop.jpg b/docs/img/scheduler_loop.jpg
new file mode 100644
index 0000000000000..a4a5e285e5440
Binary files /dev/null and b/docs/img/scheduler_loop.jpg differ
diff --git a/docs/img/tree.png b/docs/img/tree.png
index f3796b057249b..06750aa4cb202 100644
Binary files a/docs/img/tree.png and b/docs/img/tree.png differ
diff --git a/docs/index.rst b/docs/index.rst
index 4cd96eaa74e43..beb74119f7d83 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,24 +1,26 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
 
-.. image:: img/pin_large.png
-    :width: 100
-.. image:: img/incubator.jpg
-    :width: 150
+..    http://www.apache.org/licenses/LICENSE-2.0
 
-Apache Airflow (incubating) Documentation
-=========================================
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
 
-.. important::
 
-    **Disclaimer**: Apache Airflow is an effort undergoing incubation at The
-    Apache Software Foundation (ASF), sponsored by the Apache Incubator.
-    Incubation is required of all newly accepted projects until a further
-    review indicates that the infrastructure, communications, and
-    decision making process have stabilized in a manner consistent with
-    other successful ASF projects. While incubation status is not
-    necessarily a reflection of the completeness or stability of
-    the code, it does indicate that the project has yet to be fully
-    endorsed by the ASF.
+.. image:: img/pin_large.png
+    :width: 100
 
+Apache Airflow Documentation
+=========================================
 
 Airflow is a platform to programmatically author, schedule and monitor
 workflows.
@@ -86,6 +88,10 @@ Content
     timezone
     api
     integration
+    metrics
+    kubernetes
     lineage
+    changelog
     faq
-    code
+    macros
+    _api/index
diff --git a/docs/installation.rst b/docs/installation.rst
index 9e9ab3edc289d..6c2695e94167f 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Installation
 ------------
 
@@ -14,15 +31,7 @@ You can also install Airflow with support for extra features like ``s3`` or ``po
 
 .. code-block:: bash
 
-    pip install "apache-airflow[s3, postgres]"
-
-.. note:: GPL dependency
-
-    One of the dependencies of Apache Airflow by default pulls in a GPL library ('unidecode').
-    In case this is a concern you can force a non GPL library by issuing
-    ``export SLUGIFY_USES_TEXT_UNIDECODE=yes`` and then proceed with the normal installation.
-    Please note that this needs to be specified at every upgrade. Also note that if `unidecode`
-    is already present on the system the dependency will still be used.
+    pip install apache-airflow[postgres,s3]
 
 Extra Packages
 ''''''''''''''
@@ -38,70 +47,79 @@ these extra dependencies.
 
 Here's the list of the subpackages and what they enable:
 
-+---------------+----------------------------------------------+-------------------------------------------------+
-| subpackage    | install command                              | enables                                         |
-+===============+==============================================+=================================================+
-|  all          | ``pip install apache-airflow[all]``          | All Airflow features known to man               |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  all_dbs      | ``pip install apache-airflow[all_dbs]``      | All databases integrations                      |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  async        | ``pip install apache-airflow[async]``        | Async worker classes for gunicorn               |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  devel        | ``pip install apache-airflow[devel]``        | Minimum dev tools requirements                  |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  devel_hadoop | ``pip install apache-airflow[devel_hadoop]`` | Airflow + dependencies on the Hadoop stack      |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  celery       | ``pip install apache-airflow[celery]``       | CeleryExecutor                                  |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  crypto       | ``pip install apache-airflow[crypto]``       | Encrypt connection passwords in metadata db     |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  druid        | ``pip install apache-airflow[druid]``        | Druid.io related operators & hooks              |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  gcp_api      | ``pip install apache-airflow[gcp_api]``      | Google Cloud Platform hooks and operators       |
-|               |                                              | (using ``google-api-python-client``)            |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  jdbc         | ``pip install apache-airflow[jdbc]``         | JDBC hooks and operators                        |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  hdfs         | ``pip install apache-airflow[hdfs]``         | HDFS hooks and operators                        |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  hive         | ``pip install apache-airflow[hive]``         | All Hive related operators                      |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  kerberos     | ``pip install apache-airflow[kerberos]``     | kerberos integration for kerberized hadoop      |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  ldap         | ``pip install apache-airflow[ldap]``         | ldap authentication for users                   |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  mssql        | ``pip install apache-airflow[mssql]``        | Microsoft SQL operators and hook,               |
-|               |                                              | support as an Airflow backend                   |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  mysql        | ``pip install apache-airflow[mysql]``        | MySQL operators and hook, support as            |
-|               |                                              | an Airflow backend. The version of MySQL server |
-|               |                                              | has to be 5.6.4+. The exact version upper bound |
-|               |                                              | depends on version of ``mysqlclient`` package.  |
-|               |                                              | For example, ``mysqlclient`` 1.3.12 can only be |
-|               |                                              | used with MySQL server 5.6.4 through 5.7.       |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  password     | ``pip install apache-airflow[password]``     | Password Authentication for users               |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  postgres     | ``pip install apache-airflow[postgres]``     | Postgres operators and hook, support            |
-|               |                                              | as an Airflow backend                           |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  qds          | ``pip install apache-airflow[qds]``          | Enable QDS (qubole data services) support       |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  rabbitmq     | ``pip install apache-airflow[rabbitmq]``     | Rabbitmq support as a Celery backend            |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  s3           | ``pip install apache-airflow[s3]``           | ``S3KeySensor``, ``S3PrefixSensor``             |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  samba        | ``pip install apache-airflow[samba]``        | ``Hive2SambaOperator``                          |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  slack        | ``pip install apache-airflow[slack]``        | ``SlackAPIPostOperator``                        |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  vertica      | ``pip install apache-airflow[vertica]``      | Vertica hook                                    |
-|               |                                              | support as an Airflow backend                   |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  cloudant     | ``pip install apache-airflow[cloudant]``     | Cloudant hook                                   |
-+---------------+----------------------------------------------+-------------------------------------------------+
-|  redis        | ``pip install apache-airflow[redis]``        | Redis hooks and sensors                         |
-+---------------+----------------------------------------------+-------------------------------------------------+
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| subpackage          | install command                                     | enables                                                              |
++=====================+=====================================================+======================================================================+
+| all                 | ``pip install 'apache-airflow[all]'``               | All Airflow features known to man                                    |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| all_dbs             | ``pip install 'apache-airflow[all_dbs]'``           | All databases integrations                                           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| async               | ``pip install 'apache-airflow[async]'``             | Async worker classes for Gunicorn                                    |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| celery              | ``pip install 'apache-airflow[celery]'``            | CeleryExecutor                                                       |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| cloudant            | ``pip install 'apache-airflow[cloudant]'``          | Cloudant hook                                                        |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| crypto              | ``pip install 'apache-airflow[crypto]'``            | Encrypt connection passwords in metadata db                          |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| devel               | ``pip install 'apache-airflow[devel]'``             | Minimum dev tools requirements                                       |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| devel_hadoop        | ``pip install 'apache-airflow[devel_hadoop]'``      | Airflow + dependencies on the Hadoop stack                           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| druid               | ``pip install 'apache-airflow[druid]'``             | Druid related operators & hooks                                      |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| gcp_api             | ``pip install 'apache-airflow[gcp_api]'``           | Google Cloud Platform hooks and operators                            |
+|                     |                                                     | (using ``google-api-python-client``)                                 |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| github_enterprise   | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend                                       |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| google_auth         | ``pip install 'apache-airflow[google_auth]'``       | Google auth backend                                                  |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| hdfs                | ``pip install 'apache-airflow[hdfs]'``              | HDFS hooks and operators                                             |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| hive                | ``pip install 'apache-airflow[hive]'``              | All Hive related operators                                           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| jdbc                | ``pip install 'apache-airflow[jdbc]'``              | JDBC hooks and operators                                             |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| kerberos            | ``pip install 'apache-airflow[kerberos]'``          | Kerberos integration for Kerberized Hadoop                           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| kubernetes          | ``pip install 'apache-airflow[kubernetes]'``        | Kubernetes Executor and operator                                     |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| ldap                | ``pip install 'apache-airflow[ldap]'``              | LDAP authentication for users                                        |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| mssql               | ``pip install 'apache-airflow[mssql]'``             | Microsoft SQL Server operators and hook,                             |
+|                     |                                                     | support as an Airflow backend                                        |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| mysql               | ``pip install 'apache-airflow[mysql]'``             | MySQL operators and hook, support as an Airflow                      |
+|                     |                                                     | backend. The version of MySQL server has to be                       |
+|                     |                                                     | 5.6.4+. The exact version upper bound depends                        |
+|                     |                                                     | on version of ``mysqlclient`` package. For                           |
+|                     |                                                     | example, ``mysqlclient`` 1.3.12 can only be                          |
+|                     |                                                     | used with MySQL server 5.6.4 through 5.7.                            |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| oracle              | ``pip install 'apache-airflow[oracle]'``            | Oracle hooks and operators                                           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| password            | ``pip install 'apache-airflow[password]'``          | Password authentication for users                                    |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| postgres            | ``pip install 'apache-airflow[postgres]'``          | PostgreSQL operators and hook, support as an                         |
+|                     |                                                     | Airflow backend                                                      |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| qds                 | ``pip install 'apache-airflow[qds]'``               | Enable QDS (Qubole Data Service) support                             |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| rabbitmq            | ``pip install 'apache-airflow[rabbitmq]'``          | RabbitMQ support as a Celery backend                                 |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| redis               | ``pip install 'apache-airflow[redis]'``             | Redis hooks and sensors                                              |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| s3                  | ``pip install 'apache-airflow[s3]'``                | ``S3KeySensor``, ``S3PrefixSensor``                                  |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| samba               | ``pip install apache-airflow[samba]'``              | :class:`airflow.operators.hive_to_samba_operator.Hive2SambaOperator` |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| slack               | ``pip install 'apache-airflow[slack']``             | :class:`airflow.operators.slack_operator.SlackAPIOperator`           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| ssh                 | ``pip install 'apache-airflow[ssh]'``               | SSH hooks and Operator                                               |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| vertica             | ``pip install 'apache-airflow[vertica]'``           | Vertica hook support as an Airflow backend                           |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
 
 Initiating Airflow Database
 '''''''''''''''''''''''''''
diff --git a/docs/integration.rst b/docs/integration.rst
index 660b2163d9849..f172b00e67e9e 100644
--- a/docs/integration.rst
+++ b/docs/integration.rst
@@ -1,69 +1,28 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Integration
 ===========
 
-- :ref:`ReverseProxy`
 - :ref:`Azure`
 - :ref:`AWS`
 - :ref:`Databricks`
 - :ref:`GCP`
-
-.. _ReverseProxy:
-
-Reverse Proxy
--------------
-
-Airflow can be set up behind a reverse proxy, with the ability to set its endpoint with great
-flexibility.
-
-For example, you can configure your reverse proxy to get:
-
-::
-
-    https://lab.mycompany.com/myorg/airflow/
-
-To do so, you need to set the following setting in your `airflow.cfg`::
-
-    base_url = http://my_host/myorg/airflow
-
-Additionally if you use Celery Executor, you can get Flower in `/myorg/flower` with::
-
-    flower_url_prefix = /myorg/flower
-
-Your reverse proxy (ex: nginx) should be configured as follow:
-
-- pass the url and http header as it for the Airflow webserver, without any rewrite, for example::
-
-      server {
-        listen 80;
-        server_name lab.mycompany.com;
-
-        location /myorg/airflow/ {
-            proxy_pass http://localhost:8080;
-            proxy_set_header Host $host;
-            proxy_redirect off;
-            proxy_http_version 1.1;
-            proxy_set_header Upgrade $http_upgrade;
-            proxy_set_header Connection "upgrade";
-        }
-      }
-
-- rewrite the url for the flower endpoint::
-
-      server {
-          listen 80;
-          server_name lab.mycompany.com;
-
-          location /myorg/flower/ {
-              rewrite ^/myorg/flower/(.*)$ /$1 break;  # remove prefix from http header
-              proxy_pass http://localhost:5555;
-              proxy_set_header Host $host;
-              proxy_redirect off;
-              proxy_http_version 1.1;
-              proxy_set_header Upgrade $http_upgrade;
-              proxy_set_header Connection "upgrade";
-          }
-      }
-
+- :ref:`Qubole`
 
 .. _Azure:
 
@@ -82,51 +41,32 @@ Airflow connection of type `wasb` exists. Authorization can be done by supplying
 login (=Storage account name) and password (=KEY), or login and SAS token in the extra
 field (see connection `wasb_default` for an example).
 
-- :ref:`WasbBlobSensor`: Checks if a blob is present on Azure Blob storage.
-- :ref:`WasbPrefixSensor`: Checks if blobs matching a prefix are present on Azure Blob storage.
-- :ref:`FileToWasbOperator`: Uploads a local file to a container as a blob.
-- :ref:`WasbHook`: Interface with Azure Blob Storage.
-
-.. _WasbBlobSensor:
-
-WasbBlobSensor
-"""""""""""""""
-
-.. autoclass:: airflow.contrib.sensors.wasb_sensor.WasbBlobSensor
+:class:`airflow.contrib.hooks.wasb_hook.WasbHook`
+    Interface with Azure Blob Storage.
 
-.. _WasbPrefixSensor:
+:class:`airflow.contrib.sensors.wasb_sensor.WasbBlobSensor`
+    Checks if a blob is present on Azure Blob storage.
 
-WasbPrefixSensor
-"""""""""""""""""
+:class:`airflow.contrib.operators.wasb_delete_blob_operator.WasbDeleteBlobOperator`
+    Deletes blob(s) on Azure Blob Storage.
 
-.. autoclass:: airflow.contrib.sensors.wasb_sensor.WasbPrefixSensor
+:class:`airflow.contrib.sensors.wasb_sensor.WasbPrefixSensor`
+    Checks if blobs matching a prefix are present on Azure Blob storage.
 
-.. _FileToWasbOperator:
+:class:`airflow.contrib.operators.file_to_wasb.FileToWasbOperator`
+    Uploads a local file to a container as a blob.
 
-FileToWasbOperator
-"""""""""""""""""""
-
-.. autoclass:: airflow.contrib.operators.file_to_wasb.FileToWasbOperator
-
-.. _WasbHook:
-
-WasbHook
-"""""""""
-
-.. autoclass:: airflow.contrib.hooks.wasb_hook.WasbHook
 
 Azure File Share
 ''''''''''''''''
 
-Cloud variant of a SMB file share. Make sure that a Airflow connection of 
-type `wasb` exists. Authorization can be done by supplying a login (=Storage account name) 
-and password (=Storage account key), or login and SAS token in the extra field 
+Cloud variant of a SMB file share. Make sure that a Airflow connection of
+type `wasb` exists. Authorization can be done by supplying a login (=Storage account name)
+and password (=Storage account key), or login and SAS token in the extra field
 (see connection `wasb_default` for an example).
 
-AzureFileShareHook
-""""""""""""""""""
-
-.. autoclass:: airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook
+:class:`airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook`:
+    Interface with Azure File Share.
 
 Logging
 '''''''
@@ -134,174 +74,186 @@ Logging
 Airflow can be configured to read and write task logs in Azure Blob Storage.
 See :ref:`write-logs-azure`.
 
-Azure Data Lake
-''''''''''''''''''
-
-AzureDataLakeHook communicates via a REST API compatible with WebHDFS. Make sure that a
-Airflow connection of type `azure_data_lake` exists. Authorization can be done by supplying a
-login (=Client ID), password (=Client Secret) and extra fields tenant (Tenant) and account_name (Account Name)
- (see connection `azure_data_lake_default` for an example).
+Azure CosmosDB
+''''''''''''''
 
-- :ref:`AzureDataLakeHook`: Interface with Azure Data Lake.
+AzureCosmosDBHook communicates via the Azure Cosmos library. Make sure that a
+Airflow connection of type `azure_cosmos` exists. Authorization can be done by supplying a
+login (=Endpoint uri), password (=secret key) and extra fields database_name and collection_name to specify the
+default database and collection to use (see connection `azure_cosmos_default` for an example).
 
-.. _AzureDataLakeHook:
+:class:`airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook`
+    Interface with Azure CosmosDB.
 
-AzureDataLakeHook
-"""""""""
+:class:`airflow.contrib.operators.azure_cosmos_operator.AzureCosmosInsertDocumentOperator`
+    Simple operator to insert document into CosmosDB.
 
-.. autoclass:: airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook
+:class:`airflow.contrib.sensors.azure_cosmos_sensor.AzureCosmosDocumentSensor`
+    Simple sensor to detect document existence in CosmosDB.
 
-.. _AWS:
 
-AWS: Amazon Web Services
-------------------------
+Azure Data Lake
+'''''''''''''''
 
-Airflow has extensive support for Amazon Web Services. But note that the Hooks, Sensors and
-Operators are in the contrib section.
+AzureDataLakeHook communicates via a REST API compatible with WebHDFS. Make sure that a
+Airflow connection of type `azure_data_lake` exists. Authorization can be done by supplying a
+login (=Client ID), password (=Client Secret) and extra fields tenant (Tenant) and account_name (Account Name)
+(see connection `azure_data_lake_default` for an example).
 
-AWS EMR
-''''''''
+:class:`airflow.contrib.hooks.azure_data_lake_hook.AzureDataLakeHook`
+    Interface with Azure Data Lake.
 
-- :ref:`EmrAddStepsOperator` : Adds steps to an existing EMR JobFlow.
-- :ref:`EmrCreateJobFlowOperator` : Creates an EMR JobFlow, reading the config from the EMR connection.
-- :ref:`EmrTerminateJobFlowOperator` : Terminates an EMR JobFlow.
-- :ref:`EmrHook` : Interact with AWS EMR.
+:class:`airflow.contrib.operators.adls_list_operator.AzureDataLakeStorageListOperator`
+    Lists the files located in a specified Azure Data Lake path.
 
-.. _EmrAddStepsOperator:
+:class:`airflow.contrib.operators.adls_to_gcs.AdlsToGoogleCloudStorageOperator`
+    Copies files from an Azure Data Lake path to a Google Cloud Storage bucket.
 
-EmrAddStepsOperator
-"""""""""""""""""""
 
-.. autoclass:: airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator
+Azure Container Instances
+'''''''''''''''''''''''''
 
-.. _EmrCreateJobFlowOperator:
+Azure Container Instances provides a method to run a docker container without having to worry
+about managing infrastructure. The AzureContainerInstanceHook requires a service principal. The
+credentials for this principal can either be defined in the extra field ``key_path``, as an
+environment variable named ``AZURE_AUTH_LOCATION``,
+or by providing a login/password and tenantId in extras.
 
-EmrCreateJobFlowOperator
-""""""""""""""""""""""""
+The AzureContainerRegistryHook requires a host/login/password to be defined in the connection.
 
-.. autoclass:: airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator
+:class:`airflow.contrib.hooks.azure_container_volume_hook.AzureContainerVolumeHook`
+    Interface with Azure Container Volumes
 
-.. _EmrTerminateJobFlowOperator:
+:class:`airflow.contrib.operators.azure_container_instances_operator.AzureContainerInstancesOperator`
+    Start/Monitor a new ACI.
 
-EmrTerminateJobFlowOperator
-"""""""""""""""""""""""""""
+:class:`airflow.contrib.hooks.azure_container_instance_hook.AzureContainerInstanceHook`
+    Wrapper around a single ACI.
 
-.. autoclass:: airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator
+:class:`airflow.contrib.hooks.azure_container_registry_hook.AzureContainerRegistryHook`
+    Interface with ACR
 
-.. _EmrHook:
 
-EmrHook
-"""""""
 
-.. autoclass:: airflow.contrib.hooks.emr_hook.EmrHook
+.. _AWS:
 
+AWS: Amazon Web Services
+------------------------
 
-AWS S3
-''''''
+Airflow has extensive support for Amazon Web Services. But note that the Hooks, Sensors and
+Operators are in the contrib section.
 
-- :ref:`S3Hook` : Interact with AWS S3.
-- :ref:`S3FileTransformOperator` : Copies data from a source S3 location to a temporary location on the local filesystem.
-- :ref:`S3ListOperator` : Lists the files matching a key prefix from a S3 location.
-- :ref:`S3ToGoogleCloudStorageOperator` : Syncs an S3 location with a Google Cloud Storage bucket.
-- :ref:`S3ToHiveTransfer` : Moves data from S3 to Hive. The operator downloads a file from S3, stores the file locally before loading it into a Hive table.
+AWS EMR
+'''''''
 
-.. _S3Hook:
+:class:`airflow.contrib.hooks.emr_hook.EmrHook`
+    Interface with AWS EMR.
 
-S3Hook
-""""""
+:class:`airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator`
+    Adds steps to an existing EMR JobFlow.
 
-.. autoclass:: airflow.hooks.S3_hook.S3Hook
+:class:`airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator`
+    Creates an EMR JobFlow, reading the config from the EMR connection.
 
-.. _S3FileTransformOperator:
+:class:`airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator`
+    Terminates an EMR JobFlow.
 
-S3FileTransformOperator
-"""""""""""""""""""""""
 
-.. autoclass:: airflow.operators.s3_file_transform_operator.S3FileTransformOperator
+AWS S3
+''''''
 
-.. _S3ListOperator:
+:class:`airflow.hooks.S3_hook.S3Hook`
+    Interface with AWS S3.
 
-S3ListOperator
-""""""""""""""
+:class:`airflow.operators.s3_file_transform_operator.S3FileTransformOperator`
+    Copies data from a source S3 location to a temporary location on the local filesystem.
 
-.. autoclass:: airflow.contrib.operators.s3_list_operator.S3ListOperator
+:class:`airflow.contrib.operators.s3_list_operator.S3ListOperator`
+    Lists the files matching a key prefix from a S3 location.
 
-.. _S3ToGoogleCloudStorageOperator:
+:class:`airflow.contrib.operators.s3_to_gcs_operator.S3ToGoogleCloudStorageOperator`
+    Syncs an S3 location with a Google Cloud Storage bucket.
 
-S3ToGoogleCloudStorageOperator
-""""""""""""""""""""""""""""""
+:class:`airflow.contrib.operators.s3_to_gcs_transfer_operator.S3ToGoogleCloudStorageTransferOperator`
+    Syncs an S3 bucket with a Google Cloud Storage bucket using the GCP Storage Transfer Service.
 
-.. autoclass:: airflow.contrib.operators.s3_to_gcs_operator.S3ToGoogleCloudStorageOperator
+:class:`airflow.operators.s3_to_hive_operator.S3ToHiveTransfer`
+    Moves data from S3 to Hive. The operator downloads a file from S3, stores the file locally before loading it into a Hive table.
 
-.. _S3ToHiveTransfer:
 
-S3ToHiveTransfer
-""""""""""""""""
+AWS Batch Service
+'''''''''''''''''
 
-.. autoclass:: airflow.operators.s3_to_hive_operator.S3ToHiveTransfer
+:class:`airflow.contrib.operators.awsbatch_operator.AWSBatchOperator`
+    Execute a task on AWS Batch Service.
 
 
-AWS EC2 Container Service
-'''''''''''''''''''''''''
+AWS RedShift
+''''''''''''
 
-- :ref:`ECSOperator` : Execute a task on AWS EC2 Container Service.
+:class:`airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor`
+    Waits for a Redshift cluster to reach a specific status.
 
-.. _ECSOperator:
+:class:`airflow.contrib.hooks.redshift_hook.RedshiftHook`
+    Interact with AWS Redshift, using the boto3 library.
 
-ECSOperator
-"""""""""""
+:class:`airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer`
+    Executes an unload command to S3 as CSV with or without headers.
 
-.. autoclass:: airflow.contrib.operators.ecs_operator.ECSOperator
+:class:`airflow.operators.s3_to_redshift_operator.S3ToRedshiftTransfer`
+    Executes an copy command from S3 as CSV with or without headers.
 
 
-AWS Batch Service
-'''''''''''''''''
 
-- :ref:`AWSBatchOperator` : Execute a task on AWS Batch Service.
+AWS DynamoDB
+''''''''''''
 
-.. _AWSBatchOperator:
+:class:`airflow.contrib.operators.hive_to_dynamodb.HiveToDynamoDBTransferOperator`
+     Moves data from Hive to DynamoDB.
 
-AWSBatchOperator
-""""""""""""""""
+:class:`airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook`
+    Interface with AWS DynamoDB.
 
-.. autoclass:: airflow.contrib.operators.awsbatch_operator.AWSBatchOperator
 
+AWS Lambda
+''''''''''
 
-AWS RedShift
-''''''''''''
+:class:`airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook`
+    Interface with AWS Lambda.
 
-- :ref:`AwsRedshiftClusterSensor` : Waits for a Redshift cluster to reach a specific status.
-- :ref:`RedshiftHook` : Interact with AWS Redshift, using the boto3 library.
-- :ref:`RedshiftToS3Transfer` : Executes an unload command to S3 as CSV with or without headers.
-- :ref:`S3ToRedshiftTransfer` : Executes an copy command from S3 as CSV with or without headers.
 
-.. _AwsRedshiftClusterSensor:
+AWS Kinesis
+'''''''''''
 
-AwsRedshiftClusterSensor
-""""""""""""""""""""""""
+:class:`airflow.contrib.hooks.aws_firehose_hook.AwsFirehoseHook`
+    Interface with AWS Kinesis Firehose.
 
-.. autoclass:: airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor
+Amazon SageMaker
+''''''''''''''''
 
-.. _RedshiftHook:
+.. _the SageMaker Python SDK README: https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/workflow/README.rst
 
-RedshiftHook
-""""""""""""
+:class:`airflow.contrib.hooks.sagemaker_hook.SageMakerHook`
+    Interface with Amazon SageMaker.
 
-.. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook
+:class:`airflow.contrib.operators.sagemaker_training_operator.SageMakerTrainingOperator`
+    Create a SageMaker training job.
 
-.. _RedshiftToS3Transfer:
+:class:`airflow.contrib.operators.sagemaker_tuning_operator.SageMakerTuningOperator`
+    Create a SageMaker tuning job.
 
-RedshiftToS3Transfer
-""""""""""""""""""""
+:class:`airflow.contrib.operators.sagemaker_model_operator.SageMakerModelOperator`
+    Create a SageMaker model.
 
-.. autoclass:: airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer
+:class:`airflow.contrib.operators.sagemaker_transform_operator.SageMakerTransformOperator`
+    Create a SageMaker transform job.
 
-.. _S3ToRedshiftTransfer:
+:class:`airflow.contrib.operators.sagemaker_endpoint_config_operator.SageMakerEndpointConfigOperator`
+    Create a SageMaker endpoint config.
 
-S3ToRedshiftTransfer
-""""""""""""""""""""
+:class:`airflow.contrib.operators.sagemaker_endpoint_operator.SageMakerEndpointOperator`
+    Create a SageMaker endpoint.
 
-.. autoclass:: airflow.operators.s3_to_redshift_operator.S3ToRedshiftTransfer
 
 
 .. _Databricks:
@@ -313,11 +265,12 @@ Databricks
 submitting runs to the Databricks platform. Internally the operator talks to the
 ``api/2.0/jobs/runs/submit`` `endpoint <https://docs.databricks.com/api/latest/jobs.html#runs-submit>`_.
 
-DatabricksSubmitRunOperator
-'''''''''''''''''''''''''''
-
-.. autoclass:: airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator
 
+:class:`airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator`
+    Submits a Spark job run to Databricks using the
+    `api/2.0/jobs/runs/submit
+    <https://docs.databricks.com/api/latest/jobs.html#runs-submit>`_
+    API endpoint.
 
 
 .. _GCP:
@@ -329,7 +282,7 @@ Airflow has extensive support for the Google Cloud Platform. But note that most
 Operators are in the contrib section. Meaning that they have a *beta* status, meaning that
 they can have breaking changes between minor releases.
 
-See the :ref:`GCP connection type <connection-type-GCP>` documentation to
+See the :doc:`GCP connection type <howto/connection/gcp>` documentation to
 configure connections to GCP.
 
 Logging
@@ -338,438 +291,432 @@ Logging
 Airflow can be configured to read and write task logs in Google Cloud Storage.
 See :ref:`write-logs-gcp`.
 
-BigQuery
-''''''''
-
-BigQuery Operators
-""""""""""""""""""
-
-- :ref:`BigQueryCheckOperator` : Performs checks against a SQL query that will return a single row with different values.
-- :ref:`BigQueryValueCheckOperator` : Performs a simple value check using SQL code.
-- :ref:`BigQueryIntervalCheckOperator` : Checks that the values of metrics given as SQL expressions are within a certain tolerance of the ones from days_back before.
-- :ref:`BigQueryCreateEmptyTableOperator` : Creates a new, empty table in the specified BigQuery dataset optionally with schema.
-- :ref:`BigQueryCreateExternalTableOperator` : Creates a new, external table in the dataset with the data in Google Cloud Storage.
-- :ref:`BigQueryOperator` : Executes BigQuery SQL queries in a specific BigQuery database.
-- :ref:`BigQueryToBigQueryOperator` : Copy a BigQuery table to another BigQuery table.
-- :ref:`BigQueryToCloudStorageOperator` : Transfers a BigQuery table to a Google Cloud Storage bucket
-
-
-.. _BigQueryCheckOperator:
-
-BigQueryCheckOperator
-^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryCheckOperator
+GoogleCloudBaseHook
+'''''''''''''''''''
 
-.. _BigQueryValueCheckOperator:
+All hooks is based on :class:`airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook`.
 
-BigQueryValueCheckOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator
-
-.. _BigQueryIntervalCheckOperator:
-
-BigQueryIntervalCheckOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+BigQuery
+''''''''
 
-.. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator
+:class:`airflow.contrib.operators.bigquery_check_operator.BigQueryCheckOperator`
+    Performs checks against a SQL query that will return a single row with different values.
 
-.. _BigQueryGetDataOperator:
+:class:`airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator`
+    Checks that the values of metrics given as SQL expressions are within a certain tolerance of the ones from days_back before.
 
-BigQueryGetDataOperator
-^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator`
+    Performs a simple value check using SQL code.
 
-.. autoclass:: airflow.contrib.operators.bigquery_get_data.BigQueryGetDataOperator
+:class:`airflow.contrib.operators.bigquery_get_data.BigQueryGetDataOperator`
+    Fetches the data from a BigQuery table and returns data in a python list
 
-.. _BigQueryCreateEmptyTableOperator:
+:class:`airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyDatasetOperator`
+    Creates an empty BigQuery dataset.
 
-BigQueryCreateEmptyTableOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator`
+    Creates a new, empty table in the specified BigQuery dataset optionally with schema.
 
-.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator
+:class:`airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator`
+    Creates a new, external table in the dataset with the data in Google Cloud Storage.
 
-.. _BigQueryCreateExternalTableOperator:
+:class:`airflow.contrib.operators.bigquery_operator.BigQueryDeleteDatasetOperator`
+    Deletes an existing BigQuery dataset.
 
-BigQueryCreateExternalTableOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.bigquery_operator.BigQueryOperator`
+    Executes BigQuery SQL queries in a specific BigQuery database.
 
-.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator
+:class:`airflow.contrib.operators.bigquery_table_delete_operator.BigQueryTableDeleteOperator`
+    Deletes an existing BigQuery table.
 
-.. _BigQueryOperator:
+:class:`airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator`
+    Copy a BigQuery table to another BigQuery table.
 
-BigQueryOperator
-^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator`
+    Transfers a BigQuery table to a Google Cloud Storage bucket
 
-.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryOperator
 
-.. _BigQueryTableDeleteOperator:
+They also use :class:`airflow.contrib.hooks.bigquery_hook.BigQueryHook` to communicate with Google Cloud Platform.
 
-BigQueryTableDeleteOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: airflow.contrib.operators.bigquery_table_delete_operator.BigQueryTableDeleteOperator
+Cloud Spanner
+'''''''''''''
 
-.. _BigQueryToBigQueryOperator:
+:class:`airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeleteOperator`
+    deletes an existing database from a Google Cloud Spanner instance or returns success if the database is missing.
 
-BigQueryToBigQueryOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeployOperator`
+    creates a new database in a Google Cloud instance or returns success if the database already exists.
 
-.. autoclass:: airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator
+:class:`airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseQueryOperator`
+    executes an arbitrary DML query (INSERT, UPDATE, DELETE).
 
-.. _BigQueryToCloudStorageOperator:
+:class:`airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseUpdateOperator`
+    updates the structure of a Google Cloud Spanner database.
 
-BigQueryToCloudStorageOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDeleteOperator`
+    deletes a Google Cloud Spanner instance.
 
-.. autoclass:: airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator
+:class:`airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDeployOperator`
+    creates a new Google Cloud Spanner instance, or if an instance with the same name exists, updates the instance.
 
 
-BigQueryHook
-""""""""""""
+They also use :class:`airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook` to communicate with Google Cloud Platform.
 
-.. autoclass:: airflow.contrib.hooks.bigquery_hook.BigQueryHook
-    :members:
 
+Cloud SQL
+'''''''''
 
-Cloud DataFlow
-''''''''''''''
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator`
+    create a new Cloud SQL instance.
 
-DataFlow Operators
-""""""""""""""""""
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator`
+    creates a new database inside a Cloud SQL instance.
 
-- :ref:`DataFlowJavaOperator` : launching Cloud Dataflow jobs written in Java.
-- :ref:`DataflowTemplateOperator` : launching a templated Cloud DataFlow batch job.
-- :ref:`DataFlowPythonOperator` : launching Cloud Dataflow jobs written in python.
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator`
+    deletes a database from a Cloud SQL instance.
 
-.. _DataFlowJavaOperator:
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator`
+    updates a database inside a Cloud SQL instance.
 
-DataFlowJavaOperator
-^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator`
+    delete a Cloud SQL instance.
 
-.. autoclass:: airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceExportOperator`
+    exports data from a Cloud SQL instance.
 
-.. code:: python
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceImportOperator`
+    imports data into a Cloud SQL instance.
 
-    default_args = {
-        'owner': 'airflow',
-        'depends_on_past': False,
-        'start_date':
-            (2016, 8, 1),
-        'email': ['alex@vanboxel.be'],
-        'email_on_failure': False,
-        'email_on_retry': False,
-        'retries': 1,
-        'retry_delay': timedelta(minutes=30),
-        'dataflow_default_options': {
-            'project': 'my-gcp-project',
-            'zone': 'us-central1-f',
-            'stagingLocation': 'gs://bucket/tmp/dataflow/staging/',
-        }
-    }
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator`
+    patch a Cloud SQL instance.
 
-    dag = DAG('test-dag', default_args=default_args)
+:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator`
+    run query in a Cloud SQL instance.
 
-    task = DataFlowJavaOperator(
-        gcp_conn_id='gcp_default',
-        task_id='normalize-cal',
-        jar='{{var.value.gcp_dataflow_base}}pipeline-ingress-cal-normalize-1.0.jar',
-        options={
-            'autoscalingAlgorithm': 'BASIC',
-            'maxNumWorkers': '50',
-            'start': '{{ds}}',
-            'partitionType': 'DAY'
 
-        },
-        dag=dag)
+They also use :class:`airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook` and :class:`airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook` to communicate with Google Cloud Platform.
 
-.. _DataflowTemplateOperator:
 
-DataflowTemplateOperator
-^^^^^^^^^^^^^^^^^^^^^^^^
+Cloud Bigtable
+''''''''''''''
 
-.. autoclass:: airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator
+:class:`airflow.contrib.operators.gcp_bigtable_operator.BigtableClusterUpdateOperator`
+    updates the number of nodes in a Google Cloud Bigtable cluster.
 
-.. _DataFlowPythonOperator:
+:class:`airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceCreateOperator`
+    creates a Cloud Bigtable instance.
 
-DataFlowPythonOperator
-^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceDeleteOperator`
+    deletes a Google Cloud Bigtable instance.
 
-.. autoclass:: airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator
+:class:`airflow.contrib.operators.gcp_bigtable_operator.BigtableTableCreateOperator`
+    creates a table in a Google Cloud Bigtable instance.
 
+:class:`airflow.contrib.operators.gcp_bigtable_operator.BigtableTableDeleteOperator`
+    deletes a table in a Google Cloud Bigtable instance.
 
-DataFlowHook
-""""""""""""
+:class:`airflow.contrib.operators.gcp_bigtable_operator.BigtableTableWaitForReplicationSensor`
+    (sensor) waits for a table to be fully replicated.
 
-.. autoclass:: airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook
-    :members:
 
+They also use :class:`airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook` to communicate with Google Cloud Platform.
 
 
-Cloud DataProc
+Compute Engine
 ''''''''''''''
 
-DataProc Operators
-""""""""""""""""""
+:class:`airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator`
+    start an existing Google Compute Engine instance.
 
-- :ref:`DataprocClusterCreateOperator` : Create a new cluster on Google Cloud Dataproc.
-- :ref:`DataprocClusterDeleteOperator` : Delete a cluster on Google Cloud Dataproc.
-- :ref:`DataprocClusterScaleOperator` : Scale up or down a cluster on Google Cloud Dataproc.
-- :ref:`DataProcPigOperator` : Start a Pig query Job on a Cloud DataProc cluster.
-- :ref:`DataProcHiveOperator` : Start a Hive query Job on a Cloud DataProc cluster.
-- :ref:`DataProcSparkSqlOperator` : Start a Spark SQL query Job on a Cloud DataProc cluster.
-- :ref:`DataProcSparkOperator` : Start a Spark Job on a Cloud DataProc cluster.
-- :ref:`DataProcHadoopOperator` : Start a Hadoop Job on a Cloud DataProc cluster.
-- :ref:`DataProcPySparkOperator` : Start a PySpark Job on a Cloud DataProc cluster.
-- :ref:`DataprocWorkflowTemplateInstantiateOperator` : Instantiate a WorkflowTemplate on Google Cloud Dataproc.
-- :ref:`DataprocWorkflowTemplateInstantiateInlineOperator` : Instantiate a WorkflowTemplate Inline on Google Cloud Dataproc.
+:class:`airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator`
+    stop an existing Google Compute Engine instance.
 
-.. _DataprocClusterCreateOperator:
+:class:`airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator`
+    change the machine type for a stopped instance.
 
-DataprocClusterCreateOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_compute_operator.GceInstanceTemplateCopyOperator`
+    copy the Instance Template, applying specified changes.
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator
+:class:`airflow.contrib.operators.gcp_compute_operator.GceInstanceGroupManagerUpdateTemplateOperator`
+    patch the Instance Group Manager, replacing source Instance Template URL with the destination one.
 
-.. _DataprocClusterScaleOperator:
 
-DataprocClusterScaleOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The operators have the common base operator :class:`airflow.contrib.operators.gcp_compute_operator.GceBaseOperator`
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator
+They also use :class:`airflow.contrib.hooks.gcp_compute_hook.GceHook` to communicate with Google Cloud Platform.
 
-.. _DataprocClusterDeleteOperator:
 
-DataprocClusterDeleteOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Cloud Functions
+'''''''''''''''
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator
+:class:`airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator`
+    deploy Google Cloud Function to Google Cloud Platform
 
-.. _DataProcPigOperator:
+:class:`airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator`
+    delete Google Cloud Function in Google Cloud Platform
 
-DataProcPigOperator
-^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcPigOperator
+They also use :class:`airflow.contrib.hooks.gcp_function_hook.GcfHook` to communicate with Google Cloud Platform.
 
-.. _DataProcHiveOperator:
 
-DataProcHiveOperator
-^^^^^^^^^^^^^^^^^^^^
+Cloud DataFlow
+''''''''''''''
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHiveOperator
+:class:`airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator`
+    launching Cloud Dataflow jobs written in Java.
 
-.. _DataProcSparkSqlOperator:
+:class:`airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator`
+    launching a templated Cloud DataFlow batch job.
 
-DataProcSparkSqlOperator
-^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator`
+    launching Cloud Dataflow jobs written in python.
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator
 
-.. _DataProcSparkOperator:
+They also use :class:`airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook` to communicate with Google Cloud Platform.
 
-DataProcSparkOperator
-^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkOperator
+Cloud DataProc
+''''''''''''''
 
-.. _DataProcHadoopOperator:
+:class:`airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator`
+    Create a new cluster on Google Cloud Dataproc.
 
-DataProcHadoopOperator
-^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator`
+    Delete a cluster on Google Cloud Dataproc.
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator
+:class:`airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator`
+    Scale up or down a cluster on Google Cloud Dataproc.
 
-.. _DataProcPySparkOperator:
+:class:`airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator`
+    Start a Hadoop Job on a Cloud DataProc cluster.
 
-DataProcPySparkOperator
-^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.dataproc_operator.DataProcHiveOperator`
+    Start a Hive query Job on a Cloud DataProc cluster.
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator
+:class:`airflow.contrib.operators.dataproc_operator.DataProcPigOperator`
+    Start a Pig query Job on a Cloud DataProc cluster.
 
-.. _DataprocWorkflowTemplateInstantiateOperator:
+:class:`airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator`
+    Start a PySpark Job on a Cloud DataProc cluster.
 
-DataprocWorkflowTemplateInstantiateOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.dataproc_operator.DataProcSparkOperator`
+    Start a Spark Job on a Cloud DataProc cluster.
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateOperator
+:class:`airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator`
+    Start a Spark SQL query Job on a Cloud DataProc cluster.
 
-.. _DataprocWorkflowTemplateInstantiateInlineOperator:
+:class:`airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateInlineOperator`
+    Instantiate a WorkflowTemplate Inline on Google Cloud Dataproc.
 
-DataprocWorkflowTemplateInstantiateInlineOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateOperator`
+    Instantiate a WorkflowTemplate on Google Cloud Dataproc.
 
-.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateInlineOperator
 
 Cloud Datastore
 '''''''''''''''
 
-Datastore Operators
-"""""""""""""""""""
+:class:`airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator`
+    Export entities from Google Cloud Datastore to Cloud Storage.
 
-- :ref:`DatastoreExportOperator` : Export entities from Google Cloud Datastore to Cloud Storage.
-- :ref:`DatastoreImportOperator` : Import entities from Cloud Storage to Google Cloud Datastore.
+:class:`airflow.contrib.operators.datastore_import_operator.DatastoreImportOperator`
+    Import entities from Cloud Storage to Google Cloud Datastore.
 
-.. _DatastoreExportOperator:
 
-DatastoreExportOperator
-^^^^^^^^^^^^^^^^^^^^^^^
-
-.. autoclass:: airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator
-
-.. _DatastoreImportOperator:
-
-DatastoreImportOperator
-^^^^^^^^^^^^^^^^^^^^^^^
-
-.. autoclass:: airflow.contrib.operators.datastore_import_operator.DatastoreImportOperator
-
-DatastoreHook
-"""""""""""""
-
-.. autoclass:: airflow.contrib.hooks.datastore_hook.DatastoreHook
-    :members:
+They also use :class:`airflow.contrib.hooks.datastore_hook.DatastoreHook` to communicate with Google Cloud Platform.
 
 
 Cloud ML Engine
 '''''''''''''''
 
-Cloud ML Engine Operators
-"""""""""""""""""""""""""
-
-- :ref:`MLEngineBatchPredictionOperator` : Start a Cloud ML Engine batch prediction job.
-- :ref:`MLEngineModelOperator` : Manages a Cloud ML Engine model.
-- :ref:`MLEngineTrainingOperator` : Start a Cloud ML Engine training job.
-- :ref:`MLEngineVersionOperator` : Manages a Cloud ML Engine model version.
+:class:`airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator`
+    Start a Cloud ML Engine batch prediction job.
 
-.. _MLEngineBatchPredictionOperator:
+:class:`airflow.contrib.operators.mlengine_operator.MLEngineModelOperator`
+    Manages a Cloud ML Engine model.
 
-MLEngineBatchPredictionOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator`
+    Start a Cloud ML Engine training job.
 
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator
-    :members:
+:class:`airflow.contrib.operators.mlengine_operator.MLEngineVersionOperator`
+    Manages a Cloud ML Engine model version.
 
-.. _MLEngineModelOperator:
 
-MLEngineModelOperator
-^^^^^^^^^^^^^^^^^^^^^
+They also use :class:`airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook` to communicate with Google Cloud Platform.
 
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineModelOperator
-    :members:
 
-.. _MLEngineTrainingOperator:
+Cloud Storage
+'''''''''''''
 
-MLEngineTrainingOperator
-^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator`
+    Uploads a file to Google Cloud Storage.
 
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator
-    :members:
+:class:`airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator`
+    Creates a new ACL entry on the specified bucket.
 
-.. _MLEngineVersionOperator:
+:class:`airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator`
+    Creates a new ACL entry on the specified object.
 
-MLEngineVersionOperator
-^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator`
+    Downloads a file from Google Cloud Storage.
 
-.. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineVersionOperator
-    :members:
+:class:`airflow.contrib.operators.gcs_list_operator.GoogleCloudStorageListOperator`
+    List all objects from the bucket with the give string prefix and delimiter in name.
 
-Cloud ML Engine Hook
-""""""""""""""""""""
+:class:`airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator`
+    Creates a new cloud storage bucket.
 
-.. _MLEngineHook:
+:class:`airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator`
+    Loads files from Google cloud storage into BigQuery.
 
-MLEngineHook
-^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator`
+    Copies objects from a bucket to another, with renaming if requested.
 
-.. autoclass:: airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook
-    :members:
+:class:`airflow.contrib.operators.mysql_to_gcs.MySqlToGoogleCloudStorageOperator`
+    Copy data from any MySQL Database to Google cloud storage in JSON format.
 
 
-Cloud Storage
-'''''''''''''
+They also use :class:`airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook` to communicate with Google Cloud Platform.
 
-Storage Operators
-"""""""""""""""""
 
-- :ref:`FileToGoogleCloudStorageOperator` : Uploads a file to Google Cloud Storage.
-- :ref:`GoogleCloudStorageCreateBucketOperator` : Creates a new cloud storage bucket.
-- :ref:`GoogleCloudStorageListOperator` : List all objects from the bucket with the give string prefix and delimiter in name.
-- :ref:`GoogleCloudStorageDownloadOperator` : Downloads a file from Google Cloud Storage.
-- :ref:`GoogleCloudStorageToBigQueryOperator` : Loads files from Google cloud storage into BigQuery.
-- :ref:`GoogleCloudStorageToGoogleCloudStorageOperator` : Copies objects from a bucket to another, with renaming if requested.
-
-.. _FileToGoogleCloudStorageOperator:
-
-FileToGoogleCloudStorageOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Transfer Service
+''''''''''''''''
 
-.. autoclass:: airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceJobDeleteOperator`
+    Deletes a transfer job.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceJobCreateOperator`
+    Creates a transfer job.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceJobUpdateOperator`
+    Updates a transfer job.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceOperationCancelOperator`
+    Cancels a transfer operation.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceOperationGetOperator`
+    Gets a transfer operation.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceOperationPauseOperator`
+    Pauses a transfer operation
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceOperationResumeOperator`
+    Resumes a transfer operation.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceOperationsListOperator`
+    Gets a list of transfer operations.
+:class:`airflow.contrib.operators.gcp_transfer_operator.GoogleCloudStorageToGoogleCloudStorageTransferOperator`
+    Copies objects from a Google Cloud Storage bucket to another bucket.
+:class:`airflow.contrib.operators.gcp_transfer_operator.S3ToGoogleCloudStorageTransferOperator`
+    Synchronizes an S3 bucket with a Google Cloud Storage bucket.
+
+
+:class:`airflow.contrib.sensors.gcp_transfer_operator.GCPTransferServiceWaitForJobStatusSensor`
+    Waits for at least one operation belonging to the job to have the
+    expected status.
+
+
+They also use :class:`airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook` to communicate with Google Cloud Platform.
+
+
+Cloud Vision
+''''''''''''
 
-.. _GoogleCloudStorageCreateBucketOperator:
+Cloud Vision Product Search Operators
+"""""""""""""""""""""""""""""""""""""
+
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionAddProductToProductSetOperator`
+    Adds a Product to the specified ProductSet.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionAnnotateImageOperator`
+    Run image detection and annotation for an image.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductCreateOperator`
+    Creates a new Product resource.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductDeleteOperator`
+    Permanently deletes a product and its reference images.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductGetOperator`
+    Gets information associated with a Product.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator`
+    Creates a new ProductSet resource.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetDeleteOperator`
+    Permanently deletes a ProductSet.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetGetOperator`
+    Gets information associated with a ProductSet.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetUpdateOperator`
+    Makes changes to a ProductSet resource.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionProductUpdateOperator`
+    Makes changes to a Product resource.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionReferenceImageCreateOperator`
+    Creates a new ReferenceImage resource.
+:class:`airflow.contrib.operators.gcp_vision_operator.CloudVisionRemoveProductFromProductSetOperator`
+    Removes a Product from the specified ProductSet.
+
+They also use :class:`airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook` to communicate with Google Cloud Platform.
+
+Cloud Translate
+'''''''''''''''
 
-GoogleCloudStorageCreateBucketOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Cloud Translate Text Operators
+""""""""""""""""""""""""""""""
 
-.. autoclass:: airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator
+:class:`airflow.contrib.operators.gcp_translate_operator.CloudTranslateTextOperator`
+    Translate a string or list of strings.
 
-.. _GoogleCloudStorageDownloadOperator:
 
-GoogleCloudStorageDownloadOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Google Kubernetes Engine
+''''''''''''''''''''''''
 
-.. autoclass:: airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator
+:class:`airflow.contrib.operators.gcp_container_operator.GKEClusterCreateOperator`
+    Creates a Kubernetes Cluster in Google Cloud Platform
 
-.. _GoogleCloudStorageListOperator:
+:class:`airflow.contrib.operators.gcp_container_operator.GKEClusterDeleteOperator`
+    Deletes a Kubernetes Cluster in Google Cloud Platform
 
-GoogleCloudStorageListOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_container_operator.GKEPodOperator`
+    Executes a task in a Kubernetes pod in the specified Google Kubernetes Engine cluster
 
-.. autoclass:: airflow.contrib.operators.gcs_list_operator.GoogleCloudStorageListOperator
+They also use :class:`airflow.contrib.hooks.gcp_container_hook.GKEClusterHook` to communicate with Google Cloud Platform.
 
-.. _GoogleCloudStorageToBigQueryOperator:
 
-GoogleCloudStorageToBigQueryOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Google Natural Language
+'''''''''''''''''''''''
 
-.. autoclass:: airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator
+:class:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntities`
+    Finds named entities (currently proper names and common nouns) in the text along with entity types,
+    salience, mentions for each entity, and other properties.
 
-.. _GoogleCloudStorageToGoogleCloudStorageOperator:
+:class:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitySentiment`
+    Finds entities, similar to AnalyzeEntities in the text and analyzes sentiment associated with each
+    entity and its mentions.
 
-GoogleCloudStorageToGoogleCloudStorageOperator
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentiment`
+    Analyzes the sentiment of the provided text.
 
-.. autoclass:: airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator
+:class:`airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageClassifyTextOperator`
+    Classifies a document into categories.
 
-GoogleCloudStorageHook
-""""""""""""""""""""""
+They also use :class:`airflow.contrib.hooks.gcp_natural_language_operator.CloudNaturalLanguageHook` to communicate with Google Cloud Platform.
 
-.. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook
-    :members:
 
-Google Kubernetes Engine
-''''''''''''''''''''''''
+.. _Qubole:
 
-Google Kubernetes Engine Cluster Operators
-""""""""""""""""""""""""""""""""""""""""""
+Qubole
+------
 
-- :ref:`GKEClusterCreateOperator` : Creates a Kubernetes Cluster in Google Cloud Platform
-- :ref:`GKEClusterDeleteOperator` : Deletes a Kubernetes Cluster in Google Cloud Platform
+Apache Airflow has a native operator and hooks to talk to `Qubole <https://qubole.com/>`__,
+which lets you submit your big data jobs directly to Qubole from Apache Airflow.
 
-GKEClusterCreateOperator
-^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. autoclass:: airflow.contrib.operators.gcp_container_operator.GKEClusterCreateOperator
-.. _GKEClusterCreateOperator:
+:class:`airflow.contrib.operators.qubole_operator.QuboleOperator`
+    Execute tasks (commands) on QDS (https://qubole.com).
 
-GKEClusterDeleteOperator
-^^^^^^^^^^^^^^^^^^^^^^^^
+:class:`airflow.contrib.sensors.qubole_sensor.QubolePartitionSensor`
+    Wait for a Hive partition to show up in QHS (Qubole Hive Service)
+    and check for its presence via QDS APIs
 
-.. autoclass:: airflow.contrib.operators.gcp_container_operator.GKEClusterDeleteOperator
-.. _GKEClusterDeleteOperator:
+:class:`airflow.contrib.sensors.qubole_sensor.QuboleFileSensor`
+    Wait for a file or folder to be present in cloud storage
+    and check for its presence via QDS APIs
 
-Google Kubernetes Engine Hook
-"""""""""""""""""""""""""""""
+:class:`airflow.contrib.operators.qubole_check_operator.QuboleCheckOperator`
+    Performs checks against Qubole Commands. ``QuboleCheckOperator`` expects
+    a command that will be executed on QDS.
 
-.. autoclass:: airflow.contrib.hooks.gcp_container_hook.GKEClusterHook
-    :members:
+:class:`airflow.contrib.operators.qubole_check_operator.QuboleValueCheckOperator`
+    Performs a simple value check using Qubole command.
+    By default, each value on the first row of this
+    Qubole command is compared with a pre-defined value
diff --git a/docs/kubernetes.rst b/docs/kubernetes.rst
index a4916858fc15e..b3efd336aa060 100644
--- a/docs/kubernetes.rst
+++ b/docs/kubernetes.rst
@@ -1,11 +1,42 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Kubernetes
+----------
+
 Kubernetes Executor
 ^^^^^^^^^^^^^^^^^^^
 
 The kubernetes executor is introduced in Apache Airflow 1.10.0. The Kubernetes executor will create a new pod for every task instance.
 
-Example helm charts are available at `scripts/ci/kubernetes/kube/{airflow,volumes,postgres}.yaml` in the source distribution. The volumes are optional and depend on your configuration. There are two volumes available:
-- Dags: by storing all the dags onto the persistent disks, all the workers can read the dags from there. Another option is using git-sync, before starting the container, a git pull of the dags repository will be performed and used throughout the lifecycle of the pod/
-- Logs: by storing the logs onto a persistent disk, all the logs will be available for all the workers and the webserver itself. If you don't configure this, the logs will be lost after the worker pods shuts down. Another option is to use S3/GCS/etc to store the logs.
+Example helm charts are available at ``scripts/ci/kubernetes/kube/{airflow,volumes,postgres}.yaml`` in the source distribution.
+The volumes are optional and depend on your configuration. There are two volumes available:
+
+- **Dags**:
+
+  - By storing dags onto persistent disk, it will be made available to all workers
+
+  - Another option is to use ``git-sync``. Before starting the container, a git pull of the dags repository will be performed and used throughout the lifecycle of the pod
+
+- **Logs**:
+
+  - By storing logs onto a persistent disk, the files are accessible by workers and the webserver. If you don't configure this, the logs will be lost after the worker pods shuts down
+
+  - Another option is to use S3/GCS/etc to store logs
 
 
 Kubernetes Operator
@@ -19,11 +50,14 @@ Kubernetes Operator
 
     secret_file = Secret('volume', '/etc/sql_conn', 'airflow-secrets', 'sql_alchemy_conn')
     secret_env  = Secret('env', 'SQL_CONN', 'airflow-secrets', 'sql_alchemy_conn')
+    secret_all_keys  = Secret('env', None, 'airflow-secrets-2')
     volume_mount = VolumeMount('test-volume',
                                 mount_path='/root/mount_file',
                                 sub_path=None,
                                 read_only=True)
 
+    configmaps = ['test-configmap-1', 'test-configmap-2']
+
     volume_config= {
         'persistentVolumeClaim':
           {
@@ -38,11 +72,11 @@ Kubernetes Operator
             {
               "weight": 1,
               "preference": {
-                "matchExpressions": [
+                "matchExpressions": {
                   "key": "disktype",
                   "operator": "In",
                   "values": ["ssd"]
-                ]
+                }
               }
             }
           ]
@@ -81,21 +115,30 @@ Kubernetes Operator
         }
     }
 
+    tolerations = [
+        {
+            'key': "key",
+            'operator': 'Equal',
+            'value': 'value'
+         }
+    ]
+
     k = KubernetesPodOperator(namespace='default',
                               image="ubuntu:16.04",
                               cmds=["bash", "-cx"],
                               arguments=["echo", "10"],
                               labels={"foo": "bar"},
-                              secrets=[secret_file,secret_env]
-                              volume=[volume],
+                              secrets=[secret_file, secret_env, secret_all_keys],
+                              volumes=[volume],
                               volume_mounts=[volume_mount]
                               name="test",
                               task_id="task",
-                              affinity=affinity
+                              affinity=affinity,
+                              is_delete_operator_pod=True,
+                              hostnetwork=False,
+                              tolerations=tolerations,
+                              configmaps=configmaps
                               )
 
 
-.. autoclass:: airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator
-
-.. autoclass:: airflow.contrib.kubernetes.secret.Secret
-
+See :class:`airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator`
diff --git a/docs/license.rst b/docs/license.rst
index f920a01658bb7..bcb2b76bf3de6 100644
--- a/docs/license.rst
+++ b/docs/license.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 License
 =======
 
@@ -6,9 +23,9 @@ License
 
 ::
 
-    Apache License
-    Version 2.0, January 2004
-    http://www.apache.org/licenses/
+                                  Apache License
+                            Version 2.0, January 2004
+                         http://www.apache.org/licenses/
 
     TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
@@ -180,37 +197,3 @@ License
        defend, and hold each Contributor harmless for any liability
        incurred by, or claims asserted against, such Contributor by reason
        of your accepting any such warranty or additional liability.
-
-    END OF TERMS AND CONDITIONS
-
-    APPENDIX: How to apply the Apache License to your work.
-
-       To apply the Apache License to your work, attach the following
-       boilerplate notice, with the fields enclosed by brackets "[]"
-       replaced with your own identifying information. (Don't include
-       the brackets!)  The text should be enclosed in the appropriate
-       comment syntax for the file format. We also recommend that a
-       file or class name and description of purpose be included on the
-       same "printed page" as the copyright notice for easier
-       identification within third-party archives.
-
-    Copyright 2015 Apache Software Foundation
-
-    Licensed to the Apache Software Foundation (ASF) under one
-    or more contributor license agreements.  See the NOTICE file
-    distributed with this work for additional information
-    regarding copyright ownership.  The ASF licenses this file
-    to you under the Apache License, Version 2.0 (the
-    "License"); you may not use this file except in compliance
-    with the License.  You may obtain a copy of the License at
-    
-      http://www.apache.org/licenses/LICENSE-2.0
-    
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-    KIND, either express or implied.  See the License for the
-    specific language governing permissions and limitations
-    under the License.
-    Status API Training Shop Blog About
-    © 2016 GitHub, Inc. Terms Privacy Security Contact Help
diff --git a/docs/lineage.rst b/docs/lineage.rst
index 719ef0115eb0c..8aa7cac2bb9e6 100644
--- a/docs/lineage.rst
+++ b/docs/lineage.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Lineage
 =======
 
@@ -47,10 +64,19 @@ works.
     run_this.set_downstream(run_this_last)
 
 
-Tasks take the parameters `inlets` and `outlets`. Inlets can be manually defined by a list of dataset `{"datasets":
-[dataset1, dataset2]}` or can be configured to look for outlets from upstream tasks `{"task_ids": ["task_id1", "task_id2"]}`
-or can be configured to pick up outlets from direct upstream tasks `{"auto": True}` or a combination of them. Outlets 
-are defined as list of dataset `{"datasets": [dataset1, dataset2]}`. Any fields for the dataset are templated with 
+Tasks take the parameters `inlets` and `outlets`.
+
+Inlets can be manually defined by the following options:
+
+- by a list of dataset ``{"datasets": [dataset1, dataset2]}``
+
+- can be configured to look for outlets from upstream tasks ``{"task_ids": ["task_id1", "task_id2"]}``
+
+- can be configured to pick up outlets from direct upstream tasks ``{"auto": True}``
+
+- a combination of them
+
+Outlets are defined as list of dataset ``{"datasets": [dataset1, dataset2]}``. Any fields for the dataset are templated with
 the context when the task is being executed. 
 
 .. note:: Operators can add inlets and outlets automatically if the operator supports it.
@@ -68,7 +94,7 @@ Apache Atlas
 ------------
 
 Airflow can send its lineage metadata to Apache Atlas. You need to enable the `atlas` backend and configure it 
-properly, e.g. in your `airflow.cfg`:
+properly, e.g. in your ``airflow.cfg``:
 
 .. code:: python
 
diff --git a/docs/macros.rst b/docs/macros.rst
new file mode 100644
index 0000000000000..43807b0ee37a7
--- /dev/null
+++ b/docs/macros.rst
@@ -0,0 +1,118 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Macros reference
+================
+
+Variables and macros can be used in templates (see the :ref:`jinja-templating` section)
+
+The following come for free out of the box with Airflow.
+Additional custom macros can be added globally through :ref:`plugins`, or at a DAG level through the ``DAG.user_defined_macros`` argument.
+
+Default Variables
+-----------------
+The Airflow engine passes a few variables by default that are accessible
+in all templates
+
+=================================   ====================================
+Variable                            Description
+=================================   ====================================
+``{{ ds }}``                        the execution date as ``YYYY-MM-DD``
+``{{ ds_nodash }}``                 the execution date as ``YYYYMMDD``
+``{{ prev_ds }}``                   the previous execution date as ``YYYY-MM-DD``
+                                    if ``{{ ds }}`` is ``2018-01-08`` and ``schedule_interval`` is ``@weekly``,
+                                    ``{{ prev_ds }}`` will be ``2018-01-01``
+``{{ prev_ds_nodash }}``            the previous execution date as ``YYYYMMDD`` if exists, else ``None``
+``{{ next_ds }}``                   the next execution date as ``YYYY-MM-DD``
+                                    if ``{{ ds }}`` is ``2018-01-01`` and ``schedule_interval`` is ``@weekly``,
+                                    ``{{ next_ds }}`` will be ``2018-01-08``
+``{{ next_ds_nodash }}``            the next execution date as ``YYYYMMDD`` if exists, else ``None``
+``{{ yesterday_ds }}``              the day before the execution date as ``YYYY-MM-DD``
+``{{ yesterday_ds_nodash }}``       the day before the execution date as ``YYYYMMDD``
+``{{ tomorrow_ds }}``               the day after the execution date as ``YYYY-MM-DD``
+``{{ tomorrow_ds_nodash }}``        the day after the execution date as ``YYYYMMDD``
+``{{ ts }}``                        same as ``execution_date.isoformat()``. Example: ``2018-01-01T00:00:00+00:00``
+``{{ ts_nodash }}``                 same as ``ts`` without ``-``, ``:`` and TimeZone info. Example: ``20180101T000000``
+``{{ ts_nodash_with_tz }}``         same as ``ts`` without ``-`` and ``:``. Example: ``20180101T000000+0000``
+``{{ execution_date }}``            the execution_date (pendulum.Pendulum)
+``{{ prev_execution_date }}``       the previous execution date (if available) (pendulum.Pendulum)
+``{{ next_execution_date }}``       the next execution date (pendulum.Pendulum)
+``{{ dag }}``                       the DAG object
+``{{ task }}``                      the Task object
+``{{ macros }}``                    a reference to the macros package, described below
+``{{ task_instance }}``             the task_instance object
+``{{ end_date }}``                  same as ``{{ ds }}``
+``{{ latest_date }}``               same as ``{{ ds }}``
+``{{ ti }}``                        same as ``{{ task_instance }}``
+``{{ params }}``                    a reference to the user-defined params dictionary which can be overridden by
+                                    the dictionary passed through ``trigger_dag -c`` if you enabled
+                                    ``dag_run_conf_overrides_params` in ``airflow.cfg``
+``{{ var.value.my_var }}``          global defined variables represented as a dictionary
+``{{ var.json.my_var.path }}``      global defined variables represented as a dictionary
+                                    with deserialized JSON object, append the path to the
+                                    key within the JSON object
+``{{ task_instance_key_str }}``     a unique, human-readable key to the task instance
+                                    formatted ``{dag_id}_{task_id}_{ds}``
+``{{ conf }}``                      the full configuration object located at
+                                    ``airflow.configuration.conf`` which
+                                    represents the content of your
+                                    ``airflow.cfg``
+``{{ run_id }}``                    the ``run_id`` of the current DAG run
+``{{ dag_run }}``                   a reference to the DagRun object
+``{{ test_mode }}``                 whether the task instance was called using
+                                    the CLI's test subcommand
+=================================   ====================================
+
+Note that you can access the object's attributes and methods with simple
+dot notation. Here are some examples of what is possible:
+``{{ task.owner }}``, ``{{ task.task_id }}``, ``{{ ti.hostname }}``, ...
+Refer to the models documentation for more information on the objects'
+attributes and methods.
+
+The ``var`` template variable allows you to access variables defined in Airflow's
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+``{{ var.json.my_dict_var.key1 }}``
+
+Macros
+------
+Macros are a way to expose objects to your templates and live under the
+``macros`` namespace in your templates.
+
+A few commonly used libraries and methods are made available.
+
+
+=================================   ==============================================
+Variable                            Description
+=================================   ==============================================
+``macros.datetime``                 The standard lib's :class:`datetime.datetime`
+``macros.timedelta``                The standard lib's :class:`datetime.datetime`
+``macros.dateutil``                 A reference to the ``dateutil`` package
+``macros.time``                     The standard lib's :class:`datetime.time`
+``macros.uuid``                     The standard lib's :mod:`uuid`
+``macros.random``                   The standard lib's :mod:`random`
+=================================   ==============================================
+
+
+Some airflow specific macros are also defined:
+
+.. automodule:: airflow.macros
+    :show-inheritance:
+    :members:
+
+.. autofunction:: airflow.macros.hive.closest_ds_partition
+.. autofunction:: airflow.macros.hive.max_partition
diff --git a/docs/metrics.rst b/docs/metrics.rst
new file mode 100644
index 0000000000000..fc0c82e4d59ca
--- /dev/null
+++ b/docs/metrics.rst
@@ -0,0 +1,83 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Metrics
+=======
+
+Airflow can be set up to send metrics to `StatsD <https://github.com/etsy/statsd>`__.
+
+Setup
+-----
+
+First you must install statsd requirement:
+
+.. code-block:: bash
+
+   pip install 'apache-airflow[statsd]'
+
+Add the following lines to your configuration file e.g. ``airflow.cfg``
+
+.. code-block:: bash
+
+    [scheduler]
+    statsd_on = True
+    statsd_host = localhost
+    statsd_port = 8125
+    statsd_prefix = airflow
+
+Counters
+--------
+
+=================================== ================================================================
+Name                                Description
+=================================== ================================================================
+<job_name>_start                    Number of started <job_name> job, ex. SchedulerJob, LocalTaskJob
+<job_name>_end                      Number of ended <job_name> job, ex. SchedulerJob, LocalTaskJob
+operator_failures_<operator_name>   Operator <operator_name> failures
+operator_successes_<operator_name>  Operator <operator_name> successes
+ti_failures                         Overall task instances failures
+ti_successes                        Overall task instances successes
+zombies_killed                      Zombie tasks killed
+scheduler_heartbeat                 Scheduler heartbeats
+=================================== ================================================================
+
+Gauges
+------
+
+=============================================== ========================================================================
+Name                                            Description
+=============================================== ========================================================================
+collect_dags                                    Seconds taken to scan and import DAGs
+dagbag_import_errors                            DAG import errors
+dagbag_size                                     DAG bag size
+dag_processing.last_runtime.<dag_file>          Seconds spent processing <dag_file> (in most recent iteration)
+dag_processing.last_run.seconds_ago.<dag_file>  Seconds since <dag_file> was last processed
+executor.open_slots                             Number of of open slots on executor
+executor.queued_tasks                           Number of queued tasks on executor
+executor.running_tasks                          Number of running tasks on executor
+pool.starving_tasks.<pool_name>                 Number of starving tasks in the pool
+=============================================== ========================================================================
+
+Timers
+------
+
+================================= =======================================
+Name                              Description
+================================= =======================================
+dagrun.dependency-check.<dag_id>  Seconds taken to check DAG dependencies
+dag.<dag_id>.<task_id>.duration   Seconds taken to finish a task
+================================= =======================================
diff --git a/docs/plugins.rst b/docs/plugins.rst
index a21f1007ff686..51040b07302ff 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Plugins
 =======
 
@@ -72,10 +89,37 @@ looks like:
         # A list of objects created from a class derived
         # from flask_admin.BaseView
         admin_views = []
-        # A list of Blueprint object created from flask.Blueprint
+        # A list of Blueprint object created from flask.Blueprint. For use with the flask_admin based GUI
         flask_blueprints = []
-        # A list of menu links (flask_admin.base.MenuLink)
+        # A list of menu links (flask_admin.base.MenuLink). For use with the flask_admin based GUI
         menu_links = []
+        # A list of dictionaries containing FlaskAppBuilder BaseView object and some metadata. See example below
+        appbuilder_views = []
+        # A list of dictionaries containing FlaskAppBuilder BaseView object and some metadata. See example below
+        appbuilder_menu_items = []
+
+
+
+You can derive it by inheritance (please refer to the example below).
+Please note ``name`` inside this class must be specified.
+
+After the plugin is imported into Airflow,
+you can invoke it using statement like
+
+
+.. code:: python
+
+    from airflow.{type, like "operators", "sensors"}.{name specificed inside the plugin class} import *
+
+
+When you write your own plugins, make sure you understand them well.
+There are some essential properties for each type of plugin.
+For example,
+
+* For ``Operator`` plugin, an ``execute`` method is compulsory.
+* For ``Sensor`` plugin, a ``poke`` method returning a Boolean value is compulsory.
+
+Make sure you restart the webserver and scheduler after making changes to plugins so that they take effect.
 
 
 Example
@@ -116,6 +160,7 @@ definitions in Airflow.
         pass
 
     # Will show up under airflow.macros.test_plugin.plugin_macro
+    # and in templates through {{ macros.test_plugin.plugin_macro }}
     def plugin_macro():
         pass
 
@@ -137,7 +182,23 @@ definitions in Airflow.
     ml = MenuLink(
         category='Test Plugin',
         name='Test Menu Link',
-        url='https://airflow.incubator.apache.org/')
+        url='https://airflow.apache.org/')
+
+    # Creating a flask appbuilder BaseView
+    class TestAppBuilderBaseView(AppBuilderBaseView):
+        @expose("/")
+        def test(self):
+            return self.render("test_plugin/test.html", content="Hello galaxy!")
+    v_appbuilder_view = TestAppBuilderBaseView()
+    v_appbuilder_package = {"name": "Test View",
+                            "category": "Test Plugin",
+                            "view": v_appbuilder_view}
+
+    # Creating a flask appbuilder Menu Item
+    appbuilder_mitem = {"name": "Google",
+                        "category": "Search",
+                        "category_icon": "fa-th",
+                        "href": "https://www.google.com"}
 
     # Defining the plugin class
     class AirflowTestPlugin(AirflowPlugin):
@@ -150,3 +211,65 @@ definitions in Airflow.
         admin_views = [v]
         flask_blueprints = [bp]
         menu_links = [ml]
+        appbuilder_views = [v_appbuilder_package]
+        appbuilder_menu_items = [appbuilder_mitem]
+
+
+Note on role based views
+------------------------
+
+Airflow 1.10 introduced role based views using FlaskAppBuilder. You can configure which UI is used by setting
+rbac = True. To support plugin views and links for both versions of the UI and maintain backwards compatibility,
+the fields appbuilder_views and appbuilder_menu_items were added to the AirflowTestPlugin class.
+
+
+Plugins as Python packages
+--------------------------
+
+It is possible to load plugins via `setuptools entrypoint <https://packaging.python.org/guides/creating-and-discovering-plugins/#using-package-metadata>`_ mechanism. To do this link
+your plugin using an entrypoint in your package. If the package is installed, airflow
+will automatically load the registered plugins from the entrypoint list.
+
+_Note_: Neither the entrypoint name (eg, `my_plugin`) nor the name of the
+plugin class will contribute towards the module and class name of the plugin
+itself. The structure is determined by
+`airflow.plugins_manager.AirflowPlugin.name` and the class name of the plugin
+component with the pattern `airflow.{component}.{name}.{component_class_name}`.
+
+.. code-block:: python
+
+    # my_package/my_plugin.py
+    from airflow.plugins_manager import AirflowPlugin
+    from airflow.models import BaseOperator
+    from airflow.hooks.base_hook import BaseHook
+
+    class MyOperator(BaseOperator):
+      pass
+
+    class MyHook(BaseHook):
+      pass
+
+    class MyAirflowPlugin(AirflowPlugin):
+      name = 'my_namespace'
+      operators = [MyOperator]
+      hooks = [MyHook]
+
+
+.. code-block:: python
+
+    from setuptools import setup
+
+    setup(
+        name="my-package",
+        ...
+        entry_points = {
+            'airflow.plugins': [
+                'my_plugin = my_package.my_plugin:MyAirflowPlugin'
+            ]
+        }
+    )
+
+
+This will create a hook, and an operator accessible at:
+ - `airflow.hooks.my_namespace.MyHook`
+ - `airflow.operators.my_namespace.MyOperator`
diff --git a/docs/profiling.rst b/docs/profiling.rst
index 09102334b1c29..c4f1c0b4869af 100644
--- a/docs/profiling.rst
+++ b/docs/profiling.rst
@@ -1,3 +1,22 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. TODO: This section would be removed after we migrate to www_rbac completely.
+
 Data Profiling
 ==============
 
diff --git a/docs/project.rst b/docs/project.rst
index d1f2cc010c939..8c8d93cc09b25 100644
--- a/docs/project.rst
+++ b/docs/project.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Project
 =======
 
@@ -6,9 +23,12 @@ History
 
 Airflow was started in October 2014 by Maxime Beauchemin at Airbnb.
 It was open source from the very first commit and officially brought under
-the Airbnb Github and announced in June 2015.
+the Airbnb GitHub and announced in June 2015.
 
-The project joined the Apache Software Foundation's incubation program in March 2016.
+The project joined the Apache Software Foundation's Incubator program in March 2016 and the
+Foundation `announced Apache Airflow as a Top-Level Project
+<https://blogs.apache.org/foundation/entry/the-apache-software-foundation-announces44>`_
+in January 2019.
 
 
 Committers
@@ -20,9 +40,7 @@ Committers
 - @bolkedebruin (Bolke de Bruin)
 - @artwr (Arthur Wiedmer)
 - @jlowin (Jeremiah Lowin)
-- @patrickleotardif (Patrick Leo Tardif)
 - @aoen (Dan Davydov)
-- @syvineckruyk (Steven Yvinec-Kruyk)
 - @msumit (Sumit Maheshwari)
 - @alexvanboxel (Alex Van Boxel)
 - @saguziel (Alex Guziel)
@@ -30,10 +48,15 @@ Committers
 - @fokko (Fokko Driesprong)
 - @ash (Ash Berlin-Taylor)
 - @kaxilnaik (Kaxil Naik)
+- @feng-tao (Tao Feng)
+- @hiteshs (Hitesh Shah)
+- @jghoman (Jakob Homan)
+- @XD-DENG (Xiaodong Deng)
+- @dimberman (Daniel Imberman)
 
 For the full list of contributors, take a look at `Airflow's Github
 Contributor page:
-<https://github.com/apache/incubator-airflow/graphs/contributors>`_
+<https://github.com/apache/airflow/graphs/contributors>`_
 
 
 Resources & links
@@ -41,8 +64,8 @@ Resources & links
 
 * `Airflow's official documentation <http://airflow.apache.org/>`_
 * Mailing list (send emails to
-  ``dev-subscribe@airflow.incubator.apache.org`` and/or
-  ``commits-subscribe@airflow.incubator.apache.org``
+  ``dev-subscribe@airflow.apache.org`` and/or
+  ``commits-subscribe@airflow.apache.org``
   to subscribe to each)
 * `Issues on Apache's Jira <https://issues.apache.org/jira/browse/AIRFLOW>`_
 * `Gitter (chat) Channel <https://gitter.im/airbnb/airflow>`_
diff --git a/docs/scheduler.rst b/docs/scheduler.rst
index dfa0a4275770a..4d99e100b060f 100644
--- a/docs/scheduler.rst
+++ b/docs/scheduler.rst
@@ -1,10 +1,28 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Scheduling & Triggers
 =====================
 
 The Airflow scheduler monitors all tasks and all DAGs, and triggers the
 task instances whose dependencies have been met. Behind the scenes,
-it monitors and stays in sync with a folder for all DAG objects it may contain,
-and periodically (every minute or so) inspects active tasks to see whether
+it spins up a subprocess, which monitors and stays in sync with a folder
+for all DAG objects it may contain, and periodically (every minute or so)
+collects DAG parsing results and inspects active tasks to see whether
 they can be triggered.
 
 The Airflow scheduler is designed to run as a persistent service in an
@@ -21,9 +39,9 @@ has ended.
 start date, at the END of the period.
 
 The scheduler starts an instance of the executor specified in the your
-``airflow.cfg``. If it happens to be the ``LocalExecutor``, tasks will be
-executed as subprocesses; in the case of ``CeleryExecutor`` and
-``MesosExecutor``, tasks are executed remotely.
+``airflow.cfg``. If it happens to be the :class:`airflow.contrib.executors.local_executor.LocalExecutor`, tasks will be
+executed as subprocesses; in the case of :class:`airflow.executors.celery_executor.CeleryExecutor`, :class:`airflow.executors.dask_executor.DaskExecutor``, and
+:class:`airflow.contrib.executors.mesos_executor.MesosExecutor`, tasks are executed remotely.
 
 To start a scheduler, simply run the command:
 
@@ -63,6 +81,8 @@ use one of these cron "preset":
 | ``@yearly``  | Run once a year at midnight of January 1                       | ``0 0 1 1 *`` |
 +--------------+----------------------------------------------------------------+---------------+
 
+**Note**: Use ``schedule_interval=None`` and not ``schedule_interval='None'`` when
+you don't want to schedule your DAG.
 
 Your DAG will be instantiated
 for each schedule, while creating a ``DAG Run`` entry for each schedule.
@@ -94,7 +114,7 @@ interval series.
 
     """
     Code that goes along with the Airflow tutorial located at:
-    https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py
+    https://github.com/apache/airflow/blob/master/airflow/example_dags/tutorial.py
     """
     from airflow import DAG
     from airflow.operators.bash_operator import BashOperator
@@ -110,7 +130,7 @@ interval series.
         'email_on_retry': False,
         'retries': 1,
         'retry_delay': timedelta(minutes=5),
-        'schedule_interval': '@hourly',
+        'schedule_interval': '@daily',
     }
 
     dag = DAG('tutorial', catchup=False, default_args=default_args)
@@ -158,6 +178,7 @@ Here are some of the ways you can **unblock tasks**:
   states (``failed``, or ``success``)
 * Clearing a task instance will no longer delete the task instance record. Instead it updates
   max_tries and set the current task instance state to be None.
+* Marking task instances as failed can be done through the UI. This can be used to stop running task instances.
 * Marking task instances as successful can be done through the UI. This is mostly to fix false negatives,
   or for instance when the fix has been applied outside of Airflow.
 * The ``airflow backfill`` CLI subcommand has a flag to ``--mark_success`` and allows selecting
diff --git a/docs/security.rst b/docs/security.rst
index 3c328d829c03d..4058ddfe61c0e 100644
--- a/docs/security.rst
+++ b/docs/security.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Security
 ========
 
@@ -10,15 +27,27 @@ backends or creating your own.
 
 Be sure to checkout :doc:`api` for securing the API.
 
+.. note::
+
+   Airflow uses the config parser of Python. This config parser interpolates
+   '%'-signs.  Make sure escape any ``%`` signs in your config file (but not
+   environment variables) as ``%%``, otherwise Airflow might leak these
+   passwords on a config parser exception to a log.
+
 Web Authentication
 ------------------
 
 Password
 ''''''''
 
+.. note::
+
+   This is for flask-admin based web UI only. If you are using FAB-based web UI with RBAC feature,
+   please use command line interface ``create_user`` to create accounts, or do that in the FAB-based UI itself.
+
 One of the simplest mechanisms for authentication is requiring users to specify a password before logging in.
 Password authentication requires the used of the ``password`` subpackage in your requirements file. Password hashing
-uses bcrypt before storing passwords.
+uses ``bcrypt`` before storing passwords.
 
 .. code-block:: bash
 
@@ -54,8 +83,7 @@ LDAP
 ''''
 
 To turn on LDAP authentication configure your ``airflow.cfg`` as follows. Please note that the example uses
-an encrypted connection to the ldap server as you probably do not want passwords be readable on the network level.
-It is however possible to configure without encryption if you really want to.
+an encrypted connection to the ldap server as we do not want passwords be readable on the network level.
 
 Additionally, if you are using Active Directory, and are not explicitly specifying an OU that your users are in,
 you will need to change ``search_scope`` to "SUBTREE".
@@ -89,6 +117,10 @@ Valid search_scope options can be found in the `ldap3 Documentation <http://ldap
     # Set search_scope to SUBTREE if using Active Directory, and not specifying an Organizational Unit
     search_scope = LEVEL
 
+    # This option tells ldap3 to ignore schemas that are considered malformed. This sometimes comes up
+    # when using hosted ldap services.
+    ignore_malformed_schema = False
+
 The superuser_filter and data_profiler_filter are optional. If defined, these configurations allow you to specify LDAP groups that users must belong to in order to have superuser (admin) and data-profiler permissions. If undefined, all users will be superusers and data profilers.
 
 Roll your own
@@ -226,6 +258,12 @@ and in your DAG, when initializing the HiveOperator, specify:
 
     run_as_owner=True
 
+To use kerberos authentication, you must install Airflow with the `kerberos` extras group:
+
+.. code-block:: bash
+
+   pip install 'apache-airflow[kerberos]'
+
 OAuth Authentication
 --------------------
 
@@ -253,6 +291,12 @@ to only members of those teams.
 .. note:: If you do not specify a team whitelist, anyone with a valid account on
    your GHE installation will be able to login to Airflow.
 
+To use GHE authentication, you must install Airflow with the `github_enterprise` extras group:
+
+.. code-block:: bash
+
+   pip install 'apache-airflow[github_enterprise]'
+
 Setting up GHE Authentication
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -295,6 +339,12 @@ login, separated with a comma, to only members of those domains.
     oauth_callback_route = /oauth2callback
     domain = "example1.com,example2.com"
 
+To use Google authentication, you must install Airflow with the `google_auth` extras group:
+
+.. code-block:: bash
+
+   pip install 'apache-airflow[google_auth]'
+
 Setting up Google Authentication
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -337,10 +387,10 @@ certs and keys.
 .. code-block:: bash
 
     [celery]
-    CELERY_SSL_ACTIVE = True
-    CELERY_SSL_KEY = <path to key>
-    CELERY_SSL_CERT = <path to cert>
-    CELERY_SSL_CACERT = <path to cacert>
+    ssl_active = True
+    ssl_key = <path to key>
+    ssl_cert = <path to cert>
+    ssl_cacert = <path to cacert>
 
 Impersonation
 -------------
@@ -372,3 +422,171 @@ not set.
 
     [core]
     default_impersonation = airflow
+
+
+Flower Authentication
+---------------------
+
+Basic authentication for Celery Flower is supported.
+
+You can specify the details either as an optional argument in the Flower process launching
+command, or as a configuration item in your ``airflow.cfg``. For both cases, please provide
+`user:password` pairs separated by a comma.
+
+.. code-block:: bash
+
+    airflow flower --basic_auth=user1:password1,user2:password2
+
+.. code-block:: bash
+
+    [celery]
+    flower_basic_auth = user1:password1,user2:password2
+
+
+RBAC UI Security
+----------------
+
+Security of Airflow Webserver UI when running with ``rbac=True`` in the config is handled by Flask AppBuilder (FAB).
+Please read its related `security document <http://flask-appbuilder.readthedocs.io/en/latest/security.html>`_
+regarding its security model.
+
+Default Roles
+'''''''''''''
+Airflow ships with a set of roles by default: Admin, User, Op, Viewer, and Public.
+Only ``Admin`` users could configure/alter the permissions for other roles. But it is not recommended
+that ``Admin`` users alter these default roles in any way by removing
+or adding permissions to these roles.
+
+Admin
+^^^^^
+``Admin`` users have all possible permissions, including granting or revoking permissions from
+other users.
+
+Public
+^^^^^^
+``Public`` users (anonymous) don't have any permissions.
+
+Viewer
+^^^^^^
+``Viewer`` users have limited viewer permissions
+
+.. code:: python
+
+    VIEWER_PERMS = {
+        'menu_access',
+        'can_index',
+        'can_list',
+        'can_show',
+        'can_chart',
+        'can_dag_stats',
+        'can_dag_details',
+        'can_task_stats',
+        'can_code',
+        'can_log',
+        'can_get_logs_with_metadata',
+        'can_tries',
+        'can_graph',
+        'can_tree',
+        'can_task',
+        'can_task_instances',
+        'can_xcom',
+        'can_gantt',
+        'can_landing_times',
+        'can_duration',
+        'can_blocked',
+        'can_rendered',
+        'can_pickle_info',
+        'can_version',
+    }
+
+on limited web views
+
+.. code:: python
+
+    VIEWER_VMS = {
+        'Airflow',
+        'DagModelView',
+        'Browse',
+        'DAG Runs',
+        'DagRunModelView',
+        'Task Instances',
+        'TaskInstanceModelView',
+        'SLA Misses',
+        'SlaMissModelView',
+        'Jobs',
+        'JobModelView',
+        'Logs',
+        'LogModelView',
+        'Docs',
+        'Documentation',
+        'GitHub',
+        'About',
+        'Version',
+        'VersionView',
+    }
+
+User
+^^^^
+``User`` users have ``Viewer`` permissions plus additional user permissions
+
+.. code:: python
+
+    USER_PERMS = {
+        'can_dagrun_clear',
+        'can_run',
+        'can_trigger',
+        'can_add',
+        'can_edit',
+        'can_delete',
+        'can_paused',
+        'can_refresh',
+        'can_success',
+        'muldelete',
+        'set_failed',
+        'set_running',
+        'set_success',
+        'clear',
+        'can_clear',
+    }
+
+
+on User web views which is the same as Viewer web views.
+
+Op
+^^
+``Op`` users have ``User`` permissions plus additional op permissions
+
+.. code:: python
+
+    OP_PERMS = {
+        'can_conf',
+        'can_varimport',
+    }
+
+on ``User`` web views plus these additional op web views
+
+.. code:: python
+
+    OP_VMS = {
+        'Admin',
+        'Configurations',
+        'ConfigurationView',
+        'Connections',
+        'ConnectionModelView',
+        'Pools',
+        'PoolModelView',
+        'Variables',
+        'VariableModelView',
+        'XComs',
+        'XComModelView',
+    }
+
+Custom Roles
+'''''''''''''
+
+DAG Level Role
+^^^^^^^^^^^^^^
+``Admin`` can create a set of roles which are only allowed to view a certain set of dags. This is called DAG level access. Each dag defined in the dag model table
+is treated as a ``View`` which has two permissions associated with it (``can_dag_read`` and ``can_dag_edit``). There is a special view called ``all_dags`` which
+allows the role to access all the dags. The default ``Admin``, ``Viewer``, ``User``, ``Op`` roles can all access ``all_dags`` view.
+
diff --git a/docs/start.rst b/docs/start.rst
index a3e21f93f68de..620cc4dc12094 100644
--- a/docs/start.rst
+++ b/docs/start.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Quick Start
 -----------
 
@@ -33,7 +50,7 @@ if started by systemd.
 
 Out of the box, Airflow uses a sqlite database, which you should outgrow
 fairly quickly since no parallelization is possible using this database
-backend. It works in conjunction with the ``SequentialExecutor`` which will
+backend. It works in conjunction with the :class:`airflow.executors.sequential_executor.SequentialExecutor` which will
 only run task instances sequentially. While this is very limiting, it allows
 you to get up and running quickly and take a tour of the UI and the
 command line utilities.
diff --git a/docs/templates/layout.html b/docs/templates/layout.html
new file mode 100644
index 0000000000000..f6bb8fa04a78d
--- /dev/null
+++ b/docs/templates/layout.html
@@ -0,0 +1,92 @@
+<!--
+Javascript to render AIRFLOW-XXX and PR references in text
+as HTML links.
+
+Overrides extrahead block from sphinx_rtd_theme
+https://www.sphinx-doc.org/en/master/templating.html
+-->
+
+{% extends "!layout.html" %}
+
+{% block extrahead %}
+  {{ super() }}
+  <script>
+    document.addEventListener('DOMContentLoaded', function() {
+      var el = document.getElementById('changelog');
+      if (el !== null ) {
+        // [AIRFLOW-...]
+        el.innerHTML = el.innerHTML.replace(
+            /\[(AIRFLOW-[\d]+)\]/g,
+            `<a href="https://issues.apache.org/jira/browse/$1">[$1]</a>`
+        );
+        // (#...)
+        el.innerHTML = el.innerHTML.replace(
+            /\(#([\d]+)\)/g,
+            `<a href="https://github.com/apache/airflow/pull/$1">(#$1)</a>`
+        );
+      };
+    })
+  </script>
+  <style>
+    .example-header {
+      position: relative;
+      background: #9AAA7A;
+      padding: 8px 16px;
+      margin-bottom: 0;
+    }
+    .example-header--with-button {
+      padding-right: 166px;
+    }
+    .example-header:after{
+      content: '';
+      display: table;
+      clear: both;
+    }
+    .example-title {
+      display:block;
+      padding: 4px;
+      margin-right: 16px;
+      color: white;
+      overflow-x: auto;
+    }
+    .example-header-button {
+      top: 8px;
+      right: 16px;
+      position: absolute;
+    }
+    .example-header + .highlight-python {
+      margin-top: 0 !important;
+    }
+    .viewcode-button {
+      display: inline-block;
+      padding: 8px 16px;
+      border: 0;
+      margin: 0;
+      outline: 0;
+      border-radius: 2px;
+      -webkit-box-shadow: 0 3px 5px 0 rgba(0,0,0,.3);
+      box-shadow: 0 3px 6px 0 rgba(0,0,0,.3);
+      color: #404040;
+      background-color: #e7e7e7;
+      cursor: pointer;
+      font-size: 16px;
+      font-weight: 500;
+      line-height: 1;
+      text-decoration: none;
+      text-overflow: ellipsis;
+      overflow: hidden;
+      text-transform: uppercase;
+      -webkit-transition: background-color .2s;
+      transition: background-color .2s;
+      vertical-align: middle;
+      white-space: nowrap;
+    }
+    .viewcode-button:visited {
+      color: #404040;
+    }
+    .viewcode-button:hover, .viewcode-button:focus {
+      color: #404040;
+      background-color: #d6d6d6;
+    }
+  </style>
+{% endblock %}
diff --git a/docs/timezone.rst b/docs/timezone.rst
index 9e8598e2ed464..a5a9d094410ec 100644
--- a/docs/timezone.rst
+++ b/docs/timezone.rst
@@ -1,24 +1,41 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 Time zones
 ==========
 
 Support for time zones is enabled by default. Airflow stores datetime information in UTC internally and in the database.
-It allows you to run your DAGs with time zone dependent schedules. At the moment Airflow does not convert them to the 
-end user’s time zone in the user interface. There it will always be displayed in UTC. Also templates used in Operators 
+It allows you to run your DAGs with time zone dependent schedules. At the moment Airflow does not convert them to the
+end user’s time zone in the user interface. There it will always be displayed in UTC. Also templates used in Operators
 are not converted. Time zone information is exposed and it is up to the writer of DAG what do with it.
 
-This is handy if your users live in more than one time zone and you want to display datetime information according to 
+This is handy if your users live in more than one time zone and you want to display datetime information according to
 each user’s wall clock.
 
-Even if you are running Airflow in only one time zone it is still good practice to store data in UTC in your database 
-(also before Airflow became time zone aware this was also to recommended or even required setup). The main reason is 
-Daylight Saving Time (DST). Many countries have a system of DST, where clocks are moved forward in spring and backward 
-in autumn. If you’re working in local time, you’re likely to encounter errors twice a year, when the transitions 
-happen. (The pendulum and pytz documentation discusses these issues in greater detail.) This probably doesn’t matter 
-for a simple DAG, but it’s a problem if you are in, for example, financial services where you have end of day 
-deadlines to meet. 
+Even if you are running Airflow in only one time zone it is still good practice to store data in UTC in your database
+(also before Airflow became time zone aware this was also to recommended or even required setup). The main reason is
+Daylight Saving Time (DST). Many countries have a system of DST, where clocks are moved forward in spring and backward
+in autumn. If you’re working in local time, you’re likely to encounter errors twice a year, when the transitions
+happen. (The pendulum and pytz documentation discusses these issues in greater detail.) This probably doesn’t matter
+for a simple DAG, but it’s a problem if you are in, for example, financial services where you have end of day
+deadlines to meet.
 
-The time zone is set in `airflow.cfg`. By default it is set to utc, but you change it to use the system’s settings or 
-an arbitrary IANA time zone, e.g. `Europe/Amsterdam`. It is dependent on `pendulum`, which is more accurate than `pytz`. 
+The time zone is set in ``airflow.cfg``. By default it is set to utc, but you change it to use the system’s settings or
+an arbitrary IANA time zone, e.g. `Europe/Amsterdam`. It is dependent on `pendulum`, which is more accurate than `pytz`.
 Pendulum is installed when you install Airflow.
 
 Please note that the Web UI currently only runs in UTC.
@@ -28,8 +45,8 @@ Concepts
 Naïve and aware datetime objects
 ''''''''''''''''''''''''''''''''
 
-Python’s datetime.datetime objects have a tzinfo attribute that can be used to store time zone information, 
-represented as an instance of a subclass of datetime.tzinfo. When this attribute is set and describes an offset, 
+Python’s datetime.datetime objects have a tzinfo attribute that can be used to store time zone information,
+represented as an instance of a subclass of datetime.tzinfo. When this attribute is set and describes an offset,
 a datetime object is aware. Otherwise, it’s naive.
 
 You can use timezone.is_aware() and timezone.is_naive() to determine whether datetimes are aware or naive.
@@ -39,7 +56,7 @@ Because Airflow uses time-zone-aware datetime objects. If your code creates date
 .. code:: python
 
     from airflow.utils import timezone
-    
+
     now = timezone.utcnow()
     a_date = timezone.datetime(2017,1,1)
 
@@ -49,9 +66,9 @@ Interpretation of naive datetime objects
 
 Although Airflow operates fully time zone aware, it still accepts naive date time objects for `start_dates`
 and `end_dates` in your DAG definitions. This is mostly in order to preserve backwards compatibility. In
-case a naive `start_date` or `end_date` is encountered the default time zone is applied. It is applied 
+case a naive `start_date` or `end_date` is encountered the default time zone is applied. It is applied
 in such a way that it is assumed that the naive date time is already in the default time zone. In other
-words if you have a default time zone setting of `Europe/Amsterdam` and create a naive datetime `start_date` of 
+words if you have a default time zone setting of `Europe/Amsterdam` and create a naive datetime `start_date` of
 `datetime(2017,1,1)` it is assumed to be a `start_date` of Jan 1, 2017 Amsterdam time.
 
 .. code:: python
@@ -65,16 +82,16 @@ words if you have a default time zone setting of `Europe/Amsterdam` and create a
     op = DummyOperator(task_id='dummy', dag=dag)
     print(op.owner) # Airflow
 
-Unfortunately, during DST transitions, some datetimes don’t exist or are ambiguous. 
-In such situations, pendulum raises an exception. That’s why you should always create aware 
+Unfortunately, during DST transitions, some datetimes don’t exist or are ambiguous.
+In such situations, pendulum raises an exception. That’s why you should always create aware
 datetime objects when time zone support is enabled.
 
-In practice, this is rarely an issue. Airflow gives you aware datetime objects in the models and DAGs, and most often, 
-new datetime objects are created from existing ones through timedelta arithmetic. The only datetime that’s often 
+In practice, this is rarely an issue. Airflow gives you aware datetime objects in the models and DAGs, and most often,
+new datetime objects are created from existing ones through timedelta arithmetic. The only datetime that’s often
 created in application code is the current time, and timezone.utcnow() automatically does the right thing.
 
 
-Default time zone 
+Default time zone
 '''''''''''''''''
 
 The default time zone is the time zone defined by the `default_timezone` setting under `[core]`. If
@@ -92,15 +109,15 @@ it is therefore important to make sure this setting is equal on all Airflow node
 Time zone aware DAGs
 --------------------
 
-Creating a time zone aware DAG is quite simple. Just make sure to supply a time zone aware `start_date`. It is 
+Creating a time zone aware DAG is quite simple. Just make sure to supply a time zone aware `start_date`. It is
 recommended to use `pendulum` for this, but `pytz` (to be installed manually) can also be used for this.
 
 .. code:: python
 
     import pendulum
-    
+
     local_tz = pendulum.timezone("Europe/Amsterdam")
-    
+
     default_args=dict(
         start_date=datetime(2016, 1, 1, tzinfo=local_tz),
         owner='Airflow'
@@ -110,18 +127,21 @@ recommended to use `pendulum` for this, but `pytz` (to be installed manually) ca
     op = DummyOperator(task_id='dummy', dag=dag)
     print(dag.timezone) # <Timezone [Europe/Amsterdam]>
 
-
+Please note that while it is possible to set a `start_date` and `end_date` for Tasks always the DAG timezone
+or global timezone (in that order) will be used to calculate the next execution date. Upon first encounter
+the start date or end date will be converted to UTC using the timezone associated with start_date or end_date,
+then for calculations this timezone information will be disregarded.
 
 Templates
 '''''''''
 
-Airflow returns time zone aware datetimes in templates, but does not convert them to local time so they remain in UTC. 
+Airflow returns time zone aware datetimes in templates, but does not convert them to local time so they remain in UTC.
 It is left up to the DAG to handle this.
 
 .. code:: python
 
     import pendulum
-    
+
     local_tz = pendulum.timezone("Europe/Amsterdam")
     local_tz.convert(execution_date)
 
@@ -129,10 +149,10 @@ It is left up to the DAG to handle this.
 Cron schedules
 ''''''''''''''
 
-In case you set a cron schedule, Airflow assumes you will always want to run at the exact same time. It will 
-then ignore day light savings time. Thus, if you have a schedule that says 
-run at end of interval every day at 08:00 GMT+1 it will always run end of interval 08:00 GMT+1, 
-regardless if day light savings time is in place. 
+In case you set a cron schedule, Airflow assumes you will always want to run at the exact same time. It will
+then ignore day light savings time. Thus, if you have a schedule that says
+run at end of interval every day at 08:00 GMT+1 it will always run end of interval 08:00 GMT+1,
+regardless if day light savings time is in place.
 
 
 Time deltas
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 1c2dfd69b7c41..fd4d67f0ac828 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -1,3 +1,20 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 
 Tutorial
 ================
@@ -15,7 +32,7 @@ complicated, a line by line explanation follows below.
 
     """
     Code that goes along with the Airflow tutorial located at:
-    https://github.com/apache/incubator-airflow/blob/master/airflow/example_dags/tutorial.py
+    https://github.com/apache/airflow/blob/master/airflow/example_dags/tutorial.py
     """
     from airflow import DAG
     from airflow.operators.bash_operator import BashOperator
@@ -37,7 +54,7 @@ complicated, a line by line explanation follows below.
         # 'end_date': datetime(2016, 1, 1),
     }
 
-    dag = DAG('tutorial', default_args=default_args)
+    dag = DAG('tutorial', default_args=default_args, schedule_interval=timedelta(days=1))
 
     # t1, t2 and t3 are examples of tasks created by instantiating operators
     t1 = BashOperator(
@@ -147,7 +164,7 @@ define a ``schedule_interval`` of 1 day for the DAG.
 .. code:: python
 
     dag = DAG(
-        'tutorial', default_args=default_args, schedule_interval=timedelta(1))
+        'tutorial', default_args=default_args, schedule_interval=timedelta(days=1))
 
 Tasks
 -----
@@ -243,27 +260,40 @@ regarding custom filters have a look at the
 `Jinja Documentation <http://jinja.pocoo.org/docs/dev/api/#writing-filters>`_
 
 For more information on the variables and macros that can be referenced
-in templates, make sure to read through the :ref:`macros` section
+in templates, make sure to read through the :doc:`macros`
 
 Setting up Dependencies
 -----------------------
-We have two simple tasks that do not depend on each other. Here's a few ways
+We have tasks `t1`, `t2` and `t3` that do not depend on each other. Here's a few ways
 you can define dependencies between them:
 
 .. code:: python
 
-    t2.set_upstream(t1)
+    t1.set_downstream(t2)
 
     # This means that t2 will depend on t1
-    # running successfully to run
-    # It is equivalent to
-    # t1.set_downstream(t2)
+    # running successfully to run.
+    # It is equivalent to:
+    t2.set_upstream(t1)
 
-    t3.set_upstream(t1)
+    # The bit shift operator can also be
+    # used to chain operations:
+    t1 >> t2
+
+    # And the upstream dependency with the
+    # bit shift operator:
+    t2 << t1
+
+    # Chaining multiple dependencies becomes
+    # concise with the bit shift operator:
+    t1 >> t2 >> t3
 
-    # all of this is equivalent to
-    # dag.set_dependency('print_date', 'sleep')
-    # dag.set_dependency('print_date', 'templated')
+    # A list of tasks can also be set as
+    # dependencies. These operations
+    # all have the same effect:
+    t1.set_downstream([t2, t3])
+    t1 >> [t2, t3]
+    [t2, t3] << t1
 
 Note that when executing your script, Airflow will raise exceptions when
 it finds cycles in your DAG or when a dependency is referenced more
@@ -277,8 +307,8 @@ something like this:
 .. code:: python
 
     """
-    Code that goes along with the Airflow located at:
-    http://airflow.readthedocs.org/en/latest/tutorial.html
+    Code that goes along with the Airflow tutorial located at:
+    https://github.com/apache/airflow/blob/master/airflow/example_dags/tutorial.py
     """
     from airflow import DAG
     from airflow.operators.bash_operator import BashOperator
@@ -301,7 +331,7 @@ something like this:
     }
 
     dag = DAG(
-        'tutorial', default_args=default_args, schedule_interval=timedelta(1))
+        'tutorial', default_args=default_args, schedule_interval=timedelta(days=1))
 
     # t1, t2 and t3 are examples of tasks created by instantiating operators
     t1 = BashOperator(
@@ -338,8 +368,10 @@ Testing
 Running the Script
 ''''''''''''''''''
 
-Time to run some tests. First let's make sure that the pipeline
-parses. Let's assume we're saving the code from the previous step in
+Time to run some tests. First, let's make sure the pipeline
+is parsed successfully.
+
+Let's assume we're saving the code from the previous step in
 ``tutorial.py`` in the DAGs folder referenced in your ``airflow.cfg``.
 The default location for your DAGs is ``~/airflow/dags``.
 
diff --git a/docs/ui.rst b/docs/ui.rst
index 4b232fa1aea18..a2f6b8514c89a 100644
--- a/docs/ui.rst
+++ b/docs/ui.rst
@@ -1,6 +1,23 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
 UI / Screenshots
 =================
-The Airflow UI make it easy to monitor and troubleshoot your data pipelines.
+The Airflow UI makes it easy to monitor and troubleshoot your data pipelines.
 Here's a quick overview of some of the features and visualizations you
 can find in the Airflow UI.
 
diff --git a/licenses/LICENSE-bootstrap3-typeahead.txt b/licenses/LICENSE-bootstrap3-typeahead.txt
new file mode 100644
index 0000000000000..33c48448eb596
--- /dev/null
+++ b/licenses/LICENSE-bootstrap3-typeahead.txt
@@ -0,0 +1,20 @@
+/* =============================================================
+ * bootstrap3-typeahead.js v4.0.2
+ * https://github.com/bassjobsen/Bootstrap-3-Typeahead
+ * =============================================================
+ * Original written by @mdo and @fat
+ * =============================================================
+ * Copyright 2014 Bass Jobsen @bassjobsen
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License');
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an 'AS IS' BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============================================================ */
diff --git a/licenses/LICENSE-hue.txt b/licenses/LICENSE-hue.txt
new file mode 100644
index 0000000000000..8be4f5ecb89a2
--- /dev/null
+++ b/licenses/LICENSE-hue.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2009-2017 Cloudera Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/licenses/LICENSE-jqclock.txt b/licenses/LICENSE-jqclock.txt
new file mode 100644
index 0000000000000..8dada3edaf50d
--- /dev/null
+++ b/licenses/LICENSE-jqclock.txt
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/licenses/LICENSE-moment.txt b/licenses/LICENSE-moment.txt
new file mode 100644
index 0000000000000..8618b7333d6f5
--- /dev/null
+++ b/licenses/LICENSE-moment.txt
@@ -0,0 +1,22 @@
+Copyright (c) JS Foundation and other contributors
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
diff --git a/licenses/LICENSE-python-nvd3.txt b/licenses/LICENSE-python-nvd3.txt
new file mode 100644
index 0000000000000..1add6249e57b4
--- /dev/null
+++ b/licenses/LICENSE-python-nvd3.txt
@@ -0,0 +1,24 @@
+The MIT License (MIT)
+
+Python-nvd3
+
+Copyright (c) 2013 Arezqui Belaid <areski@gmail.com> and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/licenses/LICENSE-python-slugify.txt b/licenses/LICENSE-python-slugify.txt
new file mode 100644
index 0000000000000..82af695f594e8
--- /dev/null
+++ b/licenses/LICENSE-python-slugify.txt
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) Val Neekman @ Neekware Inc. http://neekware.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/licenses/LICENSE-scikit-learn.txt b/licenses/LICENSE-scikit-learn.txt
new file mode 100644
index 0000000000000..ecd7256f20d2d
--- /dev/null
+++ b/licenses/LICENSE-scikit-learn.txt
@@ -0,0 +1,31 @@
+New BSD License
+
+Copyright (c) 2007–2018 The scikit-learn developers.
+All rights reserved.
+
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  a. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  b. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  c. Neither the name of the Scikit-learn Developers  nor the names of
+     its contributors may be used to endorse or promote products
+     derived from this software without specific prior written
+     permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
diff --git a/run_unit_tests.sh b/run_unit_tests.sh
index 42c78f3439e7b..2c4abbfaad605 100755
--- a/run_unit_tests.sh
+++ b/run_unit_tests.sh
@@ -8,9 +8,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,71 +21,49 @@
 set -x
 
 # environment
-export AIRFLOW_HOME=${AIRFLOW_HOME:=~/airflow}
+export AIRFLOW_HOME=${AIRFLOW_HOME:=~}
 export AIRFLOW__CORE__UNIT_TEST_MODE=True
 
-# configuration test
-export AIRFLOW__TESTSECTION__TESTKEY=testvalue
-
-# use Airflow 2.0-style imports
-export AIRFLOW_USE_NEW_IMPORTS=1
-
 # add test/contrib to PYTHONPATH
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export PYTHONPATH=$PYTHONPATH:${DIR}/tests/test_utils
 
 # any argument received is overriding the default nose execution arguments:
-
 nose_args=$@
 
-#--with-doctest
-
 # Generate the `airflow` executable if needed
 which airflow > /dev/null || python setup.py develop
 
 echo "Initializing the DB"
+yes | airflow initdb
 yes | airflow resetdb
 
-if [ "${TRAVIS}" ]; then
-    if [ -z "$nose_args" ]; then
-      nose_args="--with-coverage \
-    --cover-erase \
-    --cover-html \
-    --cover-package=airflow \
-    --cover-html-dir=airflow/www/static/coverage \
-    --with-ignore-docstrings \
-    --rednose \
-    --with-timer \
-    -v \
-    --logging-level=DEBUG "
-    fi
+if [ -z "$nose_args" ]; then
+  nose_args="--with-coverage \
+  --cover-erase \
+  --cover-html \
+  --cover-package=airflow \
+  --cover-html-dir=airflow/www/static/coverage \
+  --with-ignore-docstrings \
+  --rednose \
+  --with-timer \
+  -v \
+  --logging-level=DEBUG "
+fi
 
-  # For impersonation tests running on SQLite on Travis, make the database world readable so other
-  # users can update it
-  AIRFLOW_DB="/home/travis/airflow/airflow.db"
-  if [ -f "${AIRFLOW_DB}" ]; then
-    sudo chmod a+rw "${AIRFLOW_DB}"
-  fi
+# For impersonation tests running on SQLite on Travis, make the database world readable so other
+# users can update it
+AIRFLOW_DB="$HOME/airflow.db"
 
-  # For impersonation tests on Travis, make airflow accessible to other users via the global PATH
-  # (which contains /usr/local/bin)
-  sudo ln -s "${VIRTUAL_ENV}/bin/airflow" /usr/local/bin/
-else
-    if [ -z "$nose_args" ]; then
-      nose_args="--with-coverage \
-    --cover-erase \
-    --cover-html \
-    --cover-package=airflow \
-    --cover-html-dir=airflow/www/static/coverage \
-    --with-ignore-docstrings \
-    --rednose \
-    --with-timer \
-    -s \
-    -v \
-    --logging-level=DEBUG "
-    fi
+if [ -f "${AIRFLOW_DB}" ]; then
+  chmod a+rw "${AIRFLOW_DB}"
+  chmod g+rwx "${AIRFLOW_HOME}"
 fi
 
+# For impersonation tests on Travis, make airflow accessible to other users via the global PATH
+# (which contains /usr/local/bin)
+sudo ln -sf "${VIRTUAL_ENV}/bin/airflow" /usr/local/bin/
+
 echo "Starting the unit tests with the following nose arguments: "$nose_args
 nosetests $nose_args
 
diff --git a/scripts/ci/ldap.sh b/scripts/ci/1-setup-env.sh
similarity index 67%
rename from scripts/ci/ldap.sh
rename to scripts/ci/1-setup-env.sh
index d0e3043a9b685..0a976b35f027b 100755
--- a/scripts/ci/ldap.sh
+++ b/scripts/ci/1-setup-env.sh
@@ -8,9 +8,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,19 +18,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -o verbose
-
-DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-LDAP_DB=/tmp/ldap_db
-
-echo "Creating database directory"
-
-rm -rf ${LDAP_DB} && mkdir ${LDAP_DB} && cp  /usr/share/doc/slapd/examples/DB_CONFIG ${LDAP_DB}
+set -exuo pipefail
 
-echo "Launching OpenLDAP ..."
+# Start MiniCluster
+java -cp "/tmp/minicluster-1.1-SNAPSHOT/*" com.ing.minicluster.MiniCluster > /dev/null &
 
-# Start slapd with non root privileges
-slapd -h "ldap://127.0.0.1:3890/" -f ${DIR}/slapd.conf
+# Set up ssh keys
+echo 'yes' | ssh-keygen -t rsa -C your_email@youremail.com -P '' -f ~/.ssh/id_rsa
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+ln -s ~/.ssh/authorized_keys ~/.ssh/authorized_keys2
+chmod 600 ~/.ssh/*
 
-# Wait for LDAP to start
-sleep 1
+# SSH Service
+sudo service ssh restart
diff --git a/scripts/ci/2-setup-kdc.sh b/scripts/ci/2-setup-kdc.sh
new file mode 100755
index 0000000000000..c5cd381c2f626
--- /dev/null
+++ b/scripts/ci/2-setup-kdc.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -exuo pipefail
+
+DIRNAME=$(cd "$(dirname "$0")"; pwd)
+
+FQDN=`hostname`
+ADMIN="admin"
+PASS="airflow"
+KRB5_KTNAME=/etc/airflow.keytab
+
+cat /etc/hosts
+echo "hostname: ${FQDN}"
+
+sudo cp $DIRNAME/krb5/krb5.conf /etc/krb5.conf
+
+echo -e "${PASS}\n${PASS}" | sudo kadmin -p ${ADMIN}/admin -w ${PASS} -q "addprinc -randkey airflow/${FQDN}"
+sudo kadmin -p ${ADMIN}/admin -w ${PASS} -q "ktadd -k ${KRB5_KTNAME} airflow"
+sudo kadmin -p ${ADMIN}/admin -w ${PASS} -q "ktadd -k ${KRB5_KTNAME} airflow/${FQDN}"
+sudo chmod 0644 ${KRB5_KTNAME}
diff --git a/scripts/ci/load_data.sh b/scripts/ci/3-setup-mysql.sh
similarity index 74%
rename from scripts/ci/load_data.sh
rename to scripts/ci/3-setup-mysql.sh
index 3422b07c20c13..ca5c1aa9f1b84 100755
--- a/scripts/ci/load_data.sh
+++ b/scripts/ci/3-setup-mysql.sh
@@ -15,14 +15,16 @@
 #  KIND, either express or implied.  See the License for the    *
 #  specific language governing permissions and limitations      *
 #  under the License.                                           *
-set -o verbose
 
-DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-DATA_DIR="${DIR}/data"
-DATA_FILE="${DATA_DIR}/baby_names.csv"
-DATABASE=airflow_ci
+set -exuo pipefail
 
-mysqladmin -u root create ${DATABASE}
-mysql -u root < ${DATA_DIR}/mysql_schema.sql
-mysqlimport --local -u root --fields-optionally-enclosed-by="\"" --fields-terminated-by=, --ignore-lines=1 ${DATABASE} ${DATA_FILE}
+MYSQL_HOST=mysql
 
+retries=3
+for ((i=0; i<retries; i++)); do
+    mysql -h ${MYSQL_HOST} -u root -e 'drop database if exists airflow; create database airflow' && exit 0
+    echo "mysql db creation did not succeed, waiting 5 seconds to retry"
+    sleep 5
+done
+
+echo "mysql db creation could not succeed" && exit 1
diff --git a/scripts/ci/5-run-tests.sh b/scripts/ci/5-run-tests.sh
new file mode 100755
index 0000000000000..436b0ab4ba558
--- /dev/null
+++ b/scripts/ci/5-run-tests.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+set -o verbose
+set -e
+
+if [ -z "$HADOOP_HOME" ]; then
+    echo "HADOOP_HOME not set - abort" >&2
+    exit 1
+fi
+
+echo "Using ${HADOOP_DISTRO} distribution of Hadoop from ${HADOOP_HOME}"
+
+pwd
+
+echo "Using travis airflow.cfg"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cp -f ${DIR}/airflow_travis.cfg ~/unittests.cfg
+
+ROOTDIR="$(dirname $(dirname $DIR))"
+export AIRFLOW__CORE__DAGS_FOLDER="$ROOTDIR/tests/dags"
+
+# add test/contrib to PYTHONPATH
+export PYTHONPATH=${PYTHONPATH:-$ROOTDIR/tests/test_utils}
+
+echo Backend: $AIRFLOW__CORE__SQL_ALCHEMY_CONN
+
+# environment
+export AIRFLOW_HOME=${AIRFLOW_HOME:=~}
+export AIRFLOW__CORE__UNIT_TEST_MODE=True
+
+# configuration test
+export AIRFLOW__TESTSECTION__TESTKEY=testvalue
+
+# use Airflow 2.0-style imports
+export AIRFLOW_USE_NEW_IMPORTS=1
+
+# any argument received is overriding the default nose execution arguments:
+nose_args=$@
+
+# Generate the `airflow` executable if needed
+which airflow > /dev/null || python setup.py develop
+
+# For impersonation tests on Travis, make airflow accessible to other users via the global PATH
+# (which contains /usr/local/bin)
+sudo ln -sf "${VIRTUAL_ENV}/bin/airflow" /usr/local/bin/
+
+# Fix codecov build path
+if [ ! -h /home/travis/build/apache/airflow ]; then
+  sudo mkdir -p /home/travis/build/apache
+  sudo ln -s ${ROOTDIR} /home/travis/build/apache/airflow
+fi
+
+if [ -z "$KUBERNETES_VERSION" ]; then
+  echo "Initializing the DB"
+  yes | airflow initdb
+  yes | airflow resetdb
+fi
+
+if [ -z "$nose_args" ]; then
+  nose_args="--with-coverage \
+  --cover-erase \
+  --cover-html \
+  --cover-package=airflow \
+  --cover-html-dir=airflow/www/static/coverage \
+  --with-ignore-docstrings \
+  --rednose \
+  --with-timer \
+  -v \
+  --logging-level=DEBUG"
+fi
+
+if [ -z "$KUBERNETES_VERSION" ]; then
+  # kdc init happens in setup_kdc.sh
+  kinit -kt ${KRB5_KTNAME} airflow
+fi
+
+# For impersonation tests running on SQLite on Travis, make the database world readable so other
+# users can update it
+AIRFLOW_DB="$HOME/airflow.db"
+
+if [ -f "${AIRFLOW_DB}" ]; then
+  chmod a+rw "${AIRFLOW_DB}"
+  chmod g+rwx "${AIRFLOW_HOME}"
+fi
+
+echo "Starting the unit tests with the following nose arguments: "$nose_args
+nosetests $nose_args
+
+# To run individual tests:
+# nosetests tests.core:CoreTest.test_scheduler_job
diff --git a/scripts/ci/check-license.sh b/scripts/ci/6-check-license.sh
similarity index 70%
rename from scripts/ci/check-license.sh
rename to scripts/ci/6-check-license.sh
index 83c942a0a4f12..8f3999043b14d 100755
--- a/scripts/ci/check-license.sh
+++ b/scripts/ci/6-check-license.sh
@@ -53,9 +53,7 @@ acquire_rat_jar () {
 FWDIR="$(cd "`dirname "$0"`"/../..; pwd)"
 cd "$FWDIR"
 
-if [ -z "${TRAVIS_CACHE}" ]; then
-    TRAVIS_CACHE=/tmp
-fi
+TMP_DIR=/tmp
 
 if test -x "$JAVA_HOME/bin/java"; then
     declare java_cmd="$JAVA_HOME/bin/java"
@@ -64,8 +62,8 @@ else
 fi
 
 export RAT_VERSION=0.12
-export rat_jar="${TRAVIS_CACHE}"/lib/apache-rat-${RAT_VERSION}.jar
-mkdir -p ${TRAVIS_CACHE}/lib
+export rat_jar="${TMP_DIR}"/lib/apache-rat-${RAT_VERSION}.jar
+mkdir -p ${TMP_DIR}/lib
 
 
 [[ -f "$rat_jar" ]] || acquire_rat_jar || {
@@ -73,6 +71,9 @@ mkdir -p ${TRAVIS_CACHE}/lib
     exit 1
 }
 
+# This is the target of a symlink in airflow/www/static/docs - and rat exclude doesn't cope with the symlink target doesn't exist
+mkdir -p docs/_build/html/
+
 echo "Running license checks. This can take a while."
 $java_cmd -jar "$rat_jar" -E "$FWDIR"/.rat-excludes  -d "$FWDIR" > rat-results.txt
 
@@ -84,24 +85,9 @@ fi
 ERRORS="$(cat rat-results.txt | grep -e "??")"
 
 if test ! -z "$ERRORS"; then
-    echo "Could not find Apache license headers in the following files:"
-    echo "$ERRORS"
-    COUNT=`echo "${ERRORS}" | wc -l`
-    # due to old builds can be removed later
-    rm -rf ${TRAVIS_CACHE}/rat-error-count
-    if [ ! -f ${TRAVIS_CACHE}/rat-error-count-builds ]; then
-        [ "${TRAVIS_PULL_REQUEST}" = "false" ] && echo ${COUNT} > ${TRAVIS_CACHE}/rat-error-count-builds
-        OLD_COUNT=${COUNT}
-    else
-        typeset -i OLD_COUNT=$(cat ${TRAVIS_CACHE}/rat-error-count-builds)
-    fi
-    if [ ${COUNT} -gt ${OLD_COUNT} ]; then
-        echo "New missing licenses (${COUNT} vs ${OLD_COUNT}) detected. Please correct them by adding them to to header of your files"
-        exit 1
-    else
-        [ "${TRAVIS_PULL_REQUEST}" = "false" ] && echo ${COUNT} > ${TRAVIS_CACHE}/rat-error-count-builds
-    fi
-    exit 0
+    echo >&2 "Could not find Apache license headers in the following files:"
+    echo >&2 "$ERRORS"
+    exit 1
 else
     echo -e "RAT checks passed."
 fi
diff --git a/scripts/ci/airflow_travis.cfg b/scripts/ci/airflow_travis.cfg
index 140ecab683f2c..15fb53e0d8303 100644
--- a/scripts/ci/airflow_travis.cfg
+++ b/scripts/ci/airflow_travis.cfg
@@ -17,11 +17,10 @@
 # under the License.
 
 [core]
-airflow_home = ~/airflow
 dags_folder = ~/airflow/dags
 base_log_folder = ~/airflow/logs
 executor = LocalExecutor
-sql_alchemy_conn = mysql://root@localhost/airflow
+sql_alchemy_conn = # overridden by tox.ini
 unit_test_mode = True
 load_examples = True
 donot_pickle = False
@@ -55,10 +54,11 @@ smtp_mail_from = airflow@example.com
 celery_app_name = airflow.executors.celery_executor
 worker_concurrency = 16
 worker_log_server_port = 8793
-broker_url = amqp://guest:guest@localhost:5672/
-result_backend = db+mysql://root@localhost/airflow
+broker_url = amqp://guest:guest@rabbitmq:5672/
+result_backend = # overridden by tox.ini
 flower_port = 5555
 default_queue = default
+sync_parallelism = 0
 
 [celery_broker_transport_options]
 visibility_timeout = 21600
diff --git a/scripts/ci/data/baby_names.csv b/scripts/ci/data/baby_names.csv
deleted file mode 100644
index b0ee22951e1ee..0000000000000
--- a/scripts/ci/data/baby_names.csv
+++ /dev/null
@@ -1,54 +0,0 @@
-"year","name","percent","sex"
-1880,"John",0.081541,"boy"
-1880,"William",0.080511,"boy"
-1880,"James",0.050057,"boy"
-1880,"Charles",0.045167,"boy"
-1880,"George",0.043292,"boy"
-1880,"Frank",0.02738,"boy"
-1880,"Joseph",0.022229,"boy"
-1880,"Thomas",0.021401,"boy"
-1880,"Henry",0.020641,"boy"
-1880,"Robert",0.020404,"boy"
-1880,"Edward",0.019965,"boy"
-1880,"Harry",0.018175,"boy"
-1880,"Walter",0.014822,"boy"
-1880,"Arthur",0.013504,"boy"
-1880,"Fred",0.013251,"boy"
-1880,"Albert",0.012609,"boy"
-1880,"Samuel",0.008648,"boy"
-1880,"David",0.007339,"boy"
-1880,"Louis",0.006993,"boy"
-1880,"Joe",0.006174,"boy"
-1880,"Charlie",0.006165,"boy"
-1880,"Clarence",0.006165,"boy"
-1880,"Richard",0.006148,"boy"
-1880,"Andrew",0.005439,"boy"
-1880,"Daniel",0.00543,"boy"
-1880,"Ernest",0.005194,"boy"
-1880,"Will",0.004966,"boy"
-1880,"Jesse",0.004805,"boy"
-1880,"Oscar",0.004594,"boy"
-1880,"Lewis",0.004366,"boy"
-1880,"Peter",0.004189,"boy"
-1880,"Benjamin",0.004138,"boy"
-1880,"Frederick",0.004079,"boy"
-1880,"Willie",0.00402,"boy"
-1880,"Alfred",0.003961,"boy"
-1880,"Sam",0.00386,"boy"
-1880,"Roy",0.003716,"boy"
-1880,"Herbert",0.003581,"boy"
-1880,"Jacob",0.003412,"boy"
-1880,"Tom",0.00337,"boy"
-1880,"Elmer",0.00315,"boy"
-1880,"Carl",0.003142,"boy"
-1880,"Lee",0.003049,"boy"
-1880,"Howard",0.003015,"boy"
-1880,"Martin",0.003015,"boy"
-1880,"Michael",0.00299,"boy"
-1880,"Bert",0.002939,"boy"
-1880,"Herman",0.002931,"boy"
-1880,"Jim",0.002914,"boy"
-1880,"Francis",0.002905,"boy"
-1880,"Harvey",0.002905,"boy"
-1880,"Earl",0.002829,"boy"
-1880,"Eugene",0.00277,"boy"
\ No newline at end of file
diff --git a/scripts/ci/data/mysql_schema.sql b/scripts/ci/data/mysql_schema.sql
deleted file mode 100644
index c2c60729d1468..0000000000000
--- a/scripts/ci/data/mysql_schema.sql
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-USE airflow_ci;
-
-CREATE TABLE IF NOT EXISTS baby_names (
-  org_year integer(4),
-  baby_name VARCHAR(25),
-  rate FLOAT(7,6),
-  sex VARCHAR(4)
-)
\ No newline at end of file
diff --git a/scripts/ci/docker-compose-kubernetes.yml b/scripts/ci/docker-compose-kubernetes.yml
new file mode 100644
index 0000000000000..0cdb2df199b7b
--- /dev/null
+++ b/scripts/ci/docker-compose-kubernetes.yml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2.2"
+services:
+  airflow-testing:
+    network_mode: host
+    environment:
+      - KUBERNETES_VERSION
+      - MINIKUBE_IP
+    volumes:
+      - /usr/local/bin/kubectl:/usr/local/bin/kubectl
+      - /usr/local/bin/minikube:/usr/local/bin/minikube
+      - ~/.kube:/home/airflow/.kube
+      - ~/.minikube:/home/airflow/.minikube
diff --git a/scripts/ci/docker-compose.yml b/scripts/ci/docker-compose.yml
new file mode 100644
index 0000000000000..1c44f7a74b0dd
--- /dev/null
+++ b/scripts/ci/docker-compose.yml
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2.2"
+services:
+  mysql:
+    image: mysql:5.6
+    restart: always
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=true
+      - MYSQL_ROOT_HOST=%
+    volumes:
+      - ./mysql/conf.d:/etc/mysql/conf.d
+
+  postgres:
+    image: postgres:9.6
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+
+  mongo:
+    image: mongo:3
+    restart: always
+
+  cassandra:
+    image: cassandra:3.0
+    restart: always
+
+  rabbitmq:
+    image: rabbitmq:3.7
+    restart: always
+
+  redis:
+    image: redis:5.0.1
+    restart: always
+
+  openldap:
+    image: osixia/openldap:1.2.0
+    restart: always
+    command: --copy-service
+    environment:
+      - LDAP_DOMAIN=example.com
+      - LDAP_ADMIN_PASSWORD=insecure
+      - LDAP_CONFIG_PASSWORD=insecure
+    volumes:
+      - ./openldap/ldif:/container/service/slapd/assets/config/bootstrap/ldif/custom
+
+  krb5-kdc-server:
+    image: godatadriven/krb5-kdc-server
+    hostname: krb5-kdc-server
+    domainname: example.com
+
+  airflow-testing:
+    image: airflowci/incubator-airflow-ci:latest
+    init: true
+    environment:
+      - USER=airflow
+      - TOX_ENV
+      - PYTHON_VERSION
+      - CI
+      - TRAVIS
+      - TRAVIS_BRANCH
+      - TRAVIS_BUILD_DIR
+      - TRAVIS_JOB_ID
+      - TRAVIS_JOB_NUMBER
+      - TRAVIS_PULL_REQUEST
+      - TRAVIS_COMMIT
+      - TRAVIS_REPO_SLUG
+      - TRAVIS_OS_NAME
+      - TRAVIS_TAG
+    depends_on:
+      - postgres
+      - mysql
+      - mongo
+      - cassandra
+      - rabbitmq
+      - redis
+      - openldap
+      - krb5-kdc-server
+    volumes:
+      - ../../:/app
+      - ~/.cache/pip:/home/airflow/.cache/pip
+      - ~/.wheelhouse/:/home/airflow/.wheelhouse/
+    working_dir: /app
diff --git a/scripts/ci/flake8_diff.sh b/scripts/ci/flake8_diff.sh
deleted file mode 100755
index 376be9bc0fb5d..0000000000000
--- a/scripts/ci/flake8_diff.sh
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (c) 2007–2017 The scikit-learn developers.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-# 3. Neither the name of the Scikit-learn Developers  nor the names of
-#    its contributors may be used to endorse or promote products
-#    derived from this software without specific prior written
-#    permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# This script is used in Travis to check that PRs do not add obvious
-# flake8 violations. It relies on two things:
-#   - find common ancestor between branch and
-#     apache/incubator-airflow remote
-#   - run flake8 --diff on the diff between the branch and the common
-#     ancestor
-#
-# Additional features:
-#   - the line numbers in Travis match the local branch on the PR
-#     author machine.
-#   - ./build_tools/travis/flake8_diff.sh can be run locally for quick
-#     turn-around
-
-set -e
-# pipefail is necessary to propagate exit codes
-set -o pipefail
-
-PROJECT=apache/incubator-airflow
-PROJECT_URL=https://github.com/$PROJECT.git
-
-# Find the remote with the project name (upstream in most cases)
-REMOTE=$(git remote -v | grep $PROJECT | cut -f1 | head -1 || echo '')
-
-# Add a temporary remote if needed. For example this is necessary when
-# Travis is configured to run in a fork. In this case 'origin' is the
-# fork and not the reference repo we want to diff against.
-if [[ -z "$REMOTE" ]]; then
-    TMP_REMOTE=tmp_reference_upstream
-    REMOTE=$TMP_REMOTE
-    git remote add $REMOTE $PROJECT_URL
-fi
-
-echo "Remotes:"
-echo '--------------------------------------------------------------------------------'
-git remote --verbose
-
-# Travis does the git clone with a limited depth (50 at the time of
-# writing). This may not be enough to find the common ancestor with
-# $REMOTE/master so we unshallow the git checkout
-if [[ -a .git/shallow ]]; then
-    echo -e '\nTrying to unshallow the repo:'
-    echo '--------------------------------------------------------------------------------'
-    git fetch --unshallow
-fi
-
-if [[ "$TRAVIS" == "true" ]]; then
-    if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]
-    then
-        # In main repo, using TRAVIS_COMMIT_RANGE to test the commits
-        # that were pushed into a branch
-        if [[ "$PROJECT" == "$TRAVIS_REPO_SLUG" ]]; then
-            if [[ -z "$TRAVIS_COMMIT_RANGE" ]]; then
-                echo "New branch, no commit range from Travis so passing this test by convention"
-                exit 0
-            fi
-            COMMIT_RANGE=$TRAVIS_COMMIT_RANGE
-        fi
-    else
-        # We want to fetch the code as it is in the PR branch and not
-        # the result of the merge into master. This way line numbers
-        # reported by Travis will match with the local code.
-        LOCAL_BRANCH_REF=travis_pr_$TRAVIS_PULL_REQUEST
-        # In Travis the PR target is always origin
-        git fetch origin pull/$TRAVIS_PULL_REQUEST/head:refs/$LOCAL_BRANCH_REF
-    fi
-fi
-
-# If not using the commit range from Travis we need to find the common
-# ancestor between $LOCAL_BRANCH_REF and $REMOTE/master
-if [[ -z "$COMMIT_RANGE" ]]; then
-    if [[ -z "$LOCAL_BRANCH_REF" ]]; then
-        LOCAL_BRANCH_REF=$(git rev-parse --abbrev-ref HEAD)
-    fi
-    echo -e "\nLast 2 commits in $LOCAL_BRANCH_REF:"
-    echo '--------------------------------------------------------------------------------'
-    git --no-pager log -2 $LOCAL_BRANCH_REF
-
-    REMOTE_MASTER_REF="$REMOTE/master"
-    # Make sure that $REMOTE_MASTER_REF is a valid reference
-    echo -e "\nFetching $REMOTE_MASTER_REF"
-    echo '--------------------------------------------------------------------------------'
-    git fetch $REMOTE master:refs/remotes/$REMOTE_MASTER_REF
-    LOCAL_BRANCH_SHORT_HASH=$(git rev-parse --short $LOCAL_BRANCH_REF)
-    REMOTE_MASTER_SHORT_HASH=$(git rev-parse --short $REMOTE_MASTER_REF)
-
-    COMMIT=$(git merge-base $LOCAL_BRANCH_REF $REMOTE_MASTER_REF) || \
-        echo "No common ancestor found for $(git show $LOCAL_BRANCH_REF -q) and $(git show $REMOTE_MASTER_REF -q)"
-
-    if [ -z "$COMMIT" ]; then
-        exit 1
-    fi
-
-    COMMIT_SHORT_HASH=$(git rev-parse --short $COMMIT)
-
-    echo -e "\nCommon ancestor between $LOCAL_BRANCH_REF ($LOCAL_BRANCH_SHORT_HASH)"\
-         "and $REMOTE_MASTER_REF ($REMOTE_MASTER_SHORT_HASH) is $COMMIT_SHORT_HASH:"
-    echo '--------------------------------------------------------------------------------'
-    git --no-pager show --no-patch $COMMIT_SHORT_HASH
-
-    COMMIT_RANGE="$COMMIT_SHORT_HASH..$LOCAL_BRANCH_SHORT_HASH"
-
-    if [[ -n "$TMP_REMOTE" ]]; then
-        git remote remove $TMP_REMOTE
-    fi
-
-else
-    echo "Got the commit range from Travis: $COMMIT_RANGE"
-fi
-
-echo -e '\nRunning flake8 on the diff in the range' "$COMMIT_RANGE" \
-     "($(git rev-list $COMMIT_RANGE | wc -l) commit(s)):"
-echo '--------------------------------------------------------------------------------'
-
-MODIFIED_FILES="$(git diff --name-only $COMMIT_RANGE || echo "no_match")"
-
-check_files() {
-    files="$1"
-    shift
-    options="$*"
-    if [ -n "$files" ]; then
-        # Conservative approach: diff without context (--unified=0) so that code
-        # that was not changed does not create failures
-        git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --show-source $options
-    fi
-}
-
-if [[ "$MODIFIED_FILES" == "no_match" ]]; then
-    echo "No file outside ignored locations has been modified"
-else
-
-    check_files "$(echo "$MODIFIED_FILES" | grep -v ^examples)"
-    check_files "$(echo "$MODIFIED_FILES" | grep ^examples)" \
-        --config ./examples/.flake8
-fi
-echo -e "No problem detected by flake8\n"
diff --git a/scripts/ci/kdc.conf b/scripts/ci/kdc.conf
deleted file mode 100644
index 5eef3053b363c..0000000000000
--- a/scripts/ci/kdc.conf
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-[kdcdefaults]
-kdc_ports = 88
-kdc_tcp_ports = 88
-
-[realms]
-TEST.LOCAL = {
-  #master_key_type = aes256-cts
-  acl_file = /etc/krb5kdc/kadm5.acl
-  dict_file = /usr/share/dict/words
-  admin_keytab = /var/krb5kdc/kadm5.keytab
-  max_renewable_life = 7d 0h 0m 0s
-  supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
-}
diff --git a/scripts/ci/krb5.conf b/scripts/ci/krb5/krb5.conf
similarity index 92%
rename from scripts/ci/krb5.conf
rename to scripts/ci/krb5/krb5.conf
index bbc802a0eb9e8..77c78d686e39e 100644
--- a/scripts/ci/krb5.conf
+++ b/scripts/ci/krb5/krb5.conf
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -30,6 +30,7 @@ renew_lifetime = 7d
 forwardable = true
 
 [realms]
- TEST.LOCAL = {
-   kdc = localhost:88
- }
+TEST.LOCAL = {
+  kdc = krb5-kdc-server:88
+  admin_server = krb5-kdc-server
+}
diff --git a/scripts/ci/kubernetes/README.md b/scripts/ci/kubernetes/README.md
index 5d1f9c195c0be..09eaa47397e93 100644
--- a/scripts/ci/kubernetes/README.md
+++ b/scripts/ci/kubernetes/README.md
@@ -1,8 +1,27 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Airflow on Kubernetes
 
 If you don't have minikube installed, please run `./minikube/start_minikube.sh` to start a k8s-instance on your local machine. Make sure that your `kubectl` is pointing to the local k8s instance.
 
-First build the docker images by running `./docker/build.sh`. This will build the image and push it to the local registry. Secondly, deploy Apache Airflow using `./kube/deploy.sh`. Finally, open the Airflow webserver page by browsing to `http://192.168.99.100:30809/admin/` (on OSX).
+First build the docker images by running `./docker/build.sh`. This will build the image and push it to the local registry. Secondly, deploy Apache Airflow using `./kube/deploy.sh -d persistent_mode`. Finally, open the Airflow webserver page by browsing to `http://192.168.99.100:30809/admin/` (on OSX).
 
 When kicking of a new job, you should be able to see new pods being kicked off:
 ```
diff --git a/scripts/ci/kubernetes/docker/Dockerfile b/scripts/ci/kubernetes/docker/Dockerfile
index 498c47b21a027..aca828746b27f 100644
--- a/scripts/ci/kubernetes/docker/Dockerfile
+++ b/scripts/ci/kubernetes/docker/Dockerfile
@@ -33,7 +33,6 @@ RUN apt-get update -y && apt-get install -y \
         unzip \
     && apt-get clean
 
-
 RUN pip install --upgrade pip
 
 # Since we install vanilla Airflow, we also want to have support for Postgres and Kubernetes
@@ -44,7 +43,7 @@ RUN pip install -U setuptools && \
 
 # install airflow
 COPY airflow.tar.gz /tmp/airflow.tar.gz
-RUN pip install /tmp/airflow.tar.gz
+RUN pip install --no-use-pep517 /tmp/airflow.tar.gz
 
 COPY airflow-test-env-init.sh /tmp/airflow-test-env-init.sh
 
diff --git a/scripts/ci/kubernetes/docker/airflow-test-env-init.sh b/scripts/ci/kubernetes/docker/airflow-test-env-init.sh
index aa86da72dfa70..6c24a5cf6878a 100755
--- a/scripts/ci/kubernetes/docker/airflow-test-env-init.sh
+++ b/scripts/ci/kubernetes/docker/airflow-test-env-init.sh
@@ -17,8 +17,12 @@
 #  specific language governing permissions and limitations      *
 #  under the License.
 
+set -x
+
 cd /usr/local/lib/python2.7/dist-packages/airflow && \
 cp -R example_dags/* /root/airflow/dags/ && \
+cp -R contrib/example_dags/example_kubernetes_*.py /root/airflow/dags/ && \
+cp -a contrib/example_dags/libs /root/airflow/dags/ && \
 airflow initdb && \
 alembic upgrade heads && \
 (airflow create_user -u airflow -l airflow -f jon -e airflow@apache.org -r Admin -p airflow || true) && \
diff --git a/scripts/ci/kubernetes/docker/build.sh b/scripts/ci/kubernetes/docker/build.sh
index b93c6b1715ddb..cb0995aaf473c 100755
--- a/scripts/ci/kubernetes/docker/build.sh
+++ b/scripts/ci/kubernetes/docker/build.sh
@@ -22,16 +22,30 @@ TAG=${2:-latest}
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
 AIRFLOW_ROOT="$DIRNAME/../../../.."
 
-ENVCONFIG=$(minikube docker-env)
-if [ $? -eq 0 ]; then
-  eval $ENVCONFIG
+set -e
+
+if [ "${VM_DRIVER:-none}" != "none" ]; then
+    ENVCONFIG=$(minikube docker-env)
+    if [ $? -eq 0 ]; then
+      eval $ENVCONFIG
+    fi
 fi
 
 echo "Airflow directory $AIRFLOW_ROOT"
 echo "Airflow Docker directory $DIRNAME"
 
+if [[ ${PYTHON_VERSION} == '3' ]]; then
+  PYTHON_DOCKER_IMAGE=python:3.6-slim
+else
+  PYTHON_DOCKER_IMAGE=python:2.7-slim
+fi
+
 cd $AIRFLOW_ROOT
-python setup.py sdist -q
+docker run -ti --rm -v ${AIRFLOW_ROOT}:/airflow \
+    -w /airflow ${PYTHON_DOCKER_IMAGE} ./scripts/ci/kubernetes/docker/compile.sh \
+
+sudo rm -rf ${AIRFLOW_ROOT}/airflow/www_rbac/node_modules
+
 echo "Copy distro $AIRFLOW_ROOT/dist/*.tar.gz ${DIRNAME}/airflow.tar.gz"
 cp $AIRFLOW_ROOT/dist/*.tar.gz ${DIRNAME}/airflow.tar.gz
 cd $DIRNAME && docker build --pull $DIRNAME --tag=${IMAGE}:${TAG}
diff --git a/scripts/ci/travis_script.sh b/scripts/ci/kubernetes/docker/compile.sh
similarity index 58%
rename from scripts/ci/travis_script.sh
rename to scripts/ci/kubernetes/docker/compile.sh
index 52571cce14a35..9a84d4d0e04fc 100755
--- a/scripts/ci/travis_script.sh
+++ b/scripts/ci/kubernetes/docker/compile.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-
+#
 #  Licensed to the Apache Software Foundation (ASF) under one   *
 #  or more contributor license agreements.  See the NOTICE file *
 #  distributed with this work for additional information        *
@@ -17,24 +17,16 @@
 #  specific language governing permissions and limitations      *
 #  under the License.                                           *
 
-DIRNAME=$(cd "$(dirname "$0")"; pwd)
-AIRFLOW_ROOT="$DIRNAME/../.."
-cd $AIRFLOW_ROOT && pip --version && ls -l $HOME/.wheelhouse && tox --version
+set -e
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends curl gnupg2
+
+curl -sL https://deb.nodesource.com/setup_8.x | bash -
 
-if [ -z "$KUBERNETES_VERSION" ];
-then
-  tox -e $TOX_ENV
-else
-  KUBERNETES_VERSION=${KUBERNETES_VERSION} $DIRNAME/kubernetes/setup_kubernetes.sh && \
-  tox -e $TOX_ENV -- tests.contrib.minikube \
-                     --with-coverage \
-                     --cover-erase \
-                     --cover-html \
-                     --cover-package=airflow \
-                     --cover-html-dir=airflow/www/static/coverage \
-                     --with-ignore-docstrings \
-                     --rednose \
-                     --with-timer \
-                     -v \
-                     --logging-level=DEBUG
-fi
+apt-get update
+apt-get install -y --no-install-recommends git nodejs
+pip install GitPython
+python setup.py compile_assets sdist -q
diff --git a/scripts/ci/kubernetes/kube/deploy.sh b/scripts/ci/kubernetes/kube/deploy.sh
index a9a42a7a12d12..6883c301a0539 100755
--- a/scripts/ci/kubernetes/kube/deploy.sh
+++ b/scripts/ci/kubernetes/kube/deploy.sh
@@ -17,19 +17,135 @@
 #  specific language governing permissions and limitations      *
 #  under the License.                                           *
 
+set -x
+
 IMAGE=${1:-airflow/ci}
 TAG=${2:-latest}
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
+TEMPLATE_DIRNAME=${DIRNAME}/templates
+BUILD_DIRNAME=${DIRNAME}/build
+
+usage() {
+    cat << EOF
+  usage: $0 options
+  OPTIONS:
+    -d Use PersistentVolume or GitSync for dags_folder. Available options are "persistent_mode" or "git_mode"
+EOF
+    exit 1;
+}
+
+while getopts ":d:" OPTION; do
+  case ${OPTION} in
+    d)
+      DAGS_VOLUME=${OPTARG};;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      exit 1
+      ;;
+    :)
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+case ${DAGS_VOLUME} in
+  "persistent_mode")
+    GIT_SYNC=0
+    ;;
+  "git_mode")
+    GIT_SYNC=1
+    ;;
+  *)
+    echo "Value \"$DAGS_VOLUME\" for dags_folder is not valid." >&2
+    usage
+    ;;
+esac
+
+if [ ! -d "$BUILD_DIRNAME" ]; then
+  mkdir -p ${BUILD_DIRNAME}
+fi
+
+rm -f ${BUILD_DIRNAME}/*
+
+if [ "${GIT_SYNC}" = 0 ]; then
+    INIT_DAGS_VOLUME_NAME=airflow-dags
+    POD_AIRFLOW_DAGS_VOLUME_NAME=airflow-dags
+    CONFIGMAP_DAGS_FOLDER=/root/airflow/dags
+    CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT=
+    CONFIGMAP_DAGS_VOLUME_CLAIM=airflow-dags
+else
+    INIT_DAGS_VOLUME_NAME=airflow-dags-fake
+    POD_AIRFLOW_DAGS_VOLUME_NAME=airflow-dags-git
+    CONFIGMAP_DAGS_FOLDER=/root/airflow/dags/repo/airflow/contrib/example_dags
+    CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT=/root/airflow/dags
+    CONFIGMAP_DAGS_VOLUME_CLAIM=
+fi
+CONFIGMAP_GIT_REPO=${TRAVIS_REPO_SLUG:-apache/airflow}
+CONFIGMAP_BRANCH=${TRAVIS_BRANCH:-master}
+
+_UNAME_OUT=$(uname -s)
+case "${_UNAME_OUT}" in
+    Linux*)     _MY_OS=linux;;
+    Darwin*)    _MY_OS=darwin;;
+    *)          echo "${_UNAME_OUT} is unsupported."
+                exit 1;;
+esac
+echo "Local OS is ${_MY_OS}"
+
+case $_MY_OS in
+  linux)
+    SED_COMMAND=sed
+  ;;
+  darwin)
+    SED_COMMAND=gsed
+    if ! $(type "$SED_COMMAND" &> /dev/null) ; then
+      echo "Could not find \"$SED_COMMAND\" binary, please install it. On OSX brew install gnu-sed" >&2
+      exit 1
+    fi
+  ;;
+  *)
+    echo "${_UNAME_OUT} is unsupported."
+    exit 1
+  ;;
+esac
+
+if [ "${GIT_SYNC}" = 0 ]; then
+  ${SED_COMMAND} -e "s/{{INIT_GIT_SYNC}}//g" \
+      ${TEMPLATE_DIRNAME}/airflow.template.yaml > ${BUILD_DIRNAME}/airflow.yaml
+else
+  ${SED_COMMAND} -e "/{{INIT_GIT_SYNC}}/{r $TEMPLATE_DIRNAME/init_git_sync.template.yaml" -e 'd}' \
+      ${TEMPLATE_DIRNAME}/airflow.template.yaml > ${BUILD_DIRNAME}/airflow.yaml
+fi
+
+${SED_COMMAND} -i "s|{{CONFIGMAP_GIT_REPO}}|$CONFIGMAP_GIT_REPO|g" ${BUILD_DIRNAME}/airflow.yaml
+${SED_COMMAND} -i "s|{{CONFIGMAP_BRANCH}}|$CONFIGMAP_BRANCH|g" ${BUILD_DIRNAME}/airflow.yaml
+${SED_COMMAND} -i "s|{{INIT_DAGS_VOLUME_NAME}}|$INIT_DAGS_VOLUME_NAME|g" ${BUILD_DIRNAME}/airflow.yaml
+${SED_COMMAND} -i "s|{{POD_AIRFLOW_DAGS_VOLUME_NAME}}|$POD_AIRFLOW_DAGS_VOLUME_NAME|g" ${BUILD_DIRNAME}/airflow.yaml
+
+${SED_COMMAND} "s|{{CONFIGMAP_DAGS_FOLDER}}|$CONFIGMAP_DAGS_FOLDER|g" \
+    ${TEMPLATE_DIRNAME}/configmaps.template.yaml > ${BUILD_DIRNAME}/configmaps.yaml
+${SED_COMMAND} -i "s|{{CONFIGMAP_GIT_REPO}}|$CONFIGMAP_GIT_REPO|g" ${BUILD_DIRNAME}/configmaps.yaml
+${SED_COMMAND} -i "s|{{CONFIGMAP_BRANCH}}|$CONFIGMAP_BRANCH|g" ${BUILD_DIRNAME}/configmaps.yaml
+${SED_COMMAND} -i "s|{{CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT}}|$CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT|g" ${BUILD_DIRNAME}/configmaps.yaml
+${SED_COMMAND} -i "s|{{CONFIGMAP_DAGS_VOLUME_CLAIM}}|$CONFIGMAP_DAGS_VOLUME_CLAIM|g" ${BUILD_DIRNAME}/configmaps.yaml
+
+# Fix file permissions
+if [[ "${TRAVIS}" == true ]]; then
+  sudo chown -R travis.travis $HOME/.kube $HOME/.minikube
+fi
 
 kubectl delete -f $DIRNAME/postgres.yaml
-kubectl delete -f $DIRNAME/airflow.yaml
+kubectl delete -f $BUILD_DIRNAME/airflow.yaml
 kubectl delete -f $DIRNAME/secrets.yaml
 
+set -e
+
 kubectl apply -f $DIRNAME/secrets.yaml
-kubectl apply -f $DIRNAME/configmaps.yaml
+kubectl apply -f $BUILD_DIRNAME/configmaps.yaml
 kubectl apply -f $DIRNAME/postgres.yaml
 kubectl apply -f $DIRNAME/volumes.yaml
-kubectl apply -f $DIRNAME/airflow.yaml
+kubectl apply -f $BUILD_DIRNAME/airflow.yaml
 
 # wait for up to 10 minutes for everything to be deployed
 for i in {1..150}
@@ -49,6 +165,12 @@ POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.na
 
 echo "------- pod description -------"
 kubectl describe pod $POD
+echo "------- webserver init container logs - init -------"
+kubectl logs $POD init
+if [ "${GIT_SYNC}" = 1 ]; then
+    echo "------- webserver init container logs - git-sync-clone -------"
+    kubectl logs $POD git-sync-clone
+fi
 echo "------- webserver logs -------"
 kubectl logs $POD webserver
 echo "------- scheduler logs -------"
diff --git a/scripts/ci/kubernetes/kube/secrets.yaml b/scripts/ci/kubernetes/kube/secrets.yaml
index 4d533b36636e6..a93a0103e0f97 100644
--- a/scripts/ci/kubernetes/kube/secrets.yaml
+++ b/scripts/ci/kubernetes/kube/secrets.yaml
@@ -20,6 +20,6 @@ metadata:
   name: airflow-secrets
 type: Opaque
 data:
-  # The sql_alchemy_conn value is a base64 encoded represenation of this connection string:
+  # The sql_alchemy_conn value is a base64 encoded representation of this connection string:
   # postgresql+psycopg2://root:root@postgres-airflow:5432/airflow
   sql_alchemy_conn: cG9zdGdyZXNxbCtwc3ljb3BnMjovL3Jvb3Q6cm9vdEBwb3N0Z3Jlcy1haXJmbG93OjU0MzIvYWlyZmxvdwo=
diff --git a/scripts/ci/kubernetes/kube/airflow.yaml b/scripts/ci/kubernetes/kube/templates/airflow.template.yaml
similarity index 89%
rename from scripts/ci/kubernetes/kube/airflow.yaml
rename to scripts/ci/kubernetes/kube/templates/airflow.template.yaml
index 4f451ba44a687..82f7b8b17da51 100644
--- a/scripts/ci/kubernetes/kube/airflow.yaml
+++ b/scripts/ci/kubernetes/kube/templates/airflow.template.yaml
@@ -49,7 +49,7 @@ spec:
         - name: airflow-configmap
           mountPath: /root/airflow/airflow.cfg
           subPath: airflow.cfg
-        - name: airflow-dags
+        - name: {{INIT_DAGS_VOLUME_NAME}}
           mountPath: /root/airflow/dags
         - name: test-volume
           mountPath: /root/test_volume
@@ -64,6 +64,7 @@ spec:
         args:
           - "-cx"
           - "./tmp/airflow-test-env-init.sh"
+{{INIT_GIT_SYNC}}
       containers:
       - name: webserver
         image: airflow
@@ -86,24 +87,10 @@ spec:
         - name: airflow-configmap
           mountPath: /root/airflow/airflow.cfg
           subPath: airflow.cfg
-        - name: airflow-dags
+        - name: {{POD_AIRFLOW_DAGS_VOLUME_NAME}}
           mountPath: /root/airflow/dags
         - name: airflow-logs
           mountPath: /root/airflow/logs
-#        readinessProbe:
-#          initialDelaySeconds: 5
-#          timeoutSeconds: 5
-#          periodSeconds: 5
-#          httpGet:
-#            path: /login
-#            port: 8080
-#        livenessProbe:
-#          initialDelaySeconds: 5
-#          timeoutSeconds: 5
-#          failureThreshold: 5
-#          httpGet:
-#            path: /login
-#            port: 8080
       - name: scheduler
         image: airflow
         imagePullPolicy: IfNotPresent
@@ -122,7 +109,7 @@ spec:
         - name: airflow-configmap
           mountPath: /root/airflow/airflow.cfg
           subPath: airflow.cfg
-        - name: airflow-dags
+        - name: {{POD_AIRFLOW_DAGS_VOLUME_NAME}}
           mountPath: /root/airflow/dags
         - name: airflow-logs
           mountPath: /root/airflow/logs
@@ -130,6 +117,10 @@ spec:
       - name: airflow-dags
         persistentVolumeClaim:
           claimName: airflow-dags
+      - name: airflow-dags-fake
+        emptyDir: {}
+      - name: airflow-dags-git
+        emptyDir: {}
       - name: test-volume
         persistentVolumeClaim:
           claimName: test-volume
@@ -151,4 +142,3 @@ spec:
       nodePort: 30809
   selector:
     name: airflow
-
diff --git a/scripts/ci/kubernetes/kube/configmaps.yaml b/scripts/ci/kubernetes/kube/templates/configmaps.template.yaml
similarity index 88%
rename from scripts/ci/kubernetes/kube/configmaps.yaml
rename to scripts/ci/kubernetes/kube/templates/configmaps.template.yaml
index 7b91aa2e8724c..732eb593d4749 100644
--- a/scripts/ci/kubernetes/kube/configmaps.yaml
+++ b/scripts/ci/kubernetes/kube/templates/configmaps.template.yaml
@@ -21,13 +21,12 @@ metadata:
 data:
   airflow.cfg: |
     [core]
-    airflow_home = /root/airflow
-    dags_folder = /root/airflow/dags
+    dags_folder = {{CONFIGMAP_DAGS_FOLDER}}
     base_log_folder = /root/airflow/logs
     logging_level = INFO
     executor = KubernetesExecutor
     parallelism = 32
-    load_examples = True
+    load_examples = False
     plugins_folder = /root/airflow/plugins
     sql_alchemy_conn = $SQL_ALCHEMY_CONN
 
@@ -122,7 +121,7 @@ data:
     expose_config = False
 
     # Set to true to turn on authentication:
-    # https://airflow.incubator.apache.org/security.html#web-authentication
+    # https://airflow.apache.org/security.html#web-authentication
     authenticate = False
 
     # Filter the list of dags by owner name (requires authentication to be enabled)
@@ -180,24 +179,43 @@ data:
     worker_container_tag = latest
     worker_container_image_pull_policy = IfNotPresent
     delete_worker_pods = True
-    git_repo = https://github.com/apache/incubator-airflow.git
-    git_branch = master
-    git_subpath = airflow/example_dags/
+    dags_in_image = False
+    git_repo = https://github.com/{{CONFIGMAP_GIT_REPO}}.git
+    git_branch = {{CONFIGMAP_BRANCH}}
+    git_subpath = airflow/contrib/example_dags/
     git_user =
     git_password =
-    dags_volume_claim = airflow-dags
+    git_sync_root = /git
+    git_sync_path = repo
+    git_dags_folder_mount_point = {{CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT}}
+    dags_volume_claim = {{CONFIGMAP_DAGS_VOLUME_CLAIM}}
     dags_volume_subpath =
     logs_volume_claim = airflow-logs
     logs_volume_subpath =
+    dags_volume_host =
+    logs_volume_host =
     in_cluster = True
     namespace = default
     gcp_service_account_keys =
 
+    # Example affinity and toleration definitions.
+    affinity = {"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/hostname","operator":"NotIn","values":["4e5e6a99-e28a-450b-bba9-e0124853de9b"]}]}]}}}
+    tolerations = [{ "key": "dedicated", "operator": "Equal", "value": "airflow", "effect": "NoSchedule" }, { "key": "prod", "operator": "Exists" }]
+
     # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
     git_sync_container_repository = gcr.io/google-containers/git-sync-amd64
     git_sync_container_tag = v2.0.5
     git_sync_init_container_name = git-sync-clone
 
+    [kubernetes_node_selectors]
+    # The Key-value pairs to be given to worker pods.
+    # The worker pods will be scheduled to the nodes of the specified key-value pairs.
+    # Should be supplied in the format: key = value
+
+    [kubernetes_annotations]
+    # The Key-value annotations pairs to be given to worker pods.
+    # Should be supplied in the format: key = value
+
     [kubernetes_secrets]
     SQL_ALCHEMY_CONN = airflow-secrets=sql_alchemy_conn
 
@@ -250,9 +268,18 @@ data:
     # This defines the port that Celery Flower runs on
     flower_port = 5555
 
+    # Securing Flower with Basic Authentication
+    # Accepts user:password pairs separated by a comma
+    # Example: flower_basic_auth = user1:password1,user2:password2
+    flower_basic_auth =
+
     # Default queue that tasks get assigned to and that worker listen on.
     default_queue = default
 
+    # How many processes CeleryExecutor uses to sync task state.
+    # 0 means to use max(1, number of cores - 1) processes.
+    sync_parallelism = 0
+
     # Import path for celery configuration options
     celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
 
diff --git a/scripts/ci/kubernetes/kube/templates/init_git_sync.template.yaml b/scripts/ci/kubernetes/kube/templates/init_git_sync.template.yaml
new file mode 100644
index 0000000000000..0129a5f91b6e5
--- /dev/null
+++ b/scripts/ci/kubernetes/kube/templates/init_git_sync.template.yaml
@@ -0,0 +1,36 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one   *
+#  or more contributor license agreements.  See the NOTICE file *
+#  distributed with this work for additional information        *
+#  regarding copyright ownership.  The ASF licenses this file   *
+#  to you under the Apache License, Version 2.0 (the            *
+#  "License"); you may not use this file except in compliance   *
+#  with the License.  You may obtain a copy of the License at   *
+#                                                               *
+#    http://www.apache.org/licenses/LICENSE-2.0                 *
+#                                                               *
+#  Unless required by applicable law or agreed to in writing,   *
+#  software distributed under the License is distributed on an  *
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+#  KIND, either express or implied.  See the License for the    *
+#  specific language governing permissions and limitations      *
+#  under the License.                                           *
+      - name: git-sync-clone
+        env:
+        - name: GIT_SYNC_REPO
+          value: https://github.com/{{CONFIGMAP_GIT_REPO}}.git
+        - name: GIT_SYNC_BRANCH
+          value: {{CONFIGMAP_BRANCH}}
+        - name: GIT_SYNC_ROOT
+          value: /git
+        - name: GIT_SYNC_DEST
+          value: repo
+        - name: GIT_SYNC_ONE_TIME
+          value: "true"
+        image: gcr.io/google-containers/git-sync-amd64:v2.0.5
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          runAsUser: 0
+        volumeMounts:
+        - mountPath: /git
+          name: airflow-dags-git
diff --git a/scripts/ci/kubernetes/minikube/_k8s.sh b/scripts/ci/kubernetes/minikube/_k8s.sh
index 6ce9d3170ceb6..8debeb866651f 100644
--- a/scripts/ci/kubernetes/minikube/_k8s.sh
+++ b/scripts/ci/kubernetes/minikube/_k8s.sh
@@ -27,7 +27,7 @@ function _wait_for_ready () {
   shift
   local attempts=40
   echo "Waiting till ready (count: $count): $@"
-  while [[ "$count" != $("$@" 2>&1 | tail -n +2 | grep -c $evidence) ]];
+  while [[ "$count" < $("$@" 2>&1 | tail -n +2 | awk '{print $2}' | grep -c $evidence) ]];
   do
     if [[ "$attempts" = "1" ]]; then
       echo "Last run: $@"
@@ -54,16 +54,14 @@ function k8s_single_node_ready () {
   k8s_all_nodes_ready 1
 }
 
-# Wait for all expected number of pods to be ready. This works only for
-# pods with up to 4 containers, as we check "1/1" to "4/4" in
-# `kubectl get pods` output.
-function k8s_all_pods_ready () {
+# Wait for at leat expected number of pods to be ready.
+function k8s_at_least_n_pods_ready () {
   local count="$1"
   shift
-  local evidence="-e 1/1 -e 2/2 -e 3/3 -e 4/4"
+  local evidence="-E '([0-9])\/(\1)'"
   _wait_for_ready "$count" "$evidence" kubectl get pods "$@"
 }
 
 function k8s_single_pod_ready () {
-  k8s_all_pods_ready 1 "$@"
+  k8s_at_least_n_pods_ready 1 "$@"
 }
diff --git a/scripts/ci/kubernetes/minikube/start_minikube.sh b/scripts/ci/kubernetes/minikube/start_minikube.sh
index 5171a26313d53..e0c621712efa4 100755
--- a/scripts/ci/kubernetes/minikube/start_minikube.sh
+++ b/scripts/ci/kubernetes/minikube/start_minikube.sh
@@ -19,24 +19,25 @@
 
 #!/usr/bin/env bash
 
+set -ex
 
 _MY_SCRIPT="${BASH_SOURCE[0]}"
 _MY_DIR=$(cd "$(dirname "$_MY_SCRIPT")" && pwd)
 # Avoids 1.7.x because of https://github.com/kubernetes/minikube/issues/2240
 _KUBERNETES_VERSION="${KUBERNETES_VERSION}"
+_MINIKUBE_VERSION="${MINIKUBE_VERSION:-v0.34.1}"
 
-echo "setting up kubernetes ${_KUBERNETES_VERSION}"
+echo "setting up kubernetes ${_KUBERNETES_VERSION}, using minikube ${_MINIKUBE_VERSION}"
 
-_MINIKUBE_VERSION="v0.26.0"
-_HELM_VERSION=v2.8.1
-_VM_DRIVER=none
+_VM_DRIVER="${VM_DRIVER:-none}"
 USE_MINIKUBE_DRIVER_NONE=true
 
 _UNAME_OUT=$(uname -s)
 case "${_UNAME_OUT}" in
     Linux*)     _MY_OS=linux;;
     Darwin*)    _MY_OS=darwin;;
-    *)          _MY_OS="UNKNOWN:${unameOut}"
+    *)          echo "${_UNAME_OUT} is unsupported."
+                exit 1;;
 esac
 echo "Local OS is ${_MY_OS}"
 
@@ -51,34 +52,34 @@ source _k8s.sh
 rm -rf tmp
 mkdir -p bin tmp
 
-sudo mkdir -p /usr/local/bin
+if [[ ! -d /usr/local/bin ]]; then
+    sudo mkdir -p /usr/local/bin
+fi
 
 if [[ ! -x /usr/local/bin/kubectl ]]; then
   echo Downloading kubectl, which is a requirement for using minikube.
   curl -Lo bin/kubectl  \
     https://storage.googleapis.com/kubernetes-release/release/${_KUBERNETES_VERSION}/bin/${_MY_OS}/amd64/kubectl
   chmod +x bin/kubectl
+  sudo mv bin/kubectl /usr/local/bin/kubectl
 fi
 if [[ ! -x /usr/local/bin/minikube ]]; then
   echo Downloading minikube.
   curl -Lo bin/minikube  \
     https://storage.googleapis.com/minikube/releases/${_MINIKUBE_VERSION}/minikube-${_MY_OS}-amd64
   chmod +x bin/minikube
+  sudo mv bin/minikube /usr/local/bin/minikube
 fi
 
-sudo mv bin/minikube /usr/local/bin/minikube
-sudo mv bin/kubectl /usr/local/bin/kubectl
-
 export PATH="${_MY_DIR}/bin:$PATH"
 
 if [[ "${USE_MINIKUBE_DRIVER_NONE:-}" = "true" ]]; then
   # Run minikube with none driver.
   # See https://blog.travis-ci.com/2017-10-26-running-kubernetes-on-travis-ci-with-minikube
-  _VM_DRIVER="--vm-driver=none"
+  _VM_DRIVER=none
   if [[ ! -x /usr/local/bin/nsenter ]]; then
     # From https://engineering.bitnami.com/articles/implementing-kubernetes-integration-tests-in-travis.html
     # Travis ubuntu trusty env doesn't have nsenter, needed for --vm-driver=none
-    which nsenter >/dev/null && return 0
     echo "INFO: Building 'nsenter' ..."
 cat <<-EOF | docker run -i --rm -v "$(pwd):/build" ubuntu:14.04 >& nsenter.build.log
         apt-get update
@@ -93,7 +94,7 @@ EOF
     if [ ! -f ./nsenter ]; then
         echo "ERROR: nsenter build failed, log:"
         cat nsenter.build.log
-        return 1
+        exit 1
     fi
     echo "INFO: nsenter build OK"
     sudo mv ./nsenter /usr/local/bin
@@ -102,12 +103,15 @@ fi
 
 echo "your path is ${PATH}"
 
-_MINIKUBE="sudo PATH=$PATH minikube"
+_MINIKUBE="sudo -E PATH=$PATH minikube"
 
-$_MINIKUBE config set bootstrapper localkube
-$_MINIKUBE start --kubernetes-version=${_KUBERNETES_VERSION}  --vm-driver=none
+$_MINIKUBE start --kubernetes-version=${_KUBERNETES_VERSION} --vm-driver=${_VM_DRIVER}
 $_MINIKUBE update-context
 
+if [[ "${TRAVIS}" == true ]]; then
+  sudo chown -R travis.travis $HOME/.kube $HOME/.minikube
+fi
+
 # Wait for Kubernetes to be up and ready.
 k8s_single_node_ready
 
@@ -118,7 +122,7 @@ echo Showing kube-system pods
 kubectl get -n kube-system pods
 
 (k8s_single_pod_ready -n kube-system -l component=kube-addon-manager) ||
-  (_ADDON=$(kubectl get pod -n kube-system -l component=kube-addon-manager
+  (_ADDON=$(kubectl get pod -n kube-system -l component=kube-addon-manager \
       --no-headers -o name| cut -d/ -f2);
    echo Addon-manager describe:;
    kubectl describe pod -n kube-system $_ADDON;
diff --git a/scripts/ci/kubernetes/minikube/stop_minikube.sh b/scripts/ci/kubernetes/minikube/stop_minikube.sh
new file mode 100755
index 0000000000000..78150be81d43f
--- /dev/null
+++ b/scripts/ci/kubernetes/minikube/stop_minikube.sh
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script was based on one made by @kimoonkim for kubernetes-hdfs
+
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -x /usr/local/bin/minikube ]]; then
+  exit 0
+fi
+
+# Fix file permissions
+if [[ "${TRAVIS}" == true ]]; then
+  sudo chown -R travis.travis $HOME/.kube $HOME/.minikube
+fi
+
+sudo minikube status
+if [[ $? = 0 ]]; then
+  sudo minikube delete
+  sudo rm -rf HOME/.kube $HOME/.minikube
+  if [[ "${TRAVIS}" == true ]]; then
+    sudo rm -rf /etc/kubernetes/*.conf
+  fi
+fi
+
+sudo chown -R travis.travis .
diff --git a/scripts/ci/kubernetes/setup_kubernetes.sh b/scripts/ci/kubernetes/setup_kubernetes.sh
index ea559a02aa997..e0ef216f07884 100755
--- a/scripts/ci/kubernetes/setup_kubernetes.sh
+++ b/scripts/ci/kubernetes/setup_kubernetes.sh
@@ -23,8 +23,12 @@ echo "For development, start minikube yourself (ie: minikube start) then run thi
 
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
 
+# Fix file permissions
+if [[ "${TRAVIS}" == true ]]; then
+  sudo chown -R travis.travis .
+fi
+
 $DIRNAME/minikube/start_minikube.sh
 $DIRNAME/docker/build.sh
-$DIRNAME/kube/deploy.sh
 
 echo "Airflow environment on kubernetes is good to go!"
diff --git a/scripts/ci/my.cnf b/scripts/ci/mysql/conf.d/airflow.cnf
similarity index 99%
rename from scripts/ci/my.cnf
rename to scripts/ci/mysql/conf.d/airflow.cnf
index 5ff549ca4ecff..4802d195a2b55 100644
--- a/scripts/ci/my.cnf
+++ b/scripts/ci/mysql/conf.d/airflow.cnf
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/scripts/ci/ldif/users.example.com.ldif b/scripts/ci/openldap/ldif/01-users.example.com.ldif
similarity index 99%
rename from scripts/ci/ldif/users.example.com.ldif
rename to scripts/ci/openldap/ldif/01-users.example.com.ldif
index ed4a7ceb7632c..bf5baf5ad8a18 100644
--- a/scripts/ci/ldif/users.example.com.ldif
+++ b/scripts/ci/openldap/ldif/01-users.example.com.ldif
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/scripts/ci/ldif/groups.example.com.ldif b/scripts/ci/openldap/ldif/02-groups.example.com.ldif
similarity index 96%
rename from scripts/ci/ldif/groups.example.com.ldif
rename to scripts/ci/openldap/ldif/02-groups.example.com.ldif
index 804dcb008bf2c..9d1deb349fa05 100644
--- a/scripts/ci/ldif/groups.example.com.ldif
+++ b/scripts/ci/openldap/ldif/02-groups.example.com.ldif
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -27,7 +27,7 @@ dn: cn=group1,ou=groups,dc=example,dc=com
 objectclass: groupofnames
 cn: group1
 description: Group 1 of users
-# add the group members all of which are 
+# add the group members all of which are
 # assumed to exist under example
 member: cn=user1,dc=example,dc=com
 
diff --git a/scripts/ci/ldif/manager.example.com.ldif b/scripts/ci/openldap/ldif/03-manager.example.com.ldif
similarity index 99%
rename from scripts/ci/ldif/manager.example.com.ldif
rename to scripts/ci/openldap/ldif/03-manager.example.com.ldif
index 1d06a4f2ebd6f..d4b90b73b8850 100644
--- a/scripts/ci/ldif/manager.example.com.ldif
+++ b/scripts/ci/openldap/ldif/03-manager.example.com.ldif
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/scripts/ci/ldif/example.com.ldif b/scripts/ci/openldap/ldif/04-rootdn.ldif
similarity index 84%
rename from scripts/ci/ldif/example.com.ldif
rename to scripts/ci/openldap/ldif/04-rootdn.ldif
index cda5d005616ef..59bd3e554db90 100644
--- a/scripts/ci/ldif/example.com.ldif
+++ b/scripts/ci/openldap/ldif/04-rootdn.ldif
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-dn: dc=example,dc=com
-dc: example
-description: LDAP Example
-objectClass: dcObject
-objectClass: organization
-o: example
+dn: cn=config
+changetype: modify
+
+dn: olcDatabase={1}{{ LDAP_BACKEND }},cn=config
+replace: olcRootDN
+olcRootDN: cn=Manager,{{ LDAP_BASE_DN }}
diff --git a/scripts/ci/slapd.conf b/scripts/ci/openldap/slapd.conf
similarity index 99%
rename from scripts/ci/slapd.conf
rename to scripts/ci/openldap/slapd.conf
index 34df186e4a256..a404530b8d285 100644
--- a/scripts/ci/slapd.conf
+++ b/scripts/ci/openldap/slapd.conf
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -38,7 +38,6 @@ disallow bind_anon
 database hdb
 
 suffix "dc=example,dc=com"
-
 rootdn "cn=Manager,dc=example,dc=com"
 rootpw insecure
 
diff --git a/scripts/ci/requirements.txt b/scripts/ci/requirements.txt
index 837c43f2f711e..bf3a89b36f4be 100644
--- a/scripts/ci/requirements.txt
+++ b/scripts/ci/requirements.txt
@@ -31,7 +31,7 @@ dill
 distributed
 docker-py
 filechunkio
-flake8
+flake8==3.5.0
 flask
 flask-admin
 flask-bcrypt
diff --git a/scripts/ci/run-ci.sh b/scripts/ci/run-ci.sh
new file mode 100755
index 0000000000000..7b0363b81d1ef
--- /dev/null
+++ b/scripts/ci/run-ci.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+set -x
+
+DIRNAME=$(cd "$(dirname "$0")"; pwd)
+AIRFLOW_ROOT="$DIRNAME/../.."
+
+nose_args=$@
+# Fix file permissions
+sudo chown -R airflow.airflow . $HOME/.cache $HOME/.wheelhouse/ $HOME/.cache/pip
+if [ -d $HOME/.minikube ]; then
+    sudo chown -R airflow.airflow $HOME/.kube $HOME/.minikube
+fi
+
+if [[ $PYTHON_VERSION == '3' ]]; then
+  PIP=pip3
+else
+  PIP=pip2
+fi
+
+sudo $PIP install --upgrade pip
+sudo $PIP install tox
+
+cd $AIRFLOW_ROOT && $PIP --version && tox --version
+
+if [ -z "$KUBERNETES_VERSION" ];
+then
+  tox -e $TOX_ENV $nose_args
+else
+  # This script runs inside a container, the path of the kubernetes certificate
+  # is /home/travis/.minikube/client.crt but the user in the container is `airflow`
+  if [ ! -d /home/travis ]; then
+    sudo mkdir -p /home/travis
+  fi
+  sudo ln -s /home/airflow/.minikube /home/travis/.minikube
+
+  tox -e $TOX_ENV -- tests.contrib.minikube \
+                     --with-coverage \
+                     --cover-erase \
+                     --cover-html \
+                     --cover-package=airflow \
+                     --cover-html-dir=airflow/www/static/coverage \
+                     --with-ignore-docstrings \
+                     --rednose \
+                     --with-timer \
+                     -v \
+                     --logging-level=DEBUG
+fi
diff --git a/scripts/ci/setup_env.sh b/scripts/ci/setup_env.sh
deleted file mode 100755
index a1d9a4c244109..0000000000000
--- a/scripts/ci/setup_env.sh
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -o verbose
-
-MINIKDC_VERSION=2.7.1
-
-HADOOP_DISTRO=${HADOOP_DISTRO:-"hdp"}
-
-ONLY_DOWNLOAD=${ONLY_DOWNLOAD:-false}
-ONLY_EXTRACT=${ONLY_EXTRACT:-false}
-
-MINICLUSTER_URL=https://github.com/bolkedebruin/minicluster/releases/download/1.1/minicluster-1.1-SNAPSHOT-bin.zip
-
-HIVE_HOME=/tmp/hive
-
-while test $# -gt 0; do
-    case "$1" in
-        -h|--help)
-            echo "Setup environment for airflow tests"
-            echo " "
-            echo "options:"
-            echo -e "\t-h, --help            show brief help"
-            echo -e "\t-o, --only-download   just download hadoop tar(s)"
-            echo -e "\t-e, --only-extract    just extract hadoop tar(s)"
-            echo -e "\t-d, --distro          select distro (hdp|cdh)"
-            exit 0
-            ;;
-        -o|--only-download)
-            shift
-            ONLY_DOWNLOAD=true
-            ;;
-        -e|--only-extract)
-            shift
-            ONLY_EXTRACT=true
-            ;;
-        -d|--distro)
-            shift
-            if test $# -gt 0; then
-                HADOOP_DISTRO=$1
-            else
-                echo "No Hadoop distro specified - abort" >&2
-                exit 1
-            fi
-            shift
-            ;;
-        *)
-            echo "Unknown options: $1" >&2
-            exit 1
-            ;;
-    esac
-done
-
-HADOOP_HOME=/tmp/hadoop-${HADOOP_DISTRO}
-MINICLUSTER_HOME=/tmp/minicluster
-
-if $ONLY_DOWNLOAD && $ONLY_EXTRACT; then
-    echo "Both only-download and only-extract specified - abort" >&2
-    exit 1
-fi
-
-mkdir -p ${HADOOP_HOME}
-mkdir -p ${TRAVIS_CACHE}/${HADOOP_DISTRO}
-mkdir -p ${TRAVIS_CACHE}/minicluster
-mkdir -p ${TRAVIS_CACHE}/hive
-mkdir -p ${HIVE_HOME}
-chmod -R 777 ${HIVE_HOME}
-sudo mkdir -p /user/hive/warehouse
-sudo chown -R ${USER} /user/
-sudo chmod -R 777 /user/
-ls -l /
-
-if [ $HADOOP_DISTRO = "cdh" ]; then
-    # URL="http://archive.cloudera.com/cdh5/cdh/5/hadoop-latest.tar.gz"
-    URL="https://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.11.0.tar.gz"
-    # HIVE_URL="http://archive.cloudera.com/cdh5/cdh/5/hive-latest.tar.gz"
-    HIVE_URL="https://archive.cloudera.com/cdh5/cdh/5/hive-1.1.0-cdh5.11.0.tar.gz"
-elif [ $HADOOP_DISTRO = "hdp" ]; then
-    URL="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/tars/hadoop-2.7.1.2.3.2.0-2950.tar.gz"
-    HIVE_URL="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/tars/apache-hive-1.2.1.2.3.2.0-2950-bin.tar.gz"
-else
-    echo "No/bad HADOOP_DISTRO='${HADOOP_DISTRO}' specified" >&2
-    exit 1
-fi
-
-if ! $ONLY_EXTRACT; then
-    echo "Downloading Hadoop from $URL to ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz"
-    curl -z ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz -o ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz -L $URL
-
-    if [ $? != 0 ]; then
-        echo "Failed to download Hadoop from $URL - abort" >&2
-        exit 1
-    fi
-fi
-
-if $ONLY_DOWNLOAD; then
-    exit 0
-fi
-
-echo "Extracting ${HADOOP_HOME}/hadoop.tar.gz into $HADOOP_HOME"
-tar zxf ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz --strip-components 1 -C $HADOOP_HOME
-
-if [ $? != 0 ]; then
-    echo "Failed to extract Hadoop from ${HADOOP_HOME}/hadoop.tar.gz to ${HADOOP_HOME} - abort" >&2
-    echo "Trying again..." >&2
-    # dont use cache
-    curl -o ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz -L $URL
-    tar zxf ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz --strip-components 1 -C $HADOOP_HOME
-    if [ $? != 0 ]; then
-        echo "Failed twice in downloading and unpacking hadoop!" >&2
-        exit 1
-    fi
-fi
-
-echo "Downloading and unpacking hive"
-curl -z ${TRAVIS_CACHE}/hive/hive.tar.gz -o ${TRAVIS_CACHE}/hive/hive.tar.gz -L ${HIVE_URL}
-tar zxf ${TRAVIS_CACHE}/hive/hive.tar.gz --strip-components 1 -C ${HIVE_HOME}
-
-if [ $? != 0 ]; then
-    echo "Failed to extract hive from ${TRAVIS_CACHE}/hive/hive.tar.gz" >&2
-    echo "Trying again..." >&2
-    # dont use cache
-    curl -o ${TRAVIS_CACHE}/hive/hive.tar.gz -L ${HIVE_URL}
-    tar zxf ${TRAVIS_CACHE}/hive/hive.tar.gz --strip-components 1 -C ${HIVE_HOME}
-    if [ $? != 0 ]; then
-        echo "Failed twice in downloading and unpacking hive!" >&2
-        exit 1
-    fi
-fi
-
-echo "Downloading and unpacking minicluster"
-curl -z ${TRAVIS_CACHE}/minicluster/minicluster.zip -o ${TRAVIS_CACHE}/minicluster/minicluster.zip -L ${MINICLUSTER_URL}
-ls -l ${TRAVIS_CACHE}/minicluster/minicluster.zip
-unzip ${TRAVIS_CACHE}/minicluster/minicluster.zip -d /tmp
-if [ $? != 0 ] ; then
-    # Try downloading w/o cache if there's a failure
-    curl -o ${TRAVIS_CACHE}/minicluster/minicluster.zip -L ${MINICLUSTER_URL}
-    ls -l ${TRAVIS_CACHE}/minicluster/minicluster.zip
-    unzip ${TRAVIS_CACHE}/minicluster/minicluster.zip -d /tmp
-    if [ $? != 0 ] ; then
-        echo "Failed twice in downloading and unpacking minicluster!" >&2
-        exit 1
-    fi
-    exit 1
-fi
-
-echo "Path = ${PATH}"
-
-java -cp "/tmp/minicluster-1.1-SNAPSHOT/*" com.ing.minicluster.MiniCluster > /dev/null &
diff --git a/scripts/ci/setup_kdc.sh b/scripts/ci/setup_kdc.sh
deleted file mode 100755
index 3e525c5e82ec7..0000000000000
--- a/scripts/ci/setup_kdc.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cat /etc/hosts
-
-FQDN=`hostname`
-
-echo "hostname: ${FQDN}"
-
-ADMIN="admin"
-PASS="airflow"
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-ln -sf /dev/urandom /dev/random
-
-cp ${DIR}/kdc.conf /etc/krb5kdc/kdc.conf
-cp ${DIR}/kadm5.acl /etc/krb5kdc/kadm5.acl
-cp ${DIR}/krb5.conf /etc/krb5.conf
-
-# create kerberos database
-echo -e "${PASS}\n${PASS}" | kdb5_util create -s
-# create admin
-echo -e "${PASS}\n${PASS}" | kadmin.local -q "addprinc ${ADMIN}/admin"
-# create airflow
-echo -e "${PASS}\n${PASS}" | kadmin.local -q "addprinc -randkey airflow"
-echo -e "${PASS}\n${PASS}" | kadmin.local -q "addprinc -randkey airflow/${FQDN}"
-kadmin.local -q "ktadd -k ${KRB5_KTNAME} airflow"
-kadmin.local -q "ktadd -k ${KRB5_KTNAME} airflow/${FQDN}"
-
-service krb5-kdc restart
-
-# make sure the keytab is readable to anyone
-chmod 664 ${KRB5_KTNAME}
-
-# don't do a kinit here as this happens under super user privileges
-# on travis
-# kinit -kt ${KRB5_KTNAME} airflow
-
diff --git a/scripts/ci/test-environment.sh b/scripts/ci/test-environment.sh
new file mode 100644
index 0000000000000..5c402d46df287
--- /dev/null
+++ b/scripts/ci/test-environment.sh
@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+docker-compose -f scripts/ci/docker-compose.yml run --rm airflow-testing "${@-bash}"
diff --git a/scripts/perf/dags/perf_dag_1.py b/scripts/perf/dags/perf_dag_1.py
index ea9714112e3ca..46b3d3e5a2f8d 100644
--- a/scripts/perf/dags/perf_dag_1.py
+++ b/scripts/perf/dags/perf_dag_1.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/scripts/perf/dags/perf_dag_2.py b/scripts/perf/dags/perf_dag_2.py
index 2289ebb5eda7b..a6916e664c40f 100644
--- a/scripts/perf/dags/perf_dag_2.py
+++ b/scripts/perf/dags/perf_dag_2.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/scripts/perf/scheduler_ops_metrics.py b/scripts/perf/scheduler_ops_metrics.py
index d4e472d34feca..ef274d6cd233e 100644
--- a/scripts/perf/scheduler_ops_metrics.py
+++ b/scripts/perf/scheduler_ops_metrics.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,7 +17,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from datetime import datetime
 import logging
 import pandas as pd
 import sys
@@ -25,6 +24,7 @@
 from airflow import configuration, settings
 from airflow.jobs import SchedulerJob
 from airflow.models import DagBag, DagModel, DagRun, TaskInstance
+from airflow.utils import timezone
 from airflow.utils.state import State
 
 SUBDIR = 'scripts/perf/dags'
@@ -53,7 +53,10 @@ class SchedulerMetricsJob(SchedulerJob):
     run on remote systems and spend the majority of their time on I/O wait.
 
     To Run:
-        $ python scripts/perf/scheduler_ops_metrics.py
+        $ python scripts/perf/scheduler_ops_metrics.py [timeout]
+
+    You can specify timeout in seconds as an optional parameter.
+    Its default value is 6 seconds.
     """
     __mapper_args__ = {
         'polymorphic_identity': 'SchedulerMetricsJob'
@@ -71,7 +74,7 @@ def print_stats(self):
             .filter(TI.dag_id.in_(DAG_IDS))
             .all()
         )
-        successful_tis = filter(lambda x: x.state == State.SUCCESS, tis)
+        successful_tis = [x for x in tis if x.state == State.SUCCESS]
         ti_perf = [(ti.dag_id, ti.task_id, ti.execution_date,
                     (ti.queued_dttm - self.start_date).total_seconds(),
                     (ti.start_date - self.start_date).total_seconds(),
@@ -117,17 +120,16 @@ def heartbeat(self):
         dagbag = DagBag(SUBDIR)
         dags = [dagbag.dags[dag_id] for dag_id in DAG_IDS]
         # the tasks in perf_dag_1 and per_dag_2 have a daily schedule interval.
-        num_task_instances = sum([(datetime.today() - task.start_date).days
+        num_task_instances = sum([(timezone.utcnow() - task.start_date).days
                                  for dag in dags for task in dag.tasks])
 
         if (len(successful_tis) == num_task_instances or
-                (datetime.now()-self.start_date).total_seconds() >
+                (timezone.utcnow() - self.start_date).total_seconds() >
                 MAX_RUNTIME_SECS):
-            if (len(successful_tis) == num_task_instances):
+            if len(successful_tis) == num_task_instances:
                 self.log.info("All tasks processed! Printing stats.")
             else:
-                self.log.info("Test timeout reached. "
-                                 "Printing available stats.")
+                self.log.info("Test timeout reached. Printing available stats.")
             self.print_stats()
             set_dags_paused_state(True)
             sys.exit()
@@ -178,6 +180,17 @@ def set_dags_paused_state(is_paused):
 
 
 def main():
+    global MAX_RUNTIME_SECS
+    if len(sys.argv) > 1:
+        try:
+            max_runtime_secs = int(sys.argv[1])
+            if max_runtime_secs < 1:
+                raise ValueError
+            MAX_RUNTIME_SECS = max_runtime_secs
+        except ValueError:
+            logging.error('Specify a positive integer for timeout.')
+            sys.exit(1)
+
     configuration.load_test_config()
 
     set_dags_paused_state(False)
diff --git a/scripts/systemd/README b/scripts/systemd/README
index c53144a04c7bd..e32df5f52f638 100644
--- a/scripts/systemd/README
+++ b/scripts/systemd/README
@@ -6,6 +6,6 @@ You can then start the different servers by using systemctl start <service>. Ena
  systemctl enable <service>.
 
 By default the environment configuration points to /etc/sysconfig/airflow . You can copy the "airflow" file in this
-directory and adjust it to your liking. Make sure to specify the SCHEDULER_RUNS variable.
+directory and adjust it to your liking.
 
-With some minor changes they probably work on other systemd systems.
\ No newline at end of file
+With some minor changes they probably work on other systemd systems.
diff --git a/setup.cfg b/setup.cfg
index 622cc1303a173..35785162aaf45 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -14,12 +14,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
 [metadata]
 name = Airflow
 summary = Airflow is a system to programmatically author, schedule and monitor data pipelines.
 description-file = README.md
 author = Apache Airflow PMC
-author-email = dev@airflow.incubator.apache.org
+author-email = dev@airflow.apache.org
 license = Apache License, Version 2.0
 
 [files]
@@ -35,3 +36,14 @@ upload-dir = docs/_build/html
 
 [easy_install]
 
+[mypy]
+ignore_missing_imports = True
+
+[mypy-airflow.migrations.*]
+ignore_errors = True
+
+[mypy-airflow._vendor.*]
+ignore_errors = True
+
+[mypy-airflow.contrib.auth.*]
+ignore_errors = True
diff --git a/setup.py b/setup.py
index 0b012eec0724f..006de0ad444a8 100644
--- a/setup.py
+++ b/setup.py
@@ -21,9 +21,11 @@
 from setuptools.command.test import test as TestCommand
 
 import imp
+import io
 import logging
 import os
 import sys
+import subprocess
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +35,9 @@
 
 PY3 = sys.version_info[0] == 3
 
+with io.open('README.md', encoding='utf-8') as f:
+    long_description = f.read()
+
 
 class Tox(TestCommand):
     user_options = [('tox-args=', None, "Arguments to pass to tox")]
@@ -67,6 +72,23 @@ def run(self):
         os.system('rm -vrf ./build ./dist ./*.pyc ./*.tgz ./*.egg-info')
 
 
+class CompileAssets(Command):
+    """
+    Custom compile assets command to compile and build the frontend
+    assets using npm and webpack.
+    """
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        subprocess.call('./airflow/www_rbac/compile_assets.sh')
+
+
 def git_version(version):
     """
     Return a version to identify the state of the underlying git repo. The version will
@@ -102,7 +124,8 @@ def write_version(filename=os.path.join(*['airflow',
     with open(filename, 'w') as a:
         a.write(text)
 
-async = [
+
+async_packages = [
     'greenlet>=0.4.9',
     'eventlet>= 0.9.7',
     'gevent>=0.13'
@@ -114,10 +137,13 @@ def write_version(filename=os.path.join(*['airflow',
     'azure-mgmt-datalake-store==0.4.0',
     'azure-datalake-store==0.0.19'
 ]
+azure_cosmos = ['azure-cosmos>=3.0.1']
+azure_container_instances = ['azure-mgmt-containerinstance']
 cassandra = ['cassandra-driver>=3.13.0']
 celery = [
     'celery>=4.1.1, <4.2.0',
-    'flower>=0.7.3, <1.0'
+    'flower>=0.7.3, <1.0',
+    'tornado>=4.2.0, <6.0',  # Dep of flower. Pin to a version that works on Py3.5.2
 ]
 cgroups = [
     'cgroupspy>=0.1.4',
@@ -128,38 +154,50 @@ def write_version(filename=os.path.join(*['airflow',
 dask = [
     'distributed>=1.17.1, <2'
 ]
-databricks = ['requests>=2.5.1, <3']
+databricks = ['requests>=2.20.0, <3']
 datadog = ['datadog>=0.14.0']
 doc = [
-    'sphinx>=1.2.3',
     'sphinx-argparse>=0.1.13',
+    'sphinx-autoapi>=0.7.1',
+    'Sphinx-PyPI-upload>=0.2.1',
     'sphinx-rtd-theme>=0.1.6',
-    'Sphinx-PyPI-upload>=0.2.1'
+    'sphinx>=1.2.3',
+    'sphinxcontrib-httpdomain>=1.7.0',
 ]
-docker = ['docker>=2.0.0']
+docker = ['docker~=3.0']
 druid = ['pydruid>=0.4.1']
 elasticsearch = [
     'elasticsearch>=5.0.0,<6.0.0',
     'elasticsearch-dsl>=5.0.0,<6.0.0'
 ]
-emr = ['boto3>=1.0.0']
+emr = ['boto3>=1.0.0, <1.8.0']
 gcp_api = [
-    'httplib2>=0.9.2',
     'google-api-python-client>=1.6.0, <2.0.0dev',
-    'google-auth>=1.0.0, <2.0.0dev',
     'google-auth-httplib2>=0.0.1',
+    'google-auth>=1.0.0, <2.0.0dev',
+    'google-cloud-bigtable==0.31.0',
     'google-cloud-container>=0.1.1',
+    'google-cloud-language>=1.1.1',
+    'google-cloud-spanner>=1.7.1',
+    'google-cloud-translate>=1.3.3',
+    'google-cloud-vision>=0.35.2',
+    'grpcio-gcp>=0.2.2',
+    'httplib2~=0.9.2',
+    'pandas-gbq',
     'PyOpenSSL',
-    'pandas-gbq'
 ]
-github_enterprise = ['Flask-OAuthlib>=0.9.1']
+flask_oauth = [
+    'Flask-OAuthlib>=0.9.1',
+    'oauthlib!=2.0.3,!=2.0.4,!=2.0.5,<3.0.0,>=1.1.2',
+    'requests-oauthlib==1.1.0'
+]
 hdfs = ['snakebite>=2.7.8']
 hive = [
     'hmsclient>=0.1.0',
     'pyhive>=0.6.0',
 ]
 jdbc = ['jaydebeapi>=1.1.1']
-jenkins = ['python-jenkins>=0.4.15']
+jenkins = ['python-jenkins>=1.0.0']
 jira = ['JIRA>1.0.7']
 kerberos = ['pykerberos>=1.1.13',
             'requests_kerberos>=0.10.0',
@@ -167,29 +205,29 @@ def write_version(filename=os.path.join(*['airflow',
             'snakebite[kerberos]>=2.7.8']
 kubernetes = ['kubernetes>=3.0.0',
               'cryptography>=2.0.0']
-ldap = ['ldap3>=0.9.9.1']
+ldap = ['ldap3>=2.5.1']
 mssql = ['pymssql>=2.1.1']
-mysql = ['mysqlclient>=1.3.6']
+mysql = ['mysqlclient>=1.3.6,<1.4']
 oracle = ['cx_Oracle>=5.1.2']
 password = [
     'bcrypt>=2.0.0',
     'flask-bcrypt>=0.7.1',
 ]
-pinot = ['pinotdb>=0.1.1']
-postgres = ['psycopg2-binary>=2.7.4']
+pinot = ['pinotdb==0.1.1']
+postgres = ['psycopg2>=2.7.4,<2.8']
 qds = ['qds-sdk>=1.9.6']
 rabbitmq = ['librabbitmq>=1.6.1']
-redis = ['redis>=2.10.5']
-s3 = ['boto3>=1.7.0']
+redis = ['redis~=3.2']
+s3 = ['boto3>=1.7.0, <1.8.0']
 salesforce = ['simple-salesforce>=0.72']
 samba = ['pysmbclient>=0.1.3']
 segment = ['analytics-python>=1.2.9']
-sendgrid = ['sendgrid>=5.2.0']
+sendgrid = ['sendgrid>=5.2.0,<6']
 slack = ['slackclient>=1.0.0']
 mongo = ['pymongo>=3.6.0']
 snowflake = ['snowflake-connector-python>=1.5.2',
              'snowflake-sqlalchemy>=1.1.0']
-ssh = ['paramiko>=2.1.1', 'pysftp>=0.2.9']
+ssh = ['paramiko>=2.1.1', 'pysftp>=0.2.9', 'sshtunnel>=0.1.4,<0.2']
 statsd = ['statsd>=3.0.1, <4.0']
 vertica = ['vertica-python>=0.5.1']
 webhdfs = ['hdfs[dataframe,avro,kerberos]>=2.0.4']
@@ -200,13 +238,13 @@ def write_version(filename=os.path.join(*['airflow',
     + cassandra + mongo
 
 devel = [
-    'click',
+    'beautifulsoup4~=4.7.1',
+    'click==6.7',
     'freezegun',
     'jira',
-    'lxml>=3.3.4',
-    'mock',
+    'mock;python_version<"3.3"',
     'mongomock',
-    'moto==1.1.19',
+    'moto==1.3.5',
     'nose',
     'nose-ignore-docstring==0.2',
     'nose-timer',
@@ -216,21 +254,24 @@ def write_version(filename=os.path.join(*['airflow',
     'pywinrm',
     'qds-sdk>=1.9.6',
     'rednose',
-    'requests_mock'
+    'requests_mock',
+    'flake8>=3.6.0',
 ]
 devel_minreq = devel + kubernetes + mysql + doc + password + s3 + cgroups
 devel_hadoop = devel_minreq + hive + hdfs + webhdfs + kerberos
+devel_azure = devel_minreq + azure_data_lake + azure_cosmos
 devel_all = (sendgrid + devel + all_dbs + doc + samba + s3 + slack + crypto + oracle +
              docker + ssh + kubernetes + celery + azure_blob_storage + redis + gcp_api +
              datadog + zendesk + jdbc + ldap + kerberos + password + webhdfs + jenkins +
-             druid + pinot + segment + snowflake + elasticsearch + azure_data_lake, atlas)
+             druid + pinot + segment + snowflake + elasticsearch + azure_data_lake + azure_cosmos +
+             atlas + azure_container_instances)
 
 # Snakebite & Google Cloud Dataflow are not Python 3 compatible :'(
 if PY3:
     devel_ci = [package for package in devel_all if package not in
                 ['snakebite>=2.7.8', 'snakebite[kerberos]>=2.7.8']]
 else:
-    devel_ci = devel_all
+    devel_ci = devel_all + ['unittest2']
 
 
 def do_setup():
@@ -238,6 +279,8 @@ def do_setup():
     setup(
         name='apache-airflow',
         description='Programmatically author, schedule and monitor data pipelines',
+        long_description=long_description,
+        long_description_content_type='text/markdown',
         license='Apache License 2.0',
         version=version,
         packages=find_packages(exclude=['tests*']),
@@ -246,39 +289,40 @@ def do_setup():
         zip_safe=False,
         scripts=['airflow/bin/airflow'],
         install_requires=[
-            'alembic>=0.8.3, <0.9',
-            'bleach==2.1.2',
+            'alembic>=0.9, <1.0',
             'configparser>=3.5.0, <3.6.0',
             'croniter>=0.3.17, <0.4',
             'dill>=0.2.2, <0.3',
-            'flask>=0.12.4, <0.13',
-            'flask-appbuilder>=1.11.1, <2.0.0',
-            'flask-admin==1.4.1',
+            'enum34~=1.1.6;python_version<"3.4"',
+            'flask>=1.0, <2.0',
+            'flask-appbuilder==1.12.3',
+            'flask-admin==1.5.3',
             'flask-caching>=1.3.3, <1.4.0',
-            'flask-login==0.2.11',
+            'flask-login>=0.3, <0.5',
             'flask-swagger==0.2.13',
             'flask-wtf>=0.14.2, <0.15',
             'funcsigs==1.0.0',
             'future>=0.16.0, <0.17',
             'gitpython>=2.0.2',
-            'gunicorn>=19.4.0, <20.0',
+            'gunicorn>=19.5.0, <20.0',
             'iso8601>=0.1.12',
-            'jinja2>=2.7.3, <2.9.0',
-            'lxml>=3.6.0, <4.0',
+            'json-merge-patch==0.2',
+            'jinja2>=2.7.3, <=2.10.0',
+            'lxml>=4.0.0',
             'markdown>=2.5.2, <3.0',
             'pandas>=0.17.1, <1.0.0',
             'pendulum==1.4.4',
-            'psutil>=4.2.0, <5.0.0',
+            'psutil>=4.2.0, <6.0.0',
             'pygments>=2.0.1, <3.0',
             'python-daemon>=2.1.1, <2.2',
             'python-dateutil>=2.3, <3',
-            'python-nvd3==0.15.0',
-            'requests>=2.5.1, <3',
+            'requests>=2.20.0, <3',
             'setproctitle>=1.1.8, <2',
-            'sqlalchemy>=1.1.15, <1.2.0',
-            'sqlalchemy-utc>=0.9.0',
-            'tabulate>=0.7.5, <0.8.0',
-            'tenacity==4.8.0',
+            'sqlalchemy>=1.1.15, <1.3.0',
+            'tabulate>=0.7.5, <0.9',
+            'tenacity==4.12.0',
+            'text-unidecode==1.2',
+            'typing;python_version<"3.5"',
             'thrift>=0.9.2',
             'tzlocal>=1.4',
             'unicodecsv>=0.14.1',
@@ -293,9 +337,11 @@ def do_setup():
             'devel_ci': devel_ci,
             'all_dbs': all_dbs,
             'atlas': atlas,
-            'async': async,
+            'async': async_packages,
             'azure_blob_storage': azure_blob_storage,
             'azure_data_lake': azure_data_lake,
+            'azure_cosmos': azure_cosmos,
+            'azure_container_instances': azure_container_instances,
             'cassandra': cassandra,
             'celery': celery,
             'cgroups': cgroups,
@@ -306,13 +352,15 @@ def do_setup():
             'datadog': datadog,
             'devel': devel_minreq,
             'devel_hadoop': devel_hadoop,
+            'devel_azure': devel_azure,
             'doc': doc,
             'docker': docker,
             'druid': druid,
             'elasticsearch': elasticsearch,
             'emr': emr,
             'gcp_api': gcp_api,
-            'github_enterprise': github_enterprise,
+            'github_enterprise': flask_oauth,
+            'google_auth': flask_oauth,
             'hdfs': hdfs,
             'hive': hive,
             'jdbc': jdbc,
@@ -351,18 +399,20 @@ def do_setup():
             'Intended Audience :: System Administrators',
             'License :: OSI Approved :: Apache Software License',
             'Programming Language :: Python :: 2.7',
-            'Programming Language :: Python :: 3.4',
+            'Programming Language :: Python :: 3.5',
             'Topic :: System :: Monitoring',
         ],
         author='Apache Software Foundation',
-        author_email='dev@airflow.incubator.apache.org',
-        url='http://airflow.incubator.apache.org/',
+        author_email='dev@airflow.apache.org',
+        url='http://airflow.apache.org/',
         download_url=(
-            'https://dist.apache.org/repos/dist/release/incubator/airflow/' + version),
+            'https://dist.apache.org/repos/dist/release/airflow/' + version),
         cmdclass={
             'test': Tox,
             'extra_clean': CleanCommand,
+            'compile_assets': CompileAssets
         },
+        python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*',
     )
 
 
diff --git a/tests/__init__.py b/tests/__init__.py
index eff9d4b9f311d..7c1a4fb314ed0 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -21,16 +21,5 @@
 
 from __future__ import absolute_import
 
-from .api import *
-from .configuration import *
-from .contrib import *
-from .core import *
-from .executors import *
-from .jobs import *
-from .impersonation import *
-from .lineage import *
-from .models import *
-from .operators import *
-from .security import *
-from .task import *
-from .utils import *
+from .api import *  # type: ignore
+from .core import *  # type: ignore
diff --git a/tests/api/__init__.py b/tests/api/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/api/__init__.py
+++ b/tests/api/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/api/client/__init__.py b/tests/api/client/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/api/client/__init__.py
+++ b/tests/api/client/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/api/common/__init__.py b/tests/api/common/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/api/common/__init__.py
+++ b/tests/api/common/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/api/common/experimental/__init__.py b/tests/api/common/experimental/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/api/common/experimental/__init__.py
+++ b/tests/api/common/experimental/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/api/common/experimental/mark_tasks.py b/tests/api/common/experimental/mark_tasks.py
deleted file mode 100644
index 3ad50539c49ac..0000000000000
--- a/tests/api/common/experimental/mark_tasks.py
+++ /dev/null
@@ -1,401 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import unittest
-
-from airflow import models
-from airflow.api.common.experimental.mark_tasks import (
-    set_state, _create_dagruns, set_dag_run_state)
-from airflow.settings import Session
-from airflow.utils.dates import days_ago
-from airflow.utils.state import State
-from datetime import datetime, timedelta
-
-DEV_NULL = "/dev/null"
-
-
-class TestMarkTasks(unittest.TestCase):
-
-    def setUp(self):
-        self.dagbag = models.DagBag(include_examples=True)
-        self.dag1 = self.dagbag.dags['test_example_bash_operator']
-        self.dag2 = self.dagbag.dags['example_subdag_operator']
-
-        self.execution_dates = [days_ago(2), days_ago(1)]
-
-        drs = _create_dagruns(self.dag1, self.execution_dates,
-                              state=State.RUNNING,
-                              run_id_template="scheduled__{}")
-        for dr in drs:
-            dr.dag = self.dag1
-            dr.verify_integrity()
-
-        drs = _create_dagruns(self.dag2,
-                              [self.dag2.default_args['start_date']],
-                              state=State.RUNNING,
-                              run_id_template="scheduled__{}")
-
-        for dr in drs:
-            dr.dag = self.dag2
-            dr.verify_integrity()
-
-        self.session = Session()
-
-    def tearDown(self):
-        self.dag1.clear()
-        self.dag2.clear()
-
-        # just to make sure we are fully cleaned up
-        self.session.query(models.DagRun).delete()
-        self.session.query(models.TaskInstance).delete()
-        self.session.commit()
-        self.session.close()
-
-    def snapshot_state(self, dag, execution_dates):
-        TI = models.TaskInstance
-        tis = self.session.query(TI).filter(
-            TI.dag_id==dag.dag_id,
-            TI.execution_date.in_(execution_dates)
-        ).all()
-
-        self.session.expunge_all()
-
-        return tis
-
-    def verify_state(self, dag, task_ids, execution_dates, state, old_tis):
-        TI = models.TaskInstance
-
-        tis = self.session.query(TI).filter(
-            TI.dag_id==dag.dag_id,
-            TI.execution_date.in_(execution_dates)
-        ).all()
-
-        self.assertTrue(len(tis) > 0)
-
-        for ti in tis:
-            if ti.task_id in task_ids and ti.execution_date in execution_dates:
-                self.assertEqual(ti.state, state)
-            else:
-                for old_ti in old_tis:
-                    if (old_ti.task_id == ti.task_id
-                            and old_ti.execution_date == ti.execution_date):
-                            self.assertEqual(ti.state, old_ti.state)
-
-    def test_mark_tasks_now(self):
-        # set one task to success but do not commit
-        snapshot = self.snapshot_state(self.dag1, self.execution_dates)
-        task = self.dag1.get_task("runme_1")
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=False, future=False,
-                            past=False, state=State.SUCCESS, commit=False)
-        self.assertEqual(len(altered), 1)
-        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
-                          None, snapshot)
-
-        # set one and only one task to success
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=False, future=False,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 1)
-        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
-                          State.SUCCESS, snapshot)
-
-        # set no tasks
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=False, future=False,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 0)
-        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
-                          State.SUCCESS, snapshot)
-
-        # set task to other than success
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=False, future=False,
-                            past=False, state=State.FAILED, commit=True)
-        self.assertEqual(len(altered), 1)
-        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
-                          State.FAILED, snapshot)
-
-        # dont alter other tasks
-        snapshot = self.snapshot_state(self.dag1, self.execution_dates)
-        task = self.dag1.get_task("runme_0")
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=False, future=False,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 1)
-        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
-                          State.SUCCESS, snapshot)
-
-    def test_mark_downstream(self):
-        # test downstream
-        snapshot = self.snapshot_state(self.dag1, self.execution_dates)
-        task = self.dag1.get_task("runme_1")
-        relatives = task.get_flat_relatives(upstream=False)
-        task_ids = [t.task_id for t in relatives]
-        task_ids.append(task.task_id)
-
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=True, future=False,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 3)
-        self.verify_state(self.dag1, task_ids, [self.execution_dates[0]],
-                          State.SUCCESS, snapshot)
-
-    def test_mark_upstream(self):
-        # test upstream
-        snapshot = self.snapshot_state(self.dag1, self.execution_dates)
-        task = self.dag1.get_task("run_after_loop")
-        relatives = task.get_flat_relatives(upstream=True)
-        task_ids = [t.task_id for t in relatives]
-        task_ids.append(task.task_id)
-
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=True, downstream=False, future=False,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 4)
-        self.verify_state(self.dag1, task_ids, [self.execution_dates[0]],
-                          State.SUCCESS, snapshot)
-
-    def test_mark_tasks_future(self):
-        # set one task to success towards end of scheduled dag runs
-        snapshot = self.snapshot_state(self.dag1, self.execution_dates)
-        task = self.dag1.get_task("runme_1")
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=False, future=True,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 2)
-        self.verify_state(self.dag1, [task.task_id], self.execution_dates,
-                          State.SUCCESS, snapshot)
-
-    def test_mark_tasks_past(self):
-        # set one task to success towards end of scheduled dag runs
-        snapshot = self.snapshot_state(self.dag1, self.execution_dates)
-        task = self.dag1.get_task("runme_1")
-        altered = set_state(task=task, execution_date=self.execution_dates[1],
-                            upstream=False, downstream=False, future=False,
-                            past=True, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 2)
-        self.verify_state(self.dag1, [task.task_id], self.execution_dates,
-                          State.SUCCESS, snapshot)
-
-    def test_mark_tasks_subdag(self):
-        # set one task to success towards end of scheduled dag runs
-        task = self.dag2.get_task("section-1")
-        relatives = task.get_flat_relatives(upstream=False)
-        task_ids = [t.task_id for t in relatives]
-        task_ids.append(task.task_id)
-
-        altered = set_state(task=task, execution_date=self.execution_dates[0],
-                            upstream=False, downstream=True, future=False,
-                            past=False, state=State.SUCCESS, commit=True)
-        self.assertEqual(len(altered), 14)
-
-        # cannot use snapshot here as that will require drilling down the
-        # the sub dag tree essentially recreating the same code as in the
-        # tested logic.
-        self.verify_state(self.dag2, task_ids, [self.execution_dates[0]],
-                          State.SUCCESS, [])
-
-
-class TestMarkDAGRun(unittest.TestCase):
-    def setUp(self):
-        self.dagbag = models.DagBag(include_examples=True)
-        self.dag1 = self.dagbag.dags['test_example_bash_operator']
-        self.dag2 = self.dagbag.dags['example_subdag_operator']
-
-        self.execution_dates = [days_ago(3), days_ago(2), days_ago(1)]
-
-        self.session = Session()
-
-    def verify_dag_run_states(self, dag, date, state=State.SUCCESS):
-        drs = models.DagRun.find(dag_id=dag.dag_id, execution_date=date)
-        dr = drs[0]
-        self.assertEqual(dr.get_state(), state)
-        tis = dr.get_task_instances(session=self.session)
-        for ti in tis:
-            self.assertEqual(ti.state, state)
-
-    def test_set_running_dag_run_state(self):
-        date = self.execution_dates[0]
-        dr = self.dag1.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.RUNNING,
-            execution_date=date,
-            session=self.session
-        )
-        for ti in dr.get_task_instances(session=self.session):
-            ti.set_state(State.RUNNING, self.session)
-
-        altered = set_dag_run_state(self.dag1, date, state=State.SUCCESS, commit=True)
-
-        # All of the task should be altered
-        self.assertEqual(len(altered), len(self.dag1.tasks))
-        self.verify_dag_run_states(self.dag1, date)
-
-    def test_set_success_dag_run_state(self):
-        date = self.execution_dates[0]
-
-        dr = self.dag1.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.SUCCESS,
-            execution_date=date,
-            session=self.session
-        )
-        for ti in dr.get_task_instances(session=self.session):
-            ti.set_state(State.SUCCESS, self.session)
-
-        altered = set_dag_run_state(self.dag1, date, state=State.SUCCESS, commit=True)
-
-        # None of the task should be altered
-        self.assertEqual(len(altered), 0)
-        self.verify_dag_run_states(self.dag1, date)
-
-    def test_set_failed_dag_run_state(self):
-        date = self.execution_dates[0]
-        dr = self.dag1.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.FAILED,
-            execution_date=date,
-            session=self.session
-        )
-        dr.get_task_instance('runme_0').set_state(State.FAILED, self.session)
-
-        altered = set_dag_run_state(self.dag1, date, state=State.SUCCESS, commit=True)
-
-        # All of the task should be altered
-        self.assertEqual(len(altered), len(self.dag1.tasks))
-        self.verify_dag_run_states(self.dag1, date)
-
-    def test_set_mixed_dag_run_state(self):
-        """
-        This test checks function set_dag_run_state with mixed task instance
-        state.
-        """
-        date = self.execution_dates[0]
-        dr = self.dag1.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.FAILED,
-            execution_date=date,
-            session=self.session
-        )
-        # success task
-        dr.get_task_instance('runme_0').set_state(State.SUCCESS, self.session)
-        # skipped task
-        dr.get_task_instance('runme_1').set_state(State.SKIPPED, self.session)
-        # retry task
-        dr.get_task_instance('runme_2').set_state(State.UP_FOR_RETRY, self.session)
-        # queued task
-        dr.get_task_instance('also_run_this').set_state(State.QUEUED, self.session)
-        # running task
-        dr.get_task_instance('run_after_loop').set_state(State.RUNNING, self.session)
-        # failed task
-        dr.get_task_instance('run_this_last').set_state(State.FAILED, self.session)
-
-        altered = set_dag_run_state(self.dag1, date, state=State.SUCCESS, commit=True)
-
-        self.assertEqual(len(altered), len(self.dag1.tasks) - 1) # only 1 task succeeded
-        self.verify_dag_run_states(self.dag1, date)
-
-    def test_set_state_without_commit(self):
-        date = self.execution_dates[0]
-
-        # Running dag run and task instances
-        dr = self.dag1.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.RUNNING,
-            execution_date=date,
-            session=self.session
-        )
-        for ti in dr.get_task_instances(session=self.session):
-            ti.set_state(State.RUNNING, self.session)
-
-        altered = set_dag_run_state(self.dag1, date, state=State.SUCCESS, commit=False)
-
-        # All of the task should be altered
-        self.assertEqual(len(altered), len(self.dag1.tasks))
-
-        # Both dag run and task instances' states should remain the same
-        self.verify_dag_run_states(self.dag1, date, State.RUNNING)
-
-    def test_set_state_with_multiple_dagruns(self):
-        dr1 = self.dag2.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.FAILED,
-            execution_date=self.execution_dates[0],
-            session=self.session
-        )
-        dr2 = self.dag2.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.FAILED,
-            execution_date=self.execution_dates[1],
-            session=self.session
-        )
-        dr3 = self.dag2.create_dagrun(
-            run_id='manual__' + datetime.now().isoformat(),
-            state=State.RUNNING,
-            execution_date=self.execution_dates[2],
-            session=self.session
-        )
-
-        altered = set_dag_run_state(self.dag2, self.execution_dates[1],
-                                state=State.SUCCESS, commit=True)
-
-        # Recursively count number of tasks in the dag
-        def count_dag_tasks(dag):
-            count = len(dag.tasks)
-            subdag_counts = [count_dag_tasks(subdag) for subdag in dag.subdags]
-            count += sum(subdag_counts)
-            return count
-
-        self.assertEqual(len(altered), count_dag_tasks(self.dag2))
-        self.verify_dag_run_states(self.dag2, self.execution_dates[1])
-
-        # Make sure other dag status are not changed
-        dr1 = models.DagRun.find(dag_id=self.dag2.dag_id, execution_date=self.execution_dates[0])
-        dr1 = dr1[0]
-        self.assertEqual(dr1.get_state(), State.FAILED)
-        dr3 = models.DagRun.find(dag_id=self.dag2.dag_id, execution_date=self.execution_dates[2])
-        dr3 = dr3[0]
-        self.assertEqual(dr3.get_state(), State.RUNNING)
-
-    def test_set_dag_run_state_edge_cases(self):
-        # Dag does not exist
-        altered = set_dag_run_state(None, self.execution_dates[0])
-        self.assertEqual(len(altered), 0)
-
-        # Invalid execution date
-        altered = set_dag_run_state(self.dag1, None)
-        self.assertEqual(len(altered), 0)
-        self.assertRaises(AssertionError, set_dag_run_state, self.dag1, timedelta(microseconds=-1))
-
-        # DagRun does not exist
-        # This will throw AssertionError since dag.latest_execution_date does not exist
-        self.assertRaises(AssertionError, set_dag_run_state, self.dag1, self.execution_dates[0])
-
-    def tearDown(self):
-        self.dag1.clear()
-        self.dag2.clear()
-
-        self.session.query(models.DagRun).delete()
-        self.session.query(models.TaskInstance).delete()
-        self.session.query(models.DagStat).delete()
-        self.session.commit()
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/tests/api/common/experimental/test_delete_dag.py b/tests/api/common/experimental/test_delete_dag.py
new file mode 100644
index 0000000000000..3ab2c030a4e60
--- /dev/null
+++ b/tests/api/common/experimental/test_delete_dag.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import models
+from airflow.api.common.experimental.delete_dag import delete_dag
+from airflow.exceptions import DagNotFound, DagFileExists
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.utils.dates import days_ago
+from airflow.utils.db import create_session
+from airflow.utils.state import State
+
+DM = models.DagModel
+DR = models.DagRun
+TI = models.TaskInstance
+LOG = models.log.Log
+TF = models.TaskFail
+TR = models.TaskReschedule
+
+
+class TestDeleteDAGCatchError(unittest.TestCase):
+
+    def setUp(self):
+        self.dagbag = models.DagBag(include_examples=True)
+        self.dag_id = 'example_bash_operator'
+        self.dag = self.dagbag.dags[self.dag_id]
+
+    def tearDown(self):
+        self.dag.clear()
+
+    def test_delete_dag_non_existent_dag(self):
+        with self.assertRaises(DagNotFound):
+            delete_dag("non-existent DAG")
+
+    def test_delete_dag_dag_still_in_dagbag(self):
+        with create_session() as session:
+            models_to_check = ['DagModel', 'DagRun', 'TaskInstance']
+            record_counts = {}
+
+            for model_name in models_to_check:
+                m = getattr(models, model_name)
+                record_counts[model_name] = session.query(m).filter(m.dag_id == self.dag_id).count()
+
+            with self.assertRaises(DagFileExists):
+                delete_dag(self.dag_id)
+
+            # No change should happen in DB
+            for model_name in models_to_check:
+                m = getattr(models, model_name)
+                self.assertEqual(
+                    session.query(m).filter(
+                        m.dag_id == self.dag_id
+                    ).count(),
+                    record_counts[model_name]
+                )
+
+
+class TestDeleteDAGSuccessfulDelete(unittest.TestCase):
+
+    def setUp(self):
+        self.key = "test_dag_id"
+
+        task = DummyOperator(task_id='dummy',
+                             dag=models.DAG(dag_id=self.key,
+                                            default_args={'start_date': days_ago(2)}),
+                             owner='airflow')
+
+        d = days_ago(1)
+        with create_session() as session:
+            session.add(DM(dag_id=self.key))
+            session.add(DR(dag_id=self.key))
+            session.add(TI(task=task,
+                           execution_date=d,
+                           state=State.SUCCESS))
+            # flush to ensure task instance if written before
+            # task reschedule because of FK constraint
+            session.flush()
+            session.add(LOG(dag_id=self.key, task_id=None, task_instance=None,
+                            execution_date=d, event="varimport"))
+            session.add(TF(task=task, execution_date=d,
+                           start_date=d, end_date=d))
+            session.add(TR(task=task, execution_date=d,
+                           start_date=d, end_date=d,
+                           try_number=1, reschedule_date=d))
+
+    def tearDown(self):
+        with create_session() as session:
+            session.query(TR).filter(TR.dag_id == self.key).delete()
+            session.query(TF).filter(TF.dag_id == self.key).delete()
+            session.query(TI).filter(TI.dag_id == self.key).delete()
+            session.query(DR).filter(DR.dag_id == self.key).delete()
+            session.query(DM).filter(DM.dag_id == self.key).delete()
+            session.query(LOG).filter(LOG.dag_id == self.key).delete()
+
+    def test_delete_dag_successful_delete(self):
+        with create_session() as session:
+            self.assertEqual(session.query(DM).filter(DM.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(DR).filter(DR.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(TI).filter(TI.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(TF).filter(TF.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(TR).filter(TR.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(LOG).filter(LOG.dag_id == self.key).count(), 1)
+
+        delete_dag(dag_id=self.key)
+
+        with create_session() as session:
+            self.assertEqual(session.query(DM).filter(DM.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(DR).filter(DR.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(TI).filter(TI.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(TF).filter(TF.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(TR).filter(TR.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(LOG).filter(LOG.dag_id == self.key).count(), 1)
+
+    def test_delete_dag_successful_delete_not_keeping_records_in_log(self):
+
+        with create_session() as session:
+            self.assertEqual(session.query(DM).filter(DM.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(DR).filter(DR.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(TI).filter(TI.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(TF).filter(TF.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(TR).filter(TR.dag_id == self.key).count(), 1)
+            self.assertEqual(session.query(LOG).filter(LOG.dag_id == self.key).count(), 1)
+
+        delete_dag(dag_id=self.key, keep_records_in_log=False)
+
+        with create_session() as session:
+            self.assertEqual(session.query(DM).filter(DM.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(DR).filter(DR.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(TI).filter(TI.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(TF).filter(TF.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(TR).filter(TR.dag_id == self.key).count(), 0)
+            self.assertEqual(session.query(LOG).filter(LOG.dag_id == self.key).count(), 0)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/api/common/experimental/test_mark_tasks.py b/tests/api/common/experimental/test_mark_tasks.py
new file mode 100644
index 0000000000000..28df2f6a46510
--- /dev/null
+++ b/tests/api/common/experimental/test_mark_tasks.py
@@ -0,0 +1,520 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+import time
+from datetime import datetime
+
+from airflow import configuration, models
+from airflow.api.common.experimental.mark_tasks import (
+    set_state, _create_dagruns, set_dag_run_state_to_success, set_dag_run_state_to_failed,
+    set_dag_run_state_to_running)
+from airflow.utils import timezone
+from airflow.utils.db import create_session, provide_session
+from airflow.utils.dates import days_ago
+from airflow.utils.state import State
+from airflow.models import DagRun
+
+DEV_NULL = "/dev/null"
+
+configuration.load_test_config()
+
+
+class TestMarkTasks(unittest.TestCase):
+
+    def setUp(self):
+        self.dagbag = models.DagBag(include_examples=True)
+        self.dag1 = self.dagbag.dags['example_bash_operator']
+        self.dag2 = self.dagbag.dags['example_subdag_operator']
+
+        self.execution_dates = [days_ago(2), days_ago(1)]
+
+        drs = _create_dagruns(self.dag1, self.execution_dates,
+                              state=State.RUNNING,
+                              run_id_template="scheduled__{}")
+        for dr in drs:
+            dr.dag = self.dag1
+            dr.verify_integrity()
+
+        drs = _create_dagruns(self.dag2,
+                              [self.dag2.default_args['start_date']],
+                              state=State.RUNNING,
+                              run_id_template="scheduled__{}")
+
+        for dr in drs:
+            dr.dag = self.dag2
+            dr.verify_integrity()
+
+    def tearDown(self):
+        self.dag1.clear()
+        self.dag2.clear()
+
+        # just to make sure we are fully cleaned up
+        with create_session() as session:
+            session.query(models.DagRun).delete()
+            session.query(models.TaskInstance).delete()
+
+    @staticmethod
+    def snapshot_state(dag, execution_dates):
+        TI = models.TaskInstance
+        with create_session() as session:
+            return session.query(TI).filter(
+                TI.dag_id == dag.dag_id,
+                TI.execution_date.in_(execution_dates)
+            ).all()
+
+    @provide_session
+    def verify_state(self, dag, task_ids, execution_dates, state, old_tis, session=None):
+        TI = models.TaskInstance
+
+        tis = session.query(TI).filter(
+            TI.dag_id == dag.dag_id,
+            TI.execution_date.in_(execution_dates)
+        ).all()
+
+        self.assertTrue(len(tis) > 0)
+
+        for ti in tis:
+            if ti.task_id in task_ids and ti.execution_date in execution_dates:
+                self.assertEqual(ti.state, state)
+            else:
+                for old_ti in old_tis:
+                    if old_ti.task_id == ti.task_id and old_ti.execution_date == ti.execution_date:
+                        self.assertEqual(ti.state, old_ti.state)
+
+    def test_mark_tasks_now(self):
+        # set one task to success but do not commit
+        snapshot = TestMarkTasks.snapshot_state(self.dag1, self.execution_dates)
+        task = self.dag1.get_task("runme_1")
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=False, future=False,
+                            past=False, state=State.SUCCESS, commit=False)
+        self.assertEqual(len(altered), 1)
+        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
+                          None, snapshot)
+
+        # set one and only one task to success
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=False, future=False,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 1)
+        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
+                          State.SUCCESS, snapshot)
+
+        # set no tasks
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=False, future=False,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 0)
+        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
+                          State.SUCCESS, snapshot)
+
+        # set task to other than success
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=False, future=False,
+                            past=False, state=State.FAILED, commit=True)
+        self.assertEqual(len(altered), 1)
+        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
+                          State.FAILED, snapshot)
+
+        # dont alter other tasks
+        snapshot = TestMarkTasks.snapshot_state(self.dag1, self.execution_dates)
+        task = self.dag1.get_task("runme_0")
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=False, future=False,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 1)
+        self.verify_state(self.dag1, [task.task_id], [self.execution_dates[0]],
+                          State.SUCCESS, snapshot)
+
+    def test_mark_downstream(self):
+        # test downstream
+        snapshot = TestMarkTasks.snapshot_state(self.dag1, self.execution_dates)
+        task = self.dag1.get_task("runme_1")
+        relatives = task.get_flat_relatives(upstream=False)
+        task_ids = [t.task_id for t in relatives]
+        task_ids.append(task.task_id)
+
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=True, future=False,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 3)
+        self.verify_state(self.dag1, task_ids, [self.execution_dates[0]], State.SUCCESS, snapshot)
+
+    def test_mark_upstream(self):
+        # test upstream
+        snapshot = TestMarkTasks.snapshot_state(self.dag1, self.execution_dates)
+        task = self.dag1.get_task("run_after_loop")
+        relatives = task.get_flat_relatives(upstream=True)
+        task_ids = [t.task_id for t in relatives]
+        task_ids.append(task.task_id)
+
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=True, downstream=False, future=False,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 4)
+        self.verify_state(self.dag1, task_ids, [self.execution_dates[0]],
+                          State.SUCCESS, snapshot)
+
+    def test_mark_tasks_future(self):
+        # set one task to success towards end of scheduled dag runs
+        snapshot = TestMarkTasks.snapshot_state(self.dag1, self.execution_dates)
+        task = self.dag1.get_task("runme_1")
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=False, future=True,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 2)
+        self.verify_state(self.dag1, [task.task_id], self.execution_dates, State.SUCCESS, snapshot)
+
+    def test_mark_tasks_past(self):
+        # set one task to success towards end of scheduled dag runs
+        snapshot = TestMarkTasks.snapshot_state(self.dag1, self.execution_dates)
+        task = self.dag1.get_task("runme_1")
+        altered = set_state(task=task, execution_date=self.execution_dates[1],
+                            upstream=False, downstream=False, future=False,
+                            past=True, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 2)
+        self.verify_state(self.dag1, [task.task_id], self.execution_dates, State.SUCCESS, snapshot)
+
+    # TODO: this skipIf should be removed once a fixing solution is found later
+    #       We skip it here because this test case is working with Postgres & SQLite
+    #       but not with MySQL
+    @unittest.skipIf('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'), "Flaky with MySQL")
+    def test_mark_tasks_subdag(self):
+        # set one task to success towards end of scheduled dag runs
+        task = self.dag2.get_task("section-1")
+        relatives = task.get_flat_relatives(upstream=False)
+        task_ids = [t.task_id for t in relatives]
+        task_ids.append(task.task_id)
+
+        altered = set_state(task=task, execution_date=self.execution_dates[0],
+                            upstream=False, downstream=True, future=False,
+                            past=False, state=State.SUCCESS, commit=True)
+        self.assertEqual(len(altered), 14)
+
+        # cannot use snapshot here as that will require drilling down the
+        # the sub dag tree essentially recreating the same code as in the
+        # tested logic.
+        self.verify_state(self.dag2, task_ids, [self.execution_dates[0]],
+                          State.SUCCESS, [])
+
+
+class TestMarkDAGRun(unittest.TestCase):
+    def setUp(self):
+        self.dagbag = models.DagBag(include_examples=True)
+        self.dag1 = self.dagbag.dags['example_bash_operator']
+        self.dag2 = self.dagbag.dags['example_subdag_operator']
+        self.execution_dates = [days_ago(2), days_ago(1), days_ago(0)]
+
+    def _set_default_task_instance_states(self, dr):
+        # success task
+        dr.get_task_instance('runme_0').set_state(State.SUCCESS)
+        # skipped task
+        dr.get_task_instance('runme_1').set_state(State.SKIPPED)
+        # retry task
+        dr.get_task_instance('runme_2').set_state(State.UP_FOR_RETRY)
+        # queued task
+        dr.get_task_instance('also_run_this').set_state(State.QUEUED)
+        # running task
+        dr.get_task_instance('run_after_loop').set_state(State.RUNNING)
+        # failed task
+        dr.get_task_instance('run_this_last').set_state(State.FAILED)
+
+    def _verify_task_instance_states_remain_default(self, dr):
+        self.assertEqual(dr.get_task_instance('runme_0').state, State.SUCCESS)
+        self.assertEqual(dr.get_task_instance('runme_1').state, State.SKIPPED)
+        self.assertEqual(dr.get_task_instance('runme_2').state, State.UP_FOR_RETRY)
+        self.assertEqual(dr.get_task_instance('also_run_this').state, State.QUEUED)
+        self.assertEqual(dr.get_task_instance('run_after_loop').state, State.RUNNING)
+        self.assertEqual(dr.get_task_instance('run_this_last').state, State.FAILED)
+
+    @provide_session
+    def _verify_task_instance_states(self, dag, date, state, session):
+        TI = models.TaskInstance
+        tis = session.query(TI)\
+            .filter(TI.dag_id == dag.dag_id, TI.execution_date == date)
+        for ti in tis:
+            self.assertEqual(ti.state, state)
+
+    def _create_test_dag_run(self, state, date):
+        return self.dag1.create_dagrun(
+            run_id='manual__' + datetime.now().isoformat(),
+            state=state,
+            execution_date=date
+        )
+
+    def _verify_dag_run_state(self, dag, date, state):
+        drs = models.DagRun.find(dag_id=dag.dag_id, execution_date=date)
+        dr = drs[0]
+
+        self.assertEqual(dr.get_state(), state)
+
+    @provide_session
+    def _verify_dag_run_dates(self, dag, date, state, middle_time, session=None):
+        # When target state is RUNNING, we should set start_date,
+        # otherwise we should set end_date.
+        DR = DagRun
+        dr = session.query(DR).filter(
+            DR.dag_id == dag.dag_id,
+            DR.execution_date == date
+        ).one()
+        if state == State.RUNNING:
+            # Since the DAG is running, the start_date must be updated after creation
+            self.assertGreater(dr.start_date, middle_time)
+            # If the dag is still running, we don't have an end date
+            self.assertIsNone(dr.end_date)
+        else:
+            # If the dag is not running, there must be an end time
+            self.assertLess(dr.start_date, middle_time)
+            self.assertGreater(dr.end_date, middle_time)
+
+    def test_set_running_dag_run_to_success(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.RUNNING, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_success(self.dag1, date, commit=True)
+
+        # All except the SUCCESS task should be altered.
+        self.assertEqual(len(altered), 5)
+        self._verify_dag_run_state(self.dag1, date, State.SUCCESS)
+        self._verify_task_instance_states(self.dag1, date, State.SUCCESS)
+        self._verify_dag_run_dates(self.dag1, date, State.SUCCESS, middle_time)
+
+    def test_set_running_dag_run_to_failed(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.RUNNING, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_failed(self.dag1, date, commit=True)
+
+        # Only running task should be altered.
+        self.assertEqual(len(altered), 1)
+        self._verify_dag_run_state(self.dag1, date, State.FAILED)
+        self.assertEqual(dr.get_task_instance('run_after_loop').state, State.FAILED)
+        self._verify_dag_run_dates(self.dag1, date, State.FAILED, middle_time)
+
+    def test_set_running_dag_run_to_running(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.RUNNING, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_running(self.dag1, date, commit=True)
+
+        # None of the tasks should be altered, only the dag itself
+        self.assertEqual(len(altered), 0)
+        self._verify_dag_run_state(self.dag1, date, State.RUNNING)
+        self._verify_task_instance_states_remain_default(dr)
+        self._verify_dag_run_dates(self.dag1, date, State.RUNNING, middle_time)
+
+    def test_set_success_dag_run_to_success(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.SUCCESS, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_success(self.dag1, date, commit=True)
+
+        # All except the SUCCESS task should be altered.
+        self.assertEqual(len(altered), 5)
+        self._verify_dag_run_state(self.dag1, date, State.SUCCESS)
+        self._verify_task_instance_states(self.dag1, date, State.SUCCESS)
+        self._verify_dag_run_dates(self.dag1, date, State.SUCCESS, middle_time)
+
+    def test_set_success_dag_run_to_failed(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.SUCCESS, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_failed(self.dag1, date, commit=True)
+
+        # Only running task should be altered.
+        self.assertEqual(len(altered), 1)
+        self._verify_dag_run_state(self.dag1, date, State.FAILED)
+        self.assertEqual(dr.get_task_instance('run_after_loop').state, State.FAILED)
+        self._verify_dag_run_dates(self.dag1, date, State.FAILED, middle_time)
+
+    def test_set_success_dag_run_to_running(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.SUCCESS, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_running(self.dag1, date, commit=True)
+
+        # None of the tasks should be altered, but only the dag object should be changed
+        self.assertEqual(len(altered), 0)
+        self._verify_dag_run_state(self.dag1, date, State.RUNNING)
+        self._verify_task_instance_states_remain_default(dr)
+        self._verify_dag_run_dates(self.dag1, date, State.RUNNING, middle_time)
+
+    def test_set_failed_dag_run_to_success(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.SUCCESS, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_success(self.dag1, date, commit=True)
+
+        # All except the SUCCESS task should be altered.
+        self.assertEqual(len(altered), 5)
+        self._verify_dag_run_state(self.dag1, date, State.SUCCESS)
+        self._verify_task_instance_states(self.dag1, date, State.SUCCESS)
+        self._verify_dag_run_dates(self.dag1, date, State.SUCCESS, middle_time)
+
+    def test_set_failed_dag_run_to_failed(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.SUCCESS, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        altered = set_dag_run_state_to_failed(self.dag1, date, commit=True)
+
+        # Only running task should be altered.
+        self.assertEqual(len(altered), 1)
+        self._verify_dag_run_state(self.dag1, date, State.FAILED)
+        self.assertEqual(dr.get_task_instance('run_after_loop').state, State.FAILED)
+        self._verify_dag_run_dates(self.dag1, date, State.FAILED, middle_time)
+
+    def test_set_failed_dag_run_to_running(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.SUCCESS, date)
+        middle_time = timezone.utcnow()
+        self._set_default_task_instance_states(dr)
+
+        time.sleep(2)
+
+        altered = set_dag_run_state_to_running(self.dag1, date, commit=True)
+
+        # None of the tasks should be altered, since we've only altered the DAG itself
+        self.assertEqual(len(altered), 0)
+        self._verify_dag_run_state(self.dag1, date, State.RUNNING)
+        self._verify_task_instance_states_remain_default(dr)
+        self._verify_dag_run_dates(self.dag1, date, State.RUNNING, middle_time)
+
+    def test_set_state_without_commit(self):
+        date = self.execution_dates[0]
+        dr = self._create_test_dag_run(State.RUNNING, date)
+        self._set_default_task_instance_states(dr)
+
+        will_be_altered = set_dag_run_state_to_running(self.dag1, date, commit=False)
+
+        # None of the tasks will be altered.
+        self.assertEqual(len(will_be_altered), 0)
+        self._verify_dag_run_state(self.dag1, date, State.RUNNING)
+        self._verify_task_instance_states_remain_default(dr)
+
+        will_be_altered = set_dag_run_state_to_failed(self.dag1, date, commit=False)
+
+        # Only the running task will be altered.
+        self.assertEqual(len(will_be_altered), 1)
+        self._verify_dag_run_state(self.dag1, date, State.RUNNING)
+        self._verify_task_instance_states_remain_default(dr)
+
+        will_be_altered = set_dag_run_state_to_success(self.dag1, date, commit=False)
+
+        # All except the SUCCESS task should be altered.
+        self.assertEqual(len(will_be_altered), 5)
+        self._verify_dag_run_state(self.dag1, date, State.RUNNING)
+        self._verify_task_instance_states_remain_default(dr)
+
+    @provide_session
+    def test_set_state_with_multiple_dagruns(self, session=None):
+        self.dag2.create_dagrun(
+            run_id='manual__' + datetime.now().isoformat(),
+            state=State.FAILED,
+            execution_date=self.execution_dates[0],
+            session=session
+        )
+        self.dag2.create_dagrun(
+            run_id='manual__' + datetime.now().isoformat(),
+            state=State.FAILED,
+            execution_date=self.execution_dates[1],
+            session=session
+        )
+        self.dag2.create_dagrun(
+            run_id='manual__' + datetime.now().isoformat(),
+            state=State.RUNNING,
+            execution_date=self.execution_dates[2],
+            session=session
+        )
+
+        altered = set_dag_run_state_to_success(self.dag2, self.execution_dates[1], commit=True)
+
+        # Recursively count number of tasks in the dag
+        def count_dag_tasks(dag):
+            count = len(dag.tasks)
+            subdag_counts = [count_dag_tasks(subdag) for subdag in dag.subdags]
+            count += sum(subdag_counts)
+            return count
+
+        self.assertEqual(len(altered), count_dag_tasks(self.dag2))
+        self._verify_dag_run_state(self.dag2, self.execution_dates[1], State.SUCCESS)
+
+        # Make sure other dag status are not changed
+        models.DagRun.find(dag_id=self.dag2.dag_id,
+                           execution_date=self.execution_dates[0])
+        self._verify_dag_run_state(self.dag2, self.execution_dates[0], State.FAILED)
+        models.DagRun.find(dag_id=self.dag2.dag_id,
+                           execution_date=self.execution_dates[2])
+        self._verify_dag_run_state(self.dag2, self.execution_dates[2], State.RUNNING)
+
+    def test_set_dag_run_state_edge_cases(self):
+        # Dag does not exist
+        altered = set_dag_run_state_to_success(None, self.execution_dates[0])
+        self.assertEqual(len(altered), 0)
+        altered = set_dag_run_state_to_failed(None, self.execution_dates[0])
+        self.assertEqual(len(altered), 0)
+        altered = set_dag_run_state_to_running(None, self.execution_dates[0])
+        self.assertEqual(len(altered), 0)
+
+        # Invalid execution date
+        altered = set_dag_run_state_to_success(self.dag1, None)
+        self.assertEqual(len(altered), 0)
+        altered = set_dag_run_state_to_failed(self.dag1, None)
+        self.assertEqual(len(altered), 0)
+        altered = set_dag_run_state_to_running(self.dag1, None)
+        self.assertEqual(len(altered), 0)
+
+        # This will throw AssertionError since dag.latest_execution_date
+        # need to be 0 does not exist.
+        self.assertRaises(AssertionError, set_dag_run_state_to_success, self.dag1,
+                          timezone.make_naive(self.execution_dates[0]))
+
+        # altered = set_dag_run_state_to_success(self.dag1, self.execution_dates[0])
+        # DagRun does not exist
+        # This will throw AssertionError since dag.latest_execution_date does not exist
+        self.assertRaises(AssertionError, set_dag_run_state_to_success,
+                          self.dag1, self.execution_dates[0])
+
+    def tearDown(self):
+        self.dag1.clear()
+        self.dag2.clear()
+
+        with create_session() as session:
+            session.query(models.DagRun).delete()
+            session.query(models.TaskInstance).delete()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/api/common/experimental/test_pool.py b/tests/api/common/experimental/test_pool.py
index e5efa2c6768d2..fac1e2e71b23d 100644
--- a/tests/api/common/experimental/test_pool.py
+++ b/tests/api/common/experimental/test_pool.py
@@ -28,7 +28,6 @@
 class TestPool(unittest.TestCase):
 
     def setUp(self):
-        super(TestPool, self).setUp()
         self.session = settings.Session()
         self.pools = []
         for i in range(2):
@@ -46,7 +45,6 @@ def tearDown(self):
         self.session.query(models.Pool).delete()
         self.session.commit()
         self.session.close()
-        super(TestPool, self).tearDown()
 
     def test_get_pool(self):
         pool = pool_api.get_pool(name=self.pools[0].pool, session=self.session)
diff --git a/tests/api/common/experimental/trigger_dag_tests.py b/tests/api/common/experimental/test_trigger_dag.py
similarity index 71%
rename from tests/api/common/experimental/trigger_dag_tests.py
rename to tests/api/common/experimental/test_trigger_dag.py
index d6354840e2da0..ec9899936569d 100644
--- a/tests/api/common/experimental/trigger_dag_tests.py
+++ b/tests/api/common/experimental/test_trigger_dag.py
@@ -19,6 +19,7 @@
 
 import mock
 import unittest
+import json
 
 from airflow.exceptions import AirflowException
 from airflow.models import DAG, DagRun
@@ -88,6 +89,44 @@ def test_trigger_dag_include_subdags(self, dag_bag_mock, dag_run_mock, dag_mock)
 
         self.assertEqual(3, len(triggers))
 
+    @mock.patch('airflow.models.DagBag')
+    def test_trigger_dag_with_str_conf(self, dag_bag_mock):
+        dag_id = "trigger_dag_with_str_conf"
+        dag = DAG(dag_id)
+        dag_bag_mock.dags = [dag_id]
+        dag_bag_mock.get_dag.return_value = dag
+        conf = "{\"foo\": \"bar\"}"
+        dag_run = DagRun()
+        triggers = _trigger_dag(
+            dag_id,
+            dag_bag_mock,
+            dag_run,
+            run_id=None,
+            conf=conf,
+            execution_date=None,
+            replace_microseconds=True)
+
+        self.assertEqual(triggers[0].conf, json.loads(conf))
+
+    @mock.patch('airflow.models.DagBag')
+    def test_trigger_dag_with_dict_conf(self, dag_bag_mock):
+        dag_id = "trigger_dag_with_dict_conf"
+        dag = DAG(dag_id)
+        dag_bag_mock.dags = [dag_id]
+        dag_bag_mock.get_dag.return_value = dag
+        conf = dict(foo="bar")
+        dag_run = DagRun()
+        triggers = _trigger_dag(
+            dag_id,
+            dag_bag_mock,
+            dag_run,
+            run_id=None,
+            conf=conf,
+            execution_date=None,
+            replace_microseconds=True)
+
+        self.assertEqual(triggers[0].conf, conf)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/cli/__init__.py
+++ b/tests/cli/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 34c82bcf9b683..55e0de349dcce 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -18,13 +18,19 @@
 # under the License.
 #
 
+from six import StringIO
+import sys
 import unittest
 
+from datetime import datetime, timedelta, time
 from mock import patch, Mock, MagicMock
 from time import sleep
 import psutil
+import pytz
+import subprocess
 from argparse import Namespace
 from airflow import settings
+import airflow.bin.cli as cli
 from airflow.bin.cli import get_num_ready_workers_running, run, get_dag
 from airflow.models import TaskInstance
 from airflow.utils import timezone
@@ -67,12 +73,14 @@ def create_mock_args(
     ignore_dependencies=False,
     force=False,
     run_as_user=None,
-    executor_config={},
+    executor_config=None,
     cfg_path=None,
     pickle=None,
     raw=None,
     interactive=None,
 ):
+    if executor_config is None:
+        executor_config = {}
     args = MagicMock(spec=Namespace)
     args.task_id = task_id
     args.dag_id = dag_id
@@ -134,7 +142,21 @@ def test_ready_prefix_on_cmdline_dead_process(self):
             self.assertEqual(get_num_ready_workers_running(self.gunicorn_master_proc), 0)
 
     def test_cli_webserver_debug(self):
-        p = psutil.Popen(["airflow", "webserver", "-d"])
+        env = os.environ.copy()
+        p = psutil.Popen(["airflow", "webserver", "-d"], env=env)
+        sleep(3)  # wait for webserver to start
+        return_code = p.poll()
+        self.assertEqual(
+            None,
+            return_code,
+            "webserver terminated with return code {} in debug mode".format(return_code))
+        p.terminate()
+        p.wait()
+
+    def test_cli_rbac_webserver_debug(self):
+        env = os.environ.copy()
+        env['AIRFLOW__WEBSERVER__RBAC'] = 'True'
+        p = psutil.Popen(["airflow", "webserver", "-d"], env=env)
         sleep(3)  # wait for webserver to start
         return_code = p.poll()
         self.assertEqual(
@@ -163,3 +185,101 @@ def test_local_run(self):
             ti.refresh_from_db()
             state = ti.current_state()
             self.assertEqual(state, State.SUCCESS)
+
+    def test_test(self):
+        """Test the `airflow test` command"""
+        args = create_mock_args(
+            task_id='print_the_context',
+            dag_id='example_python_operator',
+            subdir=None,
+            execution_date=timezone.parse('2018-01-01')
+        )
+
+        saved_stdout = sys.stdout
+        try:
+            sys.stdout = out = StringIO()
+            cli.test(args)
+
+            output = out.getvalue()
+            # Check that prints, and log messages, are shown
+            self.assertIn('END_DATE', output)
+            self.assertIn("'example_python_operator__print_the_context__20180101'", output)
+        finally:
+            sys.stdout = saved_stdout
+
+    def test_next_execution(self):
+        # A scaffolding function
+        def reset_dr_db(dag_id):
+            session = Session()
+            dr = session.query(models.DagRun).filter_by(dag_id=dag_id)
+            dr.delete()
+            session.commit()
+            session.close()
+
+        EXAMPLE_DAGS_FOLDER = os.path.join(
+            os.path.dirname(
+                os.path.dirname(
+                    os.path.dirname(os.path.realpath(__file__))
+                )
+            ),
+            "airflow/example_dags"
+        )
+
+        dagbag = models.DagBag(dag_folder=EXAMPLE_DAGS_FOLDER,
+                               include_examples=False)
+        dag_ids = ['example_bash_operator',  # schedule_interval is '0 0 * * *'
+                   'latest_only',  # schedule_interval is timedelta(hours=4)
+                   'example_python_operator',  # schedule_interval=None
+                   'example_xcom']  # schedule_interval="@once"
+
+        # The details below is determined by the schedule_interval of example DAGs
+        now = timezone.utcnow()
+        next_execution_time_for_dag1 = pytz.utc.localize(
+            datetime.combine(
+                now.date() + timedelta(days=1),
+                time(0)
+            )
+        )
+        next_execution_time_for_dag2 = now + timedelta(hours=4)
+        expected_output = [str(next_execution_time_for_dag1),
+                           str(next_execution_time_for_dag2),
+                           "None",
+                           "None"]
+
+        for i in range(len(dag_ids)):
+            dag_id = dag_ids[i]
+
+            # Clear dag run so no execution history fo each DAG
+            reset_dr_db(dag_id)
+
+            p = subprocess.Popen(["airflow", "next_execution", dag_id,
+                                  "--subdir", EXAMPLE_DAGS_FOLDER],
+                                 stdout=subprocess.PIPE)
+            p.wait()
+            stdout = []
+            for line in p.stdout:
+                stdout.append(str(line.decode("utf-8").rstrip()))
+
+            # `next_execution` function is inapplicable if no execution record found
+            # It prints `None` in such cases
+            self.assertEqual(stdout[-1], "None")
+
+            dag = dagbag.dags[dag_id]
+            # Create a DagRun for each DAG, to prepare for next step
+            dag.create_dagrun(
+                run_id='manual__' + now.isoformat(),
+                execution_date=now,
+                start_date=now,
+                state=State.FAILED
+            )
+
+            p = subprocess.Popen(["airflow", "next_execution", dag_id,
+                                  "--subdir", EXAMPLE_DAGS_FOLDER],
+                                 stdout=subprocess.PIPE)
+            p.wait()
+            stdout = []
+            for line in p.stdout:
+                stdout.append(str(line.decode("utf-8").rstrip()))
+            self.assertEqual(stdout[-1], expected_output[i])
+
+            reset_dr_db(dag_id)
diff --git a/tests/cli/test_worker_initialisation.py b/tests/cli/test_worker_initialisation.py
new file mode 100644
index 0000000000000..664257d0de4d4
--- /dev/null
+++ b/tests/cli/test_worker_initialisation.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+import sqlalchemy
+import airflow
+from argparse import Namespace
+from tests.compat import mock, patch
+
+patch('airflow.utils.cli.action_logging', lambda x: x).start()
+from airflow.bin import cli # noqa
+mock_args = Namespace(queues=1, concurrency=1)
+
+
+class TestWorkerPrecheck(unittest.TestCase):
+
+    def setUp(self):
+        airflow.configuration.load_test_config()
+
+    @mock.patch('airflow.settings.validate_session')
+    def test_error(self, mock_validate_session):
+        """
+        Test to verify the exit mechanism of airflow-worker cli
+        by mocking validate_session method
+        """
+        mock_validate_session.return_value = False
+        with self.assertRaises(SystemExit) as cm:
+            # airflow.bin.cli.worker(mock_args)
+            cli.worker(mock_args)
+        self.assertEqual(cm.exception.code, 1)
+
+    @mock.patch('airflow.configuration.conf.getboolean')
+    def test_worker_precheck_exception(self, mock_getboolean):
+        """
+        Test to check the behaviour of validate_session method
+        when worker_precheck is absent in airflow configuration
+        """
+        mock_getboolean.return_value = False
+
+        self.assertTrue(airflow.settings.validate_session())
+        mock_getboolean.assert_called_once_with('core', 'worker_precheck', fallback=False)
+
+    @mock.patch('sqlalchemy.orm.session.Session.execute')
+    @mock.patch('airflow.configuration.conf.getboolean')
+    def test_validate_session_dbapi_exception(self, mock_getboolean, mock_session):
+        """
+        Test to validate connection failure scenario on SELECT 1 query
+        """
+        mock_getboolean.return_value = True
+        mock_session.side_effect = sqlalchemy.exc.OperationalError("m1", "m2", "m3", "m4")
+        self.assertEqual(airflow.settings.validate_session(), False)
diff --git a/tests/compat.py b/tests/compat.py
new file mode 100644
index 0000000000000..8269a0f92aa93
--- /dev/null
+++ b/tests/compat.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+try:
+    from unittest import mock  # noqa: F401
+except ImportError:
+    import mock  # type: ignore  # noqa: F401
+
+patch = mock.patch
+Mock = mock.Mock
+MagicMock = mock.MagicMock
+PropertyMock = mock.PropertyMock
diff --git a/tests/configuration.py b/tests/configuration.py
deleted file mode 100644
index ca1d1d7a37df6..0000000000000
--- a/tests/configuration.py
+++ /dev/null
@@ -1,156 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import unittest
-from collections import OrderedDict
-
-import six
-
-from airflow import configuration
-from airflow.configuration import conf, AirflowConfigParser, parameterized_config
-
-
-class ConfTest(unittest.TestCase):
-
-    def setup(self):
-        configuration.load_test_config()
-
-    def test_env_var_config(self):
-        opt = conf.get('testsection', 'testkey')
-        self.assertEqual(opt, 'testvalue')
-
-    def test_conf_as_dict(self):
-        cfg_dict = conf.as_dict()
-
-        # test that configs are picked up
-        self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
-
-        # test env vars
-        self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
-
-        # test display_source
-        cfg_dict = conf.as_dict(display_source=True)
-        self.assertEqual(
-            cfg_dict['core']['load_examples'][1], 'airflow config')
-        self.assertEqual(
-            cfg_dict['testsection']['testkey'], ('< hidden >', 'env var'))
-
-        # test display_sensitive
-        cfg_dict = conf.as_dict(display_sensitive=True)
-        self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
-
-        # test display_source and display_sensitive
-        cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
-        self.assertEqual(
-            cfg_dict['testsection']['testkey'], ('testvalue', 'env var'))
-
-    def test_command_config(self):
-        TEST_CONFIG = '''[test]
-key1 = hello
-key2_cmd = printf cmd_result
-key3 = airflow
-key4_cmd = printf key4_result
-'''
-        TEST_CONFIG_DEFAULT = '''[test]
-key1 = awesome
-key2 = airflow
-
-[another]
-key6 = value6
-'''
-
-        test_conf = AirflowConfigParser(
-            default_config=parameterized_config(TEST_CONFIG_DEFAULT))
-        test_conf.read_string(TEST_CONFIG)
-        test_conf.as_command_stdout = test_conf.as_command_stdout | {
-            ('test', 'key2'),
-            ('test', 'key4'),
-        }
-        self.assertEqual('hello', test_conf.get('test', 'key1'))
-        self.assertEqual('cmd_result', test_conf.get('test', 'key2'))
-        self.assertEqual('airflow', test_conf.get('test', 'key3'))
-        self.assertEqual('key4_result', test_conf.get('test', 'key4'))
-        self.assertEqual('value6', test_conf.get('another', 'key6'))
-
-        self.assertTrue(test_conf.has_option('test', 'key1'))
-        self.assertTrue(test_conf.has_option('test', 'key2'))
-        self.assertTrue(test_conf.has_option('test', 'key3'))
-        self.assertTrue(test_conf.has_option('test', 'key4'))
-        self.assertFalse(test_conf.has_option('test', 'key5'))
-        self.assertTrue(test_conf.has_option('another', 'key6'))
-
-    def test_remove_option(self):
-        TEST_CONFIG = '''[test]
-key1 = hello
-key2 = airflow
-'''
-        TEST_CONFIG_DEFAULT = '''[test]
-key1 = awesome
-key2 = airflow
-'''
-
-        test_conf = AirflowConfigParser(
-            default_config=parameterized_config(TEST_CONFIG_DEFAULT))
-        test_conf.read_string(TEST_CONFIG)
-
-        self.assertEqual('hello', test_conf.get('test', 'key1'))
-        test_conf.remove_option('test', 'key1', remove_default=False)
-        self.assertEqual('awesome', test_conf.get('test', 'key1'))
-
-        test_conf.remove_option('test', 'key2')
-        self.assertFalse(test_conf.has_option('test', 'key2'))
-
-    def test_getsection(self):
-        TEST_CONFIG = '''
-[test]
-key1 = hello
-'''
-        TEST_CONFIG_DEFAULT = '''
-[test]
-key1 = awesome
-key2 = airflow
-
-[another]
-key3 = value3
-'''
-        test_conf = AirflowConfigParser(
-            default_config=parameterized_config(TEST_CONFIG_DEFAULT))
-        test_conf.read_string(TEST_CONFIG)
-
-        self.assertEqual(
-            OrderedDict([('key1', 'hello'), ('key2', 'airflow')]),
-            test_conf.getsection('test')
-        )
-        self.assertEqual(
-            OrderedDict([('key3', 'value3')]),
-            test_conf.getsection('another')
-        )
-
-    def test_broker_transport_options(self):
-        section_dict = conf.getsection("celery_broker_transport_options")
-        self.assertTrue(isinstance(section_dict['visibility_timeout'], int))
-
-        self.assertTrue(isinstance(section_dict['_test_only_bool'], bool))
-
-        self.assertTrue(isinstance(section_dict['_test_only_float'], float))
-
-        self.assertTrue(isinstance(section_dict['_test_only_string'], six.string_types))
diff --git a/tests/contrib/__init__.py b/tests/contrib/__init__.py
index 50b2e1d1785dd..114d189da14ab 100644
--- a/tests/contrib/__init__.py
+++ b/tests/contrib/__init__.py
@@ -7,16 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from __future__ import absolute_import
-from .operators import *
-from .sensors import *
diff --git a/tests/contrib/executors/test_kubernetes_executor.py b/tests/contrib/executors/test_kubernetes_executor.py
index 0a389204b4d6c..9c12d73a48d55 100644
--- a/tests/contrib/executors/test_kubernetes_executor.py
+++ b/tests/contrib/executors/test_kubernetes_executor.py
@@ -1,33 +1,53 @@
-# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 
 import unittest
+import uuid
 import re
 import string
 import random
+from urllib3 import HTTPResponse
 from datetime import datetime
 
+from tests.compat import mock
 try:
+    from kubernetes.client.rest import ApiException
+    from airflow import configuration
+    from airflow.configuration import conf
     from airflow.contrib.executors.kubernetes_executor import AirflowKubernetesScheduler
+    from airflow.contrib.executors.kubernetes_executor import KubernetesExecutor
+    from airflow.contrib.executors.kubernetes_executor import KubeConfig
+    from airflow.contrib.executors.kubernetes_executor import KubernetesExecutorConfig
+    from airflow.contrib.kubernetes.worker_configuration import WorkerConfiguration
+    from airflow.exceptions import AirflowConfigException
+    from airflow.contrib.kubernetes.secret import Secret
 except ImportError:
-    AirflowKubernetesScheduler = None
+    AirflowKubernetesScheduler = None  # type: ignore
 
 
 class TestAirflowKubernetesScheduler(unittest.TestCase):
-    def _gen_random_string(self, str_len):
-        return ''.join([random.choice(string.printable) for _ in range(str_len)])
+    @staticmethod
+    def _gen_random_string(seed, str_len):
+        char_list = []
+        for char_seed in range(str_len):
+            random.seed(str(seed) * char_seed)
+            char_list.append(random.choice(string.printable))
+        return ''.join(char_list)
 
     def _cases(self):
         cases = [
@@ -39,25 +59,50 @@ def _cases(self):
         ]
 
         cases.extend([
-            (self._gen_random_string(200), self._gen_random_string(200))
-            for _ in range(100)
+            (self._gen_random_string(seed, 200), self._gen_random_string(seed, 200))
+            for seed in range(100)
         ])
 
         return cases
 
-    def _is_valid_name(self, name):
-        regex = "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"
+    @staticmethod
+    def _is_valid_pod_id(name):
+        regex = r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"
         return (
             len(name) <= 253 and
             all(ch.lower() == ch for ch in name) and
             re.match(regex, name))
 
+    @staticmethod
+    def _is_safe_label_value(value):
+        regex = r'^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$'
+        return (
+            len(value) <= 63 and
+            re.match(regex, value))
+
     @unittest.skipIf(AirflowKubernetesScheduler is None,
                      'kubernetes python package is not installed')
     def test_create_pod_id(self):
         for dag_id, task_id in self._cases():
             pod_name = AirflowKubernetesScheduler._create_pod_id(dag_id, task_id)
-            self.assertTrue(self._is_valid_name(pod_name))
+            self.assertTrue(self._is_valid_pod_id(pod_name))
+
+    def test_make_safe_label_value(self):
+        for dag_id, task_id in self._cases():
+            safe_dag_id = AirflowKubernetesScheduler._make_safe_label_value(dag_id)
+            self.assertTrue(self._is_safe_label_value(safe_dag_id))
+            safe_task_id = AirflowKubernetesScheduler._make_safe_label_value(task_id)
+            self.assertTrue(self._is_safe_label_value(safe_task_id))
+            id = "my_dag_id"
+            self.assertEqual(
+                id,
+                AirflowKubernetesScheduler._make_safe_label_value(id)
+            )
+            id = "my_dag_id_" + "a" * 64
+            self.assertEqual(
+                "my_dag_id_" + "a" * 43 + "-0ce114c45",
+                AirflowKubernetesScheduler._make_safe_label_value(id)
+            )
 
     @unittest.skipIf(AirflowKubernetesScheduler is None,
                      "kubernetes python package is not installed")
@@ -69,7 +114,575 @@ def test_execution_date_serialize_deserialize(self):
         new_datetime_obj = AirflowKubernetesScheduler._label_safe_datestring_to_datetime(
             serialized_datetime)
 
-        self.assertEquals(datetime_obj, new_datetime_obj)
+        self.assertEqual(datetime_obj, new_datetime_obj)
+
+
+class TestKubernetesWorkerConfiguration(unittest.TestCase):
+    """
+    Tests that if dags_volume_subpath/logs_volume_subpath configuration
+    options are passed to worker pod config
+    """
+
+    affinity_config = {
+        'podAntiAffinity': {
+            'requiredDuringSchedulingIgnoredDuringExecution': [
+                {
+                    'topologyKey': 'kubernetes.io/hostname',
+                    'labelSelector': {
+                        'matchExpressions': [
+                            {
+                                'key': 'app',
+                                'operator': 'In',
+                                'values': ['airflow']
+                            }
+                        ]
+                    }
+                }
+            ]
+        }
+    }
+
+    tolerations_config = [
+        {
+            'key': 'dedicated',
+            'operator': 'Equal',
+            'value': 'airflow'
+        },
+        {
+            'key': 'prod',
+            'operator': 'Exists'
+        }
+    ]
+
+    def setUp(self):
+        if AirflowKubernetesScheduler is None:
+            self.skipTest("kubernetes python package is not installed")
+
+        self.resources = mock.patch(
+            'airflow.contrib.kubernetes.worker_configuration.Resources'
+        )
+
+        for patcher in [self.resources]:
+            self.mock_foo = patcher.start()
+            self.addCleanup(patcher.stop)
+
+        self.kube_config = mock.MagicMock()
+        self.kube_config.airflow_home = '/'
+        self.kube_config.airflow_dags = 'dags'
+        self.kube_config.airflow_dags = 'logs'
+        self.kube_config.dags_volume_subpath = None
+        self.kube_config.logs_volume_subpath = None
+        self.kube_config.dags_in_image = False
+        self.kube_config.dags_folder = None
+        self.kube_config.git_dags_folder_mount_point = None
+
+    def test_worker_configuration_no_subpaths(self):
+        worker_config = WorkerConfiguration(self.kube_config)
+        volumes, volume_mounts = worker_config._get_volumes_and_mounts()
+        volumes_list = [value for value in volumes.values()]
+        volume_mounts_list = [value for value in volume_mounts.values()]
+        for volume_or_mount in volumes_list + volume_mounts_list:
+            if volume_or_mount['name'] != 'airflow-config':
+                self.assertNotIn(
+                    'subPath', volume_or_mount,
+                    "subPath shouldn't be defined"
+                )
+
+    @mock.patch.object(conf, 'get')
+    @mock.patch.object(configuration, 'as_dict')
+    def test_worker_configuration_auth_both_ssh_and_user(self, mock_config_as_dict, mock_conf_get):
+        def get_conf(*args, **kwargs):
+            if(args[0] == 'core'):
+                return '1'
+            if(args[0] == 'kubernetes'):
+                if(args[1] == 'git_ssh_known_hosts_configmap_name'):
+                    return 'airflow-configmap'
+                if(args[1] == 'git_ssh_key_secret_name'):
+                    return 'airflow-secrets'
+                if(args[1] == 'git_user'):
+                    return 'some-user'
+                if(args[1] == 'git_password'):
+                    return 'some-password'
+                if(args[1] == 'git_repo'):
+                    return 'git@github.com:apache/airflow.git'
+                if(args[1] == 'git_branch'):
+                    return 'master'
+                if(args[1] == 'git_dags_folder_mount_point'):
+                    return '/usr/local/airflow/dags'
+                if(args[1] == 'delete_worker_pods'):
+                    return True
+                return '1'
+            return None
+
+        mock_conf_get.side_effect = get_conf
+        mock_config_as_dict.return_value = {'core': ''}
+
+        with self.assertRaisesRegexp(AirflowConfigException,
+                                     'either `git_user` and `git_password`.*'
+                                     'or `git_ssh_key_secret_name`.*'
+                                     'but not both$'):
+            KubeConfig()
+
+    def test_worker_with_subpaths(self):
+        self.kube_config.dags_volume_subpath = 'dags'
+        self.kube_config.logs_volume_subpath = 'logs'
+        worker_config = WorkerConfiguration(self.kube_config)
+        volumes, volume_mounts = worker_config._get_volumes_and_mounts()
+
+        for volume in [value for value in volumes.values()]:
+            self.assertNotIn(
+                'subPath', volume,
+                "subPath isn't valid configuration for a volume"
+            )
+
+        for volume_mount in [value for value in volume_mounts.values()]:
+            if volume_mount['name'] != 'airflow-config':
+                self.assertIn(
+                    'subPath', volume_mount,
+                    "subPath should've been passed to volumeMount configuration"
+                )
+
+    def test_worker_generate_dag_volume_mount_path(self):
+        self.kube_config.git_dags_folder_mount_point = '/root/airflow/git/dags'
+        self.kube_config.dags_folder = '/root/airflow/dags'
+        worker_config = WorkerConfiguration(self.kube_config)
+
+        self.kube_config.dags_volume_claim = 'airflow-dags'
+        self.kube_config.dags_volume_host = ''
+        dag_volume_mount_path = worker_config.generate_dag_volume_mount_path()
+        self.assertEqual(dag_volume_mount_path, self.kube_config.dags_folder)
+
+        self.kube_config.dags_volume_claim = ''
+        self.kube_config.dags_volume_host = '/host/airflow/dags'
+        dag_volume_mount_path = worker_config.generate_dag_volume_mount_path()
+        self.assertEqual(dag_volume_mount_path, self.kube_config.dags_folder)
+
+        self.kube_config.dags_volume_claim = ''
+        self.kube_config.dags_volume_host = ''
+        dag_volume_mount_path = worker_config.generate_dag_volume_mount_path()
+        self.assertEqual(dag_volume_mount_path,
+                         self.kube_config.git_dags_folder_mount_point)
+
+    def test_worker_environment_no_dags_folder(self):
+        self.kube_config.airflow_configmap = ''
+        self.kube_config.git_dags_folder_mount_point = ''
+        self.kube_config.dags_folder = ''
+        worker_config = WorkerConfiguration(self.kube_config)
+        env = worker_config._get_environment()
+
+        self.assertNotIn('AIRFLOW__CORE__DAGS_FOLDER', env)
+
+    def test_worker_environment_when_dags_folder_specified(self):
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_dags_folder_mount_point = ''
+        dags_folder = '/workers/path/to/dags'
+        self.kube_config.dags_folder = dags_folder
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        env = worker_config._get_environment()
+
+        self.assertEqual(dags_folder, env['AIRFLOW__CORE__DAGS_FOLDER'])
+
+    def test_worker_environment_dags_folder_using_git_sync(self):
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_sync_dest = 'repo'
+        self.kube_config.git_subpath = 'dags'
+        self.kube_config.git_dags_folder_mount_point = '/workers/path/to/dags'
+
+        dags_folder = '{}/{}/{}'.format(self.kube_config.git_dags_folder_mount_point,
+                                        self.kube_config.git_sync_dest,
+                                        self.kube_config.git_subpath)
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        env = worker_config._get_environment()
+
+        self.assertEqual(dags_folder, env['AIRFLOW__CORE__DAGS_FOLDER'])
+
+    def test_init_environment_using_git_sync_ssh_without_known_hosts(self):
+        # Tests the init environment created with git-sync SSH authentication option is correct
+        # without known hosts file
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_ssh_secret_name = 'airflow-secrets'
+        self.kube_config.git_ssh_known_hosts_configmap_name = None
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        init_containers = worker_config._get_init_containers()
+
+        self.assertTrue(init_containers)  # check not empty
+        env = init_containers[0]['env']
+
+        self.assertTrue({'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh'} in env)
+        self.assertTrue({'name': 'GIT_KNOWN_HOSTS', 'value': 'false'} in env)
+        self.assertTrue({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
+
+    def test_init_environment_using_git_sync_ssh_with_known_hosts(self):
+        # Tests the init environment created with git-sync SSH authentication option is correct
+        # with known hosts file
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_ssh_key_secret_name = 'airflow-secrets'
+        self.kube_config.git_ssh_known_hosts_configmap_name = 'airflow-configmap'
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        init_containers = worker_config._get_init_containers()
+
+        self.assertTrue(init_containers)  # check not empty
+        env = init_containers[0]['env']
+
+        self.assertTrue({'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh'} in env)
+        self.assertTrue({'name': 'GIT_KNOWN_HOSTS', 'value': 'true'} in env)
+        self.assertTrue({'name': 'GIT_SSH_KNOWN_HOSTS_FILE',
+                        'value': '/etc/git-secret/known_hosts'} in env)
+        self.assertTrue({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
+
+    def test_init_environment_using_git_sync_user_without_known_hosts(self):
+        # Tests the init environment created with git-sync User authentication option is correct
+        # without known hosts file
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_user = 'git_user'
+        self.kube_config.git_password = 'git_password'
+        self.kube_config.git_ssh_known_hosts_configmap_name = None
+        self.kube_config.git_ssh_key_secret_name = None
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        init_containers = worker_config._get_init_containers()
+
+        self.assertTrue(init_containers)  # check not empty
+        env = init_containers[0]['env']
+
+        self.assertFalse({'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh'} in env)
+        self.assertTrue({'name': 'GIT_SYNC_USERNAME', 'value': 'git_user'} in env)
+        self.assertTrue({'name': 'GIT_SYNC_PASSWORD', 'value': 'git_password'} in env)
+        self.assertTrue({'name': 'GIT_KNOWN_HOSTS', 'value': 'false'} in env)
+        self.assertFalse({'name': 'GIT_SSH_KNOWN_HOSTS_FILE',
+                          'value': '/etc/git-secret/known_hosts'} in env)
+        self.assertFalse({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
+
+    def test_init_environment_using_git_sync_user_with_known_hosts(self):
+        # Tests the init environment created with git-sync User authentication option is correct
+        # with known hosts file
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_user = 'git_user'
+        self.kube_config.git_password = 'git_password'
+        self.kube_config.git_ssh_known_hosts_configmap_name = 'airflow-configmap'
+        self.kube_config.git_ssh_key_secret_name = None
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        init_containers = worker_config._get_init_containers()
+
+        self.assertTrue(init_containers)  # check not empty
+        env = init_containers[0]['env']
+
+        self.assertFalse({'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh'} in env)
+        self.assertTrue({'name': 'GIT_SYNC_USERNAME', 'value': 'git_user'} in env)
+        self.assertTrue({'name': 'GIT_SYNC_PASSWORD', 'value': 'git_password'} in env)
+        self.assertTrue({'name': 'GIT_KNOWN_HOSTS', 'value': 'true'} in env)
+        self.assertTrue({'name': 'GIT_SSH_KNOWN_HOSTS_FILE',
+                        'value': '/etc/git-secret/known_hosts'} in env)
+        self.assertFalse({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
+
+    def test_make_pod_git_sync_ssh_without_known_hosts(self):
+        # Tests the pod created with git-sync SSH authentication option is correct without known hosts
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_ssh_key_secret_name = 'airflow-secrets'
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+        self.kube_config.worker_fs_group = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        kube_executor_config = KubernetesExecutorConfig(annotations=[],
+                                                        volumes=[],
+                                                        volume_mounts=[])
+
+        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
+                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
+                                     kube_executor_config)
+
+        init_containers = worker_config._get_init_containers()
+        git_ssh_key_file = next((x['value'] for x in init_containers[0]['env']
+                                if x['name'] == 'GIT_SSH_KEY_FILE'), None)
+        volume_mount_ssh_key = next((x['mountPath'] for x in init_containers[0]['volumeMounts']
+                                    if x['name'] == worker_config.git_sync_ssh_secret_volume_name),
+                                    None)
+        self.assertTrue(git_ssh_key_file)
+        self.assertTrue(volume_mount_ssh_key)
+        self.assertEqual(65533, pod.security_context['fsGroup'])
+        self.assertEqual(git_ssh_key_file,
+                         volume_mount_ssh_key,
+                         'The location where the git ssh secret is mounted'
+                         ' needs to be the same as the GIT_SSH_KEY_FILE path')
+
+    def test_make_pod_git_sync_ssh_with_known_hosts(self):
+        # Tests the pod created with git-sync SSH authentication option is correct with known hosts
+        self.kube_config.airflow_configmap = 'airflow-configmap'
+        self.kube_config.git_ssh_secret_name = 'airflow-secrets'
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+
+        init_containers = worker_config._get_init_containers()
+        git_ssh_known_hosts_file = next((x['value'] for x in init_containers[0]['env']
+                                         if x['name'] == 'GIT_SSH_KNOWN_HOSTS_FILE'), None)
+
+        volume_mount_ssh_known_hosts_file = next(
+            (x['mountPath'] for x in init_containers[0]['volumeMounts']
+             if x['name'] == worker_config.git_sync_ssh_known_hosts_volume_name),
+            None)
+        self.assertTrue(git_ssh_known_hosts_file)
+        self.assertTrue(volume_mount_ssh_known_hosts_file)
+        self.assertEqual(git_ssh_known_hosts_file,
+                         volume_mount_ssh_known_hosts_file,
+                         'The location where the git known hosts file is mounted'
+                         ' needs to be the same as the GIT_SSH_KNOWN_HOSTS_FILE path')
+
+    def test_make_pod_with_empty_executor_config(self):
+        self.kube_config.kube_affinity = self.affinity_config
+        self.kube_config.kube_tolerations = self.tolerations_config
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        kube_executor_config = KubernetesExecutorConfig(annotations=[],
+                                                        volumes=[],
+                                                        volume_mounts=[]
+                                                        )
+
+        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
+                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
+                                     kube_executor_config)
+
+        self.assertTrue(pod.affinity['podAntiAffinity'] is not None)
+        self.assertEqual('app',
+                         pod.affinity['podAntiAffinity']
+                         ['requiredDuringSchedulingIgnoredDuringExecution'][0]
+                         ['labelSelector']
+                         ['matchExpressions'][0]
+                         ['key'])
+
+        self.assertEqual(2, len(pod.tolerations))
+        self.assertEqual('prod', pod.tolerations[1]['key'])
+
+    def test_make_pod_with_executor_config(self):
+        worker_config = WorkerConfiguration(self.kube_config)
+        kube_executor_config = KubernetesExecutorConfig(affinity=self.affinity_config,
+                                                        tolerations=self.tolerations_config,
+                                                        annotations=[],
+                                                        volumes=[],
+                                                        volume_mounts=[]
+                                                        )
+
+        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
+                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
+                                     kube_executor_config)
+
+        self.assertTrue(pod.affinity['podAntiAffinity'] is not None)
+        self.assertEqual('app',
+                         pod.affinity['podAntiAffinity']
+                         ['requiredDuringSchedulingIgnoredDuringExecution'][0]
+                         ['labelSelector']
+                         ['matchExpressions'][0]
+                         ['key'])
+
+        self.assertEqual(2, len(pod.tolerations))
+        self.assertEqual('prod', pod.tolerations[1]['key'])
+
+    def test_worker_pvc_dags(self):
+        # Tests persistence volume config created when `dags_volume_claim` is set
+        self.kube_config.dags_volume_claim = 'airflow-dags'
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        volumes, volume_mounts = worker_config._get_volumes_and_mounts()
+
+        init_containers = worker_config._get_init_containers()
+
+        dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags']
+        dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags']
+
+        self.assertEqual('airflow-dags', dag_volume[0]['persistentVolumeClaim']['claimName'])
+        self.assertEqual(1, len(dag_volume_mount))
+        self.assertTrue(dag_volume_mount[0]['readOnly'])
+        self.assertEqual(0, len(init_containers))
+
+    def test_worker_git_dags(self):
+        # Tests persistence volume config created when `git_repo` is set
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_folder = '/usr/local/airflow/dags'
+        self.kube_config.worker_dags_folder = '/usr/local/airflow/dags'
+
+        self.kube_config.git_sync_container_repository = 'gcr.io/google-containers/git-sync-amd64'
+        self.kube_config.git_sync_container_tag = 'v2.0.5'
+        self.kube_config.git_sync_container = 'gcr.io/google-containers/git-sync-amd64:v2.0.5'
+        self.kube_config.git_sync_init_container_name = 'git-sync-clone'
+        self.kube_config.git_subpath = 'dags_folder'
+        self.kube_config.git_sync_root = '/git'
+        self.kube_config.git_dags_folder_mount_point = '/usr/local/airflow/dags/repo/dags_folder'
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        volumes, volume_mounts = worker_config._get_volumes_and_mounts()
+
+        dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags']
+        dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags']
+
+        self.assertTrue('emptyDir' in dag_volume[0])
+        self.assertEqual(self.kube_config.git_dags_folder_mount_point, dag_volume_mount[0]['mountPath'])
+        self.assertTrue(dag_volume_mount[0]['readOnly'])
+
+        init_container = worker_config._get_init_containers()[0]
+        init_container_volume_mount = [mount for mount in init_container['volumeMounts']
+                                       if mount['name'] == 'airflow-dags']
+
+        self.assertEqual('git-sync-clone', init_container['name'])
+        self.assertEqual('gcr.io/google-containers/git-sync-amd64:v2.0.5', init_container['image'])
+        self.assertEqual(1, len(init_container_volume_mount))
+        self.assertFalse(init_container_volume_mount[0]['readOnly'])
+
+    def test_worker_container_dags(self):
+        # Tests that the 'airflow-dags' persistence volume is NOT created when `dags_in_image` is set
+        self.kube_config.dags_in_image = True
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        volumes, volume_mounts = worker_config._get_volumes_and_mounts()
+
+        dag_volume = [volume for volume in volumes.values() if volume['name'] == 'airflow-dags']
+        dag_volume_mount = [mount for mount in volume_mounts.values() if mount['name'] == 'airflow-dags']
+
+        init_containers = worker_config._get_init_containers()
+
+        self.assertEqual(0, len(dag_volume))
+        self.assertEqual(0, len(dag_volume_mount))
+        self.assertEqual(0, len(init_containers))
+
+    def test_kubernetes_environment_variables(self):
+        # Tests the kubernetes environment variables get copied into the worker pods
+        input_environment = {
+            'ENVIRONMENT': 'prod',
+            'LOG_LEVEL': 'warning'
+        }
+        self.kube_config.kube_env_vars = input_environment
+        worker_config = WorkerConfiguration(self.kube_config)
+        env = worker_config._get_environment()
+        for key in input_environment:
+            self.assertIn(key, env)
+            self.assertIn(input_environment[key], env.values())
+
+        core_executor = 'AIRFLOW__CORE__EXECUTOR'
+        input_environment = {
+            core_executor: 'NotLocalExecutor'
+        }
+        self.kube_config.kube_env_vars = input_environment
+        worker_config = WorkerConfiguration(self.kube_config)
+        env = worker_config._get_environment()
+        self.assertEqual(env[core_executor], 'LocalExecutor')
+
+    def test_get_secrets(self):
+        # Test when secretRef is None and kube_secrets is not empty
+        self.kube_config.kube_secrets = {
+            'AWS_SECRET_KEY': 'airflow-secret=aws_secret_key',
+            'POSTGRES_PASSWORD': 'airflow-secret=postgres_credentials'
+        }
+        self.kube_config.env_from_secret_ref = None
+        worker_config = WorkerConfiguration(self.kube_config)
+        secrets = worker_config._get_secrets()
+        secrets.sort(key=lambda secret: secret.deploy_target)
+        expected = [
+            Secret('env', 'AWS_SECRET_KEY', 'airflow-secret', 'aws_secret_key'),
+            Secret('env', 'POSTGRES_PASSWORD', 'airflow-secret', 'postgres_credentials')
+        ]
+        self.assertListEqual(expected, secrets)
+
+        # Test when secret is not empty and kube_secrets is empty dict
+        self.kube_config.kube_secrets = {}
+        self.kube_config.env_from_secret_ref = 'secret_a,secret_b'
+        worker_config = WorkerConfiguration(self.kube_config)
+        secrets = worker_config._get_secrets()
+        expected = [
+            Secret('env', None, 'secret_a'),
+            Secret('env', None, 'secret_b')
+        ]
+        self.assertListEqual(expected, secrets)
+
+    def test_get_configmaps(self):
+        # Test when configmap is empty
+        self.kube_config.env_from_configmap_ref = ''
+        worker_config = WorkerConfiguration(self.kube_config)
+        configmaps = worker_config._get_configmaps()
+        self.assertListEqual([], configmaps)
+
+        # test when configmap is not empty
+        self.kube_config.env_from_configmap_ref = 'configmap_a,configmap_b'
+        worker_config = WorkerConfiguration(self.kube_config)
+        configmaps = worker_config._get_configmaps()
+        self.assertListEqual(['configmap_a', 'configmap_b'], configmaps)
+
+
+class TestKubernetesExecutor(unittest.TestCase):
+    """
+    Tests if an ApiException from the Kube Client will cause the task to
+    be rescheduled.
+    """
+    @unittest.skipIf(AirflowKubernetesScheduler is None,
+                     'kubernetes python package is not installed')
+    @mock.patch('airflow.contrib.executors.kubernetes_executor.KubernetesJobWatcher')
+    @mock.patch('airflow.contrib.executors.kubernetes_executor.get_kube_client')
+    def test_run_next_exception(self, mock_get_kube_client, mock_kubernetes_job_watcher):
+
+        # When a quota is exceeded this is the ApiException we get
+        r = HTTPResponse()
+        r.body = {
+            "kind": "Status",
+            "apiVersion": "v1",
+            "metadata": {},
+            "status": "Failure",
+            "message": "pods \"podname\" is forbidden: " +
+            "exceeded quota: compute-resources, " +
+            "requested: limits.memory=4Gi, " +
+            "used: limits.memory=6508Mi, " +
+            "limited: limits.memory=10Gi",
+            "reason": "Forbidden",
+            "details": {"name": "podname", "kind": "pods"},
+            "code": 403},
+        r.status = 403
+        r.reason = "Forbidden"
+
+        # A mock kube_client that throws errors when making a pod
+        mock_kube_client = mock.patch('kubernetes.client.CoreV1Api', autospec=True)
+        mock_kube_client.create_namespaced_pod = mock.MagicMock(
+            side_effect=ApiException(http_resp=r))
+        mock_get_kube_client.return_value = mock_kube_client
+
+        kubernetesExecutor = KubernetesExecutor()
+        kubernetesExecutor.start()
+
+        # Execute a task while the Api Throws errors
+        try_number = 1
+        kubernetesExecutor.execute_async(key=('dag', 'task', datetime.utcnow(), try_number),
+                                         command='command', executor_config={})
+        kubernetesExecutor.sync()
+        kubernetesExecutor.sync()
+
+        assert mock_kube_client.create_namespaced_pod.called
+        self.assertFalse(kubernetesExecutor.task_queue.empty())
+
+        # Disable the ApiException
+        mock_kube_client.create_namespaced_pod.side_effect = None
+
+        # Execute the task without errors should empty the queue
+        kubernetesExecutor.sync()
+        assert mock_kube_client.create_namespaced_pod.called
+        self.assertTrue(kubernetesExecutor.task_queue.empty())
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/executors/test_mesos_executor.py b/tests/contrib/executors/test_mesos_executor.py
index ea3faa208c11a..f3771111f719b 100644
--- a/tests/contrib/executors/test_mesos_executor.py
+++ b/tests/contrib/executors/test_mesos_executor.py
@@ -18,20 +18,14 @@
 # under the License.
 
 import unittest
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 try:
     from mesos.interface import mesos_pb2
     from airflow.contrib.executors.mesos_executor import AirflowMesosScheduler
     mock_mesos = True
 except ImportError:
-    mock_mesos = None
+    mock_mesos = None  # type: ignore
 
 from airflow import configuration
 from queue import Queue
diff --git a/tests/contrib/hooks/__init__.py b/tests/contrib/hooks/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/contrib/hooks/__init__.py
+++ b/tests/contrib/hooks/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/hooks/test_aws_dynamodb_hook.py b/tests/contrib/hooks/test_aws_dynamodb_hook.py
index eabd352190213..835c042b2dc62 100644
--- a/tests/contrib/hooks/test_aws_dynamodb_hook.py
+++ b/tests/contrib/hooks/test_aws_dynamodb_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/hooks/test_aws_firehose_hook.py b/tests/contrib/hooks/test_aws_firehose_hook.py
index f22bcde2b1e29..441242c49ef50 100644
--- a/tests/contrib/hooks/test_aws_firehose_hook.py
+++ b/tests/contrib/hooks/test_aws_firehose_hook.py
@@ -57,7 +57,7 @@ def test_insert_batch_records_kinesis_firehose(self):
         )
 
         stream_arn = response['DeliveryStreamARN']
-        self.assertEquals(
+        self.assertEqual(
             stream_arn, "arn:aws:firehose:us-east-1:123456789012:deliverystream/test_airflow")
 
         records = [{"Data": str(uuid.uuid4())}
@@ -65,8 +65,8 @@ def test_insert_batch_records_kinesis_firehose(self):
 
         response = hook.put_records(records)
 
-        self.assertEquals(response['FailedPutCount'], 0)
-        self.assertEquals(response['ResponseMetadata']['HTTPStatusCode'], 200)
+        self.assertEqual(response['FailedPutCount'], 0)
+        self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'], 200)
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/hooks/test_aws_glue_catalog_hook.py b/tests/contrib/hooks/test_aws_glue_catalog_hook.py
new file mode 100644
index 0000000000000..57ec7a1a793d3
--- /dev/null
+++ b/tests/contrib/hooks/test_aws_glue_catalog_hook.py
@@ -0,0 +1,175 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import boto3
+from airflow.contrib.hooks.aws_glue_catalog_hook import AwsGlueCatalogHook
+from tests.compat import mock
+
+try:
+    from moto import mock_glue
+except ImportError:
+    mock_glue = None
+
+DB_NAME = 'db'
+TABLE_NAME = 'table'
+TABLE_INPUT = {
+    "Name": TABLE_NAME,
+    "StorageDescriptor": {
+        "Columns": [
+            {
+                "Name": "string",
+                "Type": "string",
+                "Comment": "string"
+            }
+        ],
+        "Location": "s3://mybucket/{}/{}".format(DB_NAME, TABLE_NAME),
+    }
+}
+
+@unittest.skipIf(mock_glue is None,
+                 "Skipping test because moto.mock_glue is not available")
+class TestAwsGlueCatalogHook(unittest.TestCase):
+
+    @mock_glue
+    def setUp(self):
+        self.client = boto3.client('glue', region_name='us-east-1')
+        self.hook = AwsGlueCatalogHook(region_name="us-east-1")
+
+    @mock_glue
+    def test_get_conn_returns_a_boto3_connection(self):
+        hook = AwsGlueCatalogHook(region_name="us-east-1")
+        self.assertIsNotNone(hook.get_conn())
+
+    @mock_glue
+    def test_conn_id(self):
+        hook = AwsGlueCatalogHook(aws_conn_id='my_aws_conn_id', region_name="us-east-1")
+        self.assertEqual(hook.aws_conn_id, 'my_aws_conn_id')
+
+    @mock_glue
+    def test_region(self):
+        hook = AwsGlueCatalogHook(region_name="us-west-2")
+        self.assertEqual(hook.region_name, 'us-west-2')
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'get_conn')
+    def test_get_partitions_empty(self, mock_get_conn):
+        response = set()
+        mock_get_conn.get_paginator.paginate.return_value = response
+        hook = AwsGlueCatalogHook(region_name="us-east-1")
+
+        self.assertEqual(hook.get_partitions('db', 'tbl'), set())
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'get_conn')
+    def test_get_partitions(self, mock_get_conn):
+        response = [{
+            'Partitions': [{
+                'Values': ['2015-01-01']
+            }]
+        }]
+        mock_paginator = mock.Mock()
+        mock_paginator.paginate.return_value = response
+        mock_conn = mock.Mock()
+        mock_conn.get_paginator.return_value = mock_paginator
+        mock_get_conn.return_value = mock_conn
+        hook = AwsGlueCatalogHook(region_name="us-east-1")
+        result = hook.get_partitions('db',
+                                     'tbl',
+                                     expression='foo=bar',
+                                     page_size=2,
+                                     max_items=3)
+
+        self.assertEqual(result, set([('2015-01-01',)]))
+        mock_conn.get_paginator.assert_called_once_with('get_partitions')
+        mock_paginator.paginate.assert_called_once_with(DatabaseName='db',
+                                                        TableName='tbl',
+                                                        Expression='foo=bar',
+                                                        PaginationConfig={
+                                                            'PageSize': 2,
+                                                            'MaxItems': 3})
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'get_partitions')
+    def test_check_for_partition(self, mock_get_partitions):
+        mock_get_partitions.return_value = set([('2018-01-01',)])
+        hook = AwsGlueCatalogHook(region_name="us-east-1")
+
+        self.assertTrue(hook.check_for_partition('db', 'tbl', 'expr'))
+        mock_get_partitions.assert_called_once_with('db', 'tbl', 'expr', max_items=1)
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'get_partitions')
+    def test_check_for_partition_false(self, mock_get_partitions):
+        mock_get_partitions.return_value = set()
+        hook = AwsGlueCatalogHook(region_name="us-east-1")
+
+        self.assertFalse(hook.check_for_partition('db', 'tbl', 'expr'))
+
+    @mock_glue
+    def test_get_table_exists(self):
+        self.client.create_database(
+            DatabaseInput={
+                'Name': DB_NAME
+            }
+        )
+        self.client.create_table(
+            DatabaseName=DB_NAME,
+            TableInput=TABLE_INPUT
+        )
+
+        result = self.hook.get_table(DB_NAME, TABLE_NAME)
+
+        self.assertEqual(result['Name'], TABLE_INPUT['Name'])
+        self.assertEqual(result['StorageDescriptor']['Location'],
+                         TABLE_INPUT['StorageDescriptor']['Location'])
+
+    @mock_glue
+    def test_get_table_not_exists(self):
+        self.client.create_database(
+            DatabaseInput={
+                'Name': DB_NAME
+            }
+        )
+        self.client.create_table(
+            DatabaseName=DB_NAME,
+            TableInput=TABLE_INPUT
+        )
+
+        with self.assertRaises(Exception):
+            self.hook.get_table(DB_NAME, 'dummy_table')
+
+    @mock_glue
+    def test_get_table_location(self):
+        self.client.create_database(
+            DatabaseInput={
+                'Name': DB_NAME
+            }
+        )
+        self.client.create_table(
+            DatabaseName=DB_NAME,
+            TableInput=TABLE_INPUT
+        )
+
+        result = self.hook.get_table_location(DB_NAME, TABLE_NAME)
+        self.assertEqual(result, TABLE_INPUT['StorageDescriptor']['Location'])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_aws_hook.py b/tests/contrib/hooks/test_aws_hook.py
index 719d6d17372b0..61acec69fb446 100644
--- a/tests/contrib/hooks/test_aws_hook.py
+++ b/tests/contrib/hooks/test_aws_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,26 +19,21 @@
 #
 
 import unittest
+
 import boto3
 
 from airflow import configuration
-from airflow.models import Connection
+from airflow.models.connection import Connection
 from airflow.contrib.hooks.aws_hook import AwsHook
+from tests.compat import mock
 
 try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
-try:
-    from moto import mock_emr, mock_dynamodb2, mock_sts
+    from moto import mock_emr, mock_dynamodb2, mock_sts, mock_iam
 except ImportError:
     mock_emr = None
     mock_dynamodb2 = None
     mock_sts = None
+    mock_iam = None
 
 
 class TestAwsHook(unittest.TestCase):
@@ -146,6 +141,26 @@ def test_get_credentials_from_extra(self, mock_get_connection):
         self.assertEqual(credentials_from_hook.secret_key, 'aws_secret_access_key')
         self.assertIsNone(credentials_from_hook.token)
 
+    @mock.patch('airflow.contrib.hooks.aws_hook._parse_s3_config',
+                return_value=('aws_access_key_id', 'aws_secret_access_key'))
+    @mock.patch.object(AwsHook, 'get_connection')
+    def test_get_credentials_from_extra_with_s3_config_and_profile(
+        self, mock_get_connection, mock_parse_s3_config
+    ):
+        mock_connection = Connection(
+            extra='{"s3_config_format": "aws", '
+                  '"profile": "test", '
+                  '"s3_config_file": "aws-credentials", '
+                  '"region_name": "us-east-1"}')
+        mock_get_connection.return_value = mock_connection
+        hook = AwsHook()
+        hook._get_credentials(region_name=None)
+        mock_parse_s3_config.assert_called_with(
+            'aws-credentials',
+            'aws',
+            'test'
+        )
+
     @unittest.skipIf(mock_sts is None, 'mock_sts package not present')
     @mock.patch.object(AwsHook, 'get_connection')
     @mock_sts
@@ -164,6 +179,35 @@ def test_get_credentials_from_role_arn(self, mock_get_connection):
                          'gRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15'
                          'fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE')
 
+    @unittest.skipIf(mock_sts is None, 'mock_sts package not present')
+    @mock.patch.object(AwsHook, 'get_connection')
+    @mock_sts
+    def test_get_credentials_from_role_arn_with_external_id(self, mock_get_connection):
+        mock_connection = Connection(
+            extra='{"role_arn":"arn:aws:iam::123456:role/role_arn",'
+                  ' "external_id":"external_id"}')
+        mock_get_connection.return_value = mock_connection
+        hook = AwsHook()
+        credentials_from_hook = hook.get_credentials()
+        self.assertEqual(credentials_from_hook.access_key, 'AKIAIOSFODNN7EXAMPLE')
+        self.assertEqual(credentials_from_hook.secret_key,
+                         'aJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY')
+        self.assertEqual(credentials_from_hook.token,
+                         'BQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh'
+                         '3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4I'
+                         'gRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15'
+                         'fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE')
+
+    @unittest.skipIf(mock_iam is None, 'mock_iam package not present')
+    @mock_iam
+    def test_expand_role(self):
+        conn = boto3.client('iam', region_name='us-east-1')
+        conn.create_role(RoleName='test-role', AssumeRolePolicyDocument='some policy')
+        hook = AwsHook()
+        arn = hook.expand_role('test-role')
+        expect_arn = conn.get_role(RoleName='test-role').get('Role').get('Arn')
+        self.assertEqual(arn, expect_arn)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_aws_lambda_hook.py b/tests/contrib/hooks/test_aws_lambda_hook.py
index a35a8fcd1e269..ec97098e5119f 100644
--- a/tests/contrib/hooks/test_aws_lambda_hook.py
+++ b/tests/contrib/hooks/test_aws_lambda_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,7 +23,6 @@
 import json
 import textwrap
 import zipfile
-import base64
 
 from airflow.contrib.hooks.aws_lambda_hook import AwsLambdaHook
 
@@ -42,7 +41,8 @@ def test_get_conn_returns_a_boto3_connection(self):
                              function_name="test_function", region_name="us-east-1")
         self.assertIsNotNone(hook.get_conn())
 
-    def lambda_function(self):
+    @staticmethod
+    def lambda_function():
         code = textwrap.dedent("""
 def lambda_handler(event, context):
     return event
@@ -78,7 +78,7 @@ def test_invoke_lambda_function(self):
         input = {'hello': 'airflow'}
         response = hook.invoke_lambda(payload=json.dumps(input))
 
-        self.assertEquals(response["StatusCode"], 202)
+        self.assertEqual(response["StatusCode"], 202)
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/hooks/test_aws_sns_hook.py b/tests/contrib/hooks/test_aws_sns_hook.py
new file mode 100644
index 0000000000000..480e29d08842e
--- /dev/null
+++ b/tests/contrib/hooks/test_aws_sns_hook.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+
+from airflow.contrib.hooks.aws_sns_hook import AwsSnsHook
+
+try:
+    from moto import mock_sns
+except ImportError:
+    mock_sns = None
+
+
+@unittest.skipIf(mock_sns is None, 'moto package not present')
+class TestAwsLambdaHook(unittest.TestCase):
+
+    @mock_sns
+    def test_get_conn_returns_a_boto3_connection(self):
+        hook = AwsSnsHook(aws_conn_id='aws_default')
+        self.assertIsNotNone(hook.get_conn())
+
+    @mock_sns
+    def test_publish_to_target(self):
+        hook = AwsSnsHook(aws_conn_id='aws_default')
+
+        message = "Hello world"
+        topic_name = "test-topic"
+        target = hook.get_conn().create_topic(Name=topic_name).get('TopicArn')
+
+        response = hook.publish_to_target(target, message)
+
+        self.assertTrue('MessageId' in response)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_azure_container_instance_hook.py b/tests/contrib/hooks/test_azure_container_instance_hook.py
new file mode 100644
index 0000000000000..afaf2ff40dc4a
--- /dev/null
+++ b/tests/contrib/hooks/test_azure_container_instance_hook.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import unittest
+from collections import namedtuple
+from mock import patch
+
+from airflow import configuration
+from airflow.models.connection import Connection
+from airflow.contrib.hooks.azure_container_instance_hook import AzureContainerInstanceHook
+from airflow.utils import db
+
+from azure.mgmt.containerinstance.models import (Container,
+                                                 ContainerGroup,
+                                                 ContainerState,
+                                                 Event,
+                                                 Logs,
+                                                 ResourceRequests,
+                                                 ResourceRequirements)
+
+
+class TestAzureContainerInstanceHook(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='azure_container_instance_test',
+                conn_type='azure_container_instances',
+                login='login',
+                password='key',
+                extra=json.dumps({'tenantId': 'tenant_id',
+                                  'subscriptionId': 'subscription_id'})
+            )
+        )
+
+        self.resources = ResourceRequirements(requests=ResourceRequests(
+            memory_in_gb='4',
+            cpu='1'))
+        with patch('azure.common.credentials.ServicePrincipalCredentials.__init__',
+                   autospec=True, return_value=None):
+            with patch('azure.mgmt.containerinstance.ContainerInstanceManagementClient'):
+                self.testHook = AzureContainerInstanceHook(conn_id='azure_container_instance_test')
+
+    @patch('azure.mgmt.containerinstance.models.ContainerGroup')
+    @patch('azure.mgmt.containerinstance.operations.ContainerGroupsOperations.create_or_update')
+    def test_create_or_update(self, create_or_update_mock, container_group_mock):
+        self.testHook.create_or_update('resource_group', 'aci-test', container_group_mock)
+        create_or_update_mock.assert_called_with('resource_group', 'aci-test', container_group_mock)
+
+    @patch('airflow.contrib.hooks.azure_container_instance_hook'
+           '.AzureContainerInstanceHook._get_instance_view')
+    def test_get_state_exitcode_details(self, get_instance_view_mock):
+        expected_state = ContainerState(state='testing', exit_code=1, detail_status='details')
+        instance_view = {"current_state": expected_state}
+        named_instance = namedtuple("InstanceView", instance_view.keys())(*instance_view.values())
+        get_instance_view_mock.return_value = named_instance
+
+        state, exit_code, details = self.testHook.get_state_exitcode_details('resource-group', 'test')
+
+        self.assertEqual(state, expected_state.state)
+        self.assertEqual(exit_code, expected_state.exit_code)
+        self.assertEqual(details, expected_state.detail_status)
+
+    @patch('airflow.contrib.hooks.azure_container_instance_hook'
+           '.AzureContainerInstanceHook._get_instance_view')
+    def test_get_messages(self, get_instance_view_mock):
+        expected_messages = ['test1', 'test2']
+        events = [Event(message=m) for m in expected_messages]
+        instance_view = {"events": events}
+        named_instance = namedtuple("Events", instance_view.keys())(*instance_view.values())
+        get_instance_view_mock.return_value = named_instance
+
+        messages = self.testHook.get_messages('resource-group', 'test')
+
+        self.assertSequenceEqual(messages, expected_messages)
+
+    @patch('azure.mgmt.containerinstance.operations.ContainerOperations.list_logs')
+    def test_get_logs(self, list_logs_mock):
+        expected_messages = ['log line 1\n', 'log line 2\n', 'log line 3\n']
+        logs = Logs(content=''.join(expected_messages))
+        list_logs_mock.return_value = logs
+
+        logs = self.testHook.get_logs('resource_group', 'name', 'name')
+
+        self.assertSequenceEqual(logs, expected_messages)
+
+    @patch('azure.mgmt.containerinstance.operations.ContainerGroupsOperations.delete')
+    def test_delete(self, delete_mock):
+        self.testHook.delete('resource_group', 'aci-test')
+        delete_mock.assert_called_with('resource_group', 'aci-test')
+
+    @patch('azure.mgmt.containerinstance.operations.ContainerGroupsOperations.list_by_resource_group')
+    def test_exists_with_existing(self, list_mock):
+        list_mock.return_value = [ContainerGroup(os_type='Linux',
+                                                 containers=[Container(name='test1',
+                                                                       image='hello-world',
+                                                                       resources=self.resources)])]
+        self.assertFalse(self.testHook.exists('test', 'test1'))
+
+    @patch('azure.mgmt.containerinstance.operations.ContainerGroupsOperations.list_by_resource_group')
+    def test_exists_with_not_existing(self, list_mock):
+        list_mock.return_value = [ContainerGroup(os_type='Linux',
+                                                 containers=[Container(name='test1',
+                                                                       image='hello-world',
+                                                                       resources=self.resources)])]
+        self.assertFalse(self.testHook.exists('test', 'not found'))
diff --git a/tests/contrib/hooks/test_azure_container_registry_hook.py b/tests/contrib/hooks/test_azure_container_registry_hook.py
new file mode 100644
index 0000000000000..02c72cce832fc
--- /dev/null
+++ b/tests/contrib/hooks/test_azure_container_registry_hook.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.models.connection import Connection
+from airflow.contrib.hooks.azure_container_registry_hook import AzureContainerRegistryHook
+from airflow.utils import db
+
+
+class TestAzureContainerRegistryHook(unittest.TestCase):
+
+    def test_get_conn(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='azure_container_registry',
+                login='myuser',
+                password='password',
+                host='test.cr',
+            )
+        )
+        hook = AzureContainerRegistryHook(conn_id='azure_container_registry')
+        self.assertIsNotNone(hook.connection)
+        self.assertEqual(hook.connection.username, 'myuser')
+        self.assertEqual(hook.connection.password, 'password')
+        self.assertEqual(hook.connection.server, 'test.cr')
diff --git a/tests/contrib/hooks/test_azure_container_volume_hook.py b/tests/contrib/hooks/test_azure_container_volume_hook.py
new file mode 100644
index 0000000000000..25b3c53ab196d
--- /dev/null
+++ b/tests/contrib/hooks/test_azure_container_volume_hook.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.models.connection import Connection
+from airflow.contrib.hooks.azure_container_volume_hook import AzureContainerVolumeHook
+from airflow.utils import db
+
+
+class TestAzureContainerVolumeHook(unittest.TestCase):
+
+    def test_get_file_volume(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='wasb_test_key',
+                conn_type='wasb',
+                login='login',
+                password='key'
+            )
+        )
+        hook = AzureContainerVolumeHook(wasb_conn_id='wasb_test_key')
+        volume = hook.get_file_volume(mount_name='mount',
+                                      share_name='share',
+                                      storage_account_name='storage',
+                                      read_only=True)
+        self.assertIsNotNone(volume)
+        self.assertEqual(volume.name, 'mount')
+        self.assertEqual(volume.azure_file.share_name, 'share')
+        self.assertEqual(volume.azure_file.storage_account_key, 'key')
+        self.assertEqual(volume.azure_file.storage_account_name, 'storage')
+        self.assertEqual(volume.azure_file.read_only, True)
diff --git a/tests/contrib/hooks/test_azure_cosmos_hook.py b/tests/contrib/hooks/test_azure_cosmos_hook.py
new file mode 100644
index 0000000000000..8994a61625ed9
--- /dev/null
+++ b/tests/contrib/hooks/test_azure_cosmos_hook.py
@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+import json
+import unittest
+import uuid
+
+from airflow.exceptions import AirflowException
+from airflow.contrib.hooks.azure_cosmos_hook import AzureCosmosDBHook
+
+from airflow import configuration
+from airflow.models.connection import Connection
+from airflow.utils import db
+from tests.compat import mock
+
+import logging
+
+
+class TestAzureCosmosDbHook(unittest.TestCase):
+
+    # Set up an environment to test with
+    def setUp(self):
+        # set up some test variables
+        self.test_end_point = 'https://test_endpoint:443'
+        self.test_master_key = 'magic_test_key'
+        self.test_database_name = 'test_database_name'
+        self.test_collection_name = 'test_collection_name'
+        self.test_database_default = 'test_database_default'
+        self.test_collection_default = 'test_collection_default'
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='azure_cosmos_test_key_id',
+                conn_type='azure_cosmos',
+                login=self.test_end_point,
+                password=self.test_master_key,
+                extra=json.dumps({'database_name': self.test_database_default,
+                                  'collection_name': self.test_collection_default})
+            )
+        )
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_create_database(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.cosmos.create_database(self.test_database_name)
+        expected_calls = [mock.call().CreateDatabase({'id': self.test_database_name})]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_create_database_exception(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.assertRaises(AirflowException, self.cosmos.create_database, None)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_create_container_exception(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.assertRaises(AirflowException, self.cosmos.create_collection, None)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_create_container(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.cosmos.create_collection(self.test_collection_name, self.test_database_name)
+        expected_calls = [mock.call().CreateContainer(
+            'dbs/test_database_name',
+            {'id': self.test_collection_name})]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_create_container_default(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.cosmos.create_collection(self.test_collection_name)
+        expected_calls = [mock.call().CreateContainer(
+            'dbs/test_database_default',
+            {'id': self.test_collection_name})]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_upsert_document_default(self, cosmos_mock):
+        test_id = str(uuid.uuid4())
+        cosmos_mock.return_value.CreateItem.return_value = {'id': test_id}
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        returned_item = self.cosmos.upsert_document({'id': test_id})
+        expected_calls = [mock.call().CreateItem(
+            'dbs/' + self.test_database_default + '/colls/' + self.test_collection_default,
+            {'id': test_id})]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+        logging.getLogger().info(returned_item)
+        self.assertEqual(returned_item['id'], test_id)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_upsert_document(self, cosmos_mock):
+        test_id = str(uuid.uuid4())
+        cosmos_mock.return_value.CreateItem.return_value = {'id': test_id}
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        returned_item = self.cosmos.upsert_document(
+            {'data1': 'somedata'},
+            database_name=self.test_database_name,
+            collection_name=self.test_collection_name,
+            document_id=test_id)
+
+        expected_calls = [mock.call().CreateItem(
+            'dbs/' + self.test_database_name + '/colls/' + self.test_collection_name,
+            {'data1': 'somedata', 'id': test_id})]
+
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+        logging.getLogger().info(returned_item)
+        self.assertEqual(returned_item['id'], test_id)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_insert_documents(self, cosmos_mock):
+        test_id1 = str(uuid.uuid4())
+        test_id2 = str(uuid.uuid4())
+        test_id3 = str(uuid.uuid4())
+        documents = [
+            {'id': test_id1, 'data': 'data1'},
+            {'id': test_id2, 'data': 'data2'},
+            {'id': test_id3, 'data': 'data3'}]
+
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        returned_item = self.cosmos.insert_documents(documents)
+        expected_calls = [
+            mock.call().CreateItem(
+                'dbs/' + self.test_database_default + '/colls/' + self.test_collection_default,
+                {'data': 'data1', 'id': test_id1}),
+            mock.call().CreateItem(
+                'dbs/' + self.test_database_default + '/colls/' + self.test_collection_default,
+                {'data': 'data2', 'id': test_id2}),
+            mock.call().CreateItem(
+                'dbs/' + self.test_database_default + '/colls/' + self.test_collection_default,
+                {'data': 'data3', 'id': test_id3})]
+        logging.getLogger().info(returned_item)
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_delete_database(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.cosmos.delete_database(self.test_database_name)
+        expected_calls = [mock.call().DeleteDatabase('dbs/test_database_name')]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_delete_database_exception(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.assertRaises(AirflowException, self.cosmos.delete_database, None)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_delete_container_exception(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.assertRaises(AirflowException, self.cosmos.delete_collection, None)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_delete_container(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.cosmos.delete_collection(self.test_collection_name, self.test_database_name)
+        expected_calls = [mock.call().DeleteContainer('dbs/test_database_name/colls/test_collection_name')]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_delete_container_default(self, cosmos_mock):
+        self.cosmos = AzureCosmosDBHook(azure_cosmos_conn_id='azure_cosmos_test_key_id')
+        self.cosmos.delete_collection(self.test_collection_name)
+        expected_calls = [mock.call().DeleteContainer('dbs/test_database_default/colls/test_collection_name')]
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_azure_data_lake_hook.py b/tests/contrib/hooks/test_azure_data_lake_hook.py
index d99581b829209..107c4e871ad8e 100644
--- a/tests/contrib/hooks/test_azure_data_lake_hook.py
+++ b/tests/contrib/hooks/test_azure_data_lake_hook.py
@@ -23,16 +23,9 @@
 import unittest
 
 from airflow import configuration
-from airflow import models
+from airflow.models.connection import Connection
 from airflow.utils import db
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class TestAzureDataLakeHook(unittest.TestCase):
@@ -40,7 +33,7 @@ class TestAzureDataLakeHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='adl_test_key',
                 conn_type='azure_data_lake',
                 login='client_id',
diff --git a/tests/contrib/hooks/test_azure_fileshare_hook.py b/tests/contrib/hooks/test_azure_fileshare_hook.py
index 5803cd83f7a0f..bea3978ccf40c 100644
--- a/tests/contrib/hooks/test_azure_fileshare_hook.py
+++ b/tests/contrib/hooks/test_azure_fileshare_hook.py
@@ -23,18 +23,10 @@
 import unittest
 
 from airflow import configuration
-from airflow import models
 from airflow.contrib.hooks.azure_fileshare_hook import AzureFileShareHook
+from airflow.models.connection import Connection
 from airflow.utils import db
-
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class TestAzureFileshareHook(unittest.TestCase):
@@ -42,13 +34,13 @@ class TestAzureFileshareHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='wasb_test_key', conn_type='wasb',
                 login='login', password='key'
             )
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='wasb_test_sas_token', conn_type='wasb',
                 login='login', extra=json.dumps({'sas_token': 'token'})
             )
diff --git a/tests/contrib/hooks/test_bigquery_hook.py b/tests/contrib/hooks/test_bigquery_hook.py
index 37ad32d7ec036..e0617cb8e2613 100644
--- a/tests/contrib/hooks/test_bigquery_hook.py
+++ b/tests/contrib/hooks/test_bigquery_hook.py
@@ -20,12 +20,15 @@
 
 import unittest
 import warnings
+from typing import List
 
 from google.auth.exceptions import GoogleAuthError
 import mock
+from googleapiclient.errors import HttpError
 
 from airflow.contrib.hooks import bigquery_hook as hook
-from airflow.contrib.hooks.bigquery_hook import _cleanse_time_partitioning
+from airflow.contrib.hooks.bigquery_hook import _cleanse_time_partitioning, \
+    _validate_value, _api_resource_configs_duplication_check
 
 bq_available = True
 
@@ -52,13 +55,14 @@ def test_throws_exception_with_invalid_query(self):
         self.assertIn('Reason: ', str(context.exception), "")
 
     @unittest.skipIf(not bq_available, 'BQ is not available to run tests')
-    def test_suceeds_with_explicit_legacy_query(self):
+    def test_succeeds_with_explicit_legacy_query(self):
         df = self.instance.get_pandas_df('select 1', dialect='legacy')
         self.assertEqual(df.iloc(0)[0][0], 1)
 
     @unittest.skipIf(not bq_available, 'BQ is not available to run tests')
-    def test_suceeds_with_explicit_std_query(self):
-        df = self.instance.get_pandas_df('select * except(b) from (select 1 a, 2 b)', dialect='standard')
+    def test_succeeds_with_explicit_std_query(self):
+        df = self.instance.get_pandas_df(
+            'select * except(b) from (select 1 a, 2 b)', dialect='standard')
         self.assertEqual(df.iloc(0)[0][0], 1)
 
     @unittest.skipIf(not bq_available, 'BQ is not available to run tests')
@@ -192,7 +196,7 @@ def test_invalid_source_format(self):
 
 # Helpers to test_cancel_queries that have mock_poll_job_complete returning false,
 # unless mock_job_cancel was called with the same job_id
-mock_canceled_jobs = []
+mock_canceled_jobs = []  # type: List
 
 
 def mock_poll_job_complete(job_id):
@@ -205,6 +209,16 @@ def mock_job_cancel(projectId, jobId):
 
 
 class TestBigQueryBaseCursor(unittest.TestCase):
+    def test_bql_deprecation_warning(self):
+        with warnings.catch_warnings(record=True) as w:
+            hook.BigQueryBaseCursor("test", "test").run_query(
+                bql='select * from test_table'
+            )
+            yield
+        self.assertIn(
+            'Deprecated parameter `bql`',
+            w[0].message.args[0])
+
     def test_invalid_schema_update_options(self):
         with self.assertRaises(Exception) as context:
             hook.BigQueryBaseCursor("test", "test").run_load(
@@ -215,16 +229,6 @@ def test_invalid_schema_update_options(self):
             )
         self.assertIn("THIS IS NOT VALID", str(context.exception))
 
-    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
-    def test_bql_deprecation_warning(self, mock_rwc):
-        with warnings.catch_warnings(record=True) as w:
-            hook.BigQueryBaseCursor("test", "test").run_query(
-                bql='select * from test_table'
-            )
-        self.assertIn(
-            'Deprecated parameter `bql`',
-            w[0].message.args[0])
-
     def test_nobql_nosql_param_error(self):
         with self.assertRaises(TypeError) as context:
             hook.BigQueryBaseCursor("test", "test").run_query(
@@ -246,9 +250,7 @@ def test_invalid_schema_update_and_write_disposition(self):
             )
         self.assertIn("schema_update_options is only", str(context.exception))
 
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.LoggingMixin")
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.time")
-    def test_cancel_queries(self, mocked_time, mocked_logging):
+    def test_cancel_queries(self):
         project_id = 12345
         running_job_id = 3
 
@@ -280,13 +282,340 @@ def test_run_query_sql_dialect_override(self, run_with_config):
             args, kwargs = run_with_config.call_args
             self.assertIs(args[0]['query']['useLegacySql'], bool_val)
 
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_query_sql_dialect_legacy_with_query_params(self, run_with_config):
+        cursor = hook.BigQueryBaseCursor(mock.Mock(), "project_id")
+        params = [{
+            'name': "param_name",
+            'parameterType': {'type': "STRING"},
+            'parameterValue': {'value': "param_value"}
+        }]
+        cursor.run_query('query', use_legacy_sql=False, query_params=params)
+        args, kwargs = run_with_config.call_args
+        self.assertIs(args[0]['query']['useLegacySql'], False)
 
-class TestTimePartitioningInRunJob(unittest.TestCase):
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_query_sql_dialect_legacy_with_query_params_fails(self, run_with_config):
+        cursor = hook.BigQueryBaseCursor(mock.Mock(), "project_id")
+        params = [{
+            'name': "param_name",
+            'parameterType': {'type': "STRING"},
+            'parameterValue': {'value': "param_value"}
+        }]
+        with self.assertRaises(ValueError):
+            cursor.run_query('query', use_legacy_sql=True, query_params=params)
 
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.LoggingMixin")
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.time")
     @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
-    def test_run_load_default(self, mocked_rwc, mocked_time, mocked_logging):
+    def test_api_resource_configs(self, run_with_config):
+        for bool_val in [True, False]:
+            cursor = hook.BigQueryBaseCursor(mock.Mock(), "project_id")
+            cursor.run_query('query',
+                             api_resource_configs={
+                                 'query': {'useQueryCache': bool_val}})
+            args, kwargs = run_with_config.call_args
+            self.assertIs(args[0]['query']['useQueryCache'], bool_val)
+            self.assertIs(args[0]['query']['useLegacySql'], True)
+
+    def test_api_resource_configs_duplication_warning(self):
+        with self.assertRaises(ValueError):
+            cursor = hook.BigQueryBaseCursor(mock.Mock(), "project_id")
+            cursor.run_query('query',
+                             use_legacy_sql=True,
+                             api_resource_configs={
+                                 'query': {'useLegacySql': False}})
+
+    def test_validate_value(self):
+        with self.assertRaises(TypeError):
+            _validate_value("case_1", "a", dict)
+        self.assertIsNone(_validate_value("case_2", 0, int))
+
+    def test_duplication_check(self):
+        with self.assertRaises(ValueError):
+            key_one = True
+            _api_resource_configs_duplication_check(
+                "key_one", key_one, {"key_one": False})
+        self.assertIsNone(_api_resource_configs_duplication_check(
+            "key_one", key_one, {"key_one": True}))
+
+    def test_insert_all_succeed(self):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table'
+        rows = [
+            {"json": {"a_key": "a_value_0"}}
+        ]
+        body = {
+            "rows": rows,
+            "ignoreUnknownValues": False,
+            "kind": "bigquery#tableDataInsertAllRequest",
+            "skipInvalidRows": False,
+        }
+
+        mock_service = mock.Mock()
+        method = mock_service.tabledata.return_value.insertAll
+        method.return_value.execute.return_value = {
+            "kind": "bigquery#tableDataInsertAllResponse"
+        }
+        cursor = hook.BigQueryBaseCursor(mock_service, 'project_id')
+        cursor.insert_all(project_id, dataset_id, table_id, rows)
+        method.assert_called_with(projectId=project_id, datasetId=dataset_id,
+                                  tableId=table_id, body=body)
+
+    def test_insert_all_fail(self):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table'
+        rows = [
+            {"json": {"a_key": "a_value_0"}}
+        ]
+
+        mock_service = mock.Mock()
+        method = mock_service.tabledata.return_value.insertAll
+        method.return_value.execute.return_value = {
+            "kind": "bigquery#tableDataInsertAllResponse",
+            "insertErrors": [
+                {
+                    "index": 1,
+                    "errors": []
+                }
+            ]
+        }
+        cursor = hook.BigQueryBaseCursor(mock_service, 'project_id')
+        with self.assertRaises(Exception):
+            cursor.insert_all(project_id, dataset_id, table_id,
+                              rows, fail_on_error=True)
+
+    def test_create_view_fails_on_exception(self):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table_view'
+        view = {
+            'incorrect_key': 'SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`',
+            "useLegacySql": False
+        }
+
+        mock_service = mock.Mock()
+        method = mock_service.tables.return_value.insert
+        method.return_value.execute.side_effect = HttpError(
+            resp={'status': '400'}, content=b'Query is required for views')
+        cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+        with self.assertRaises(Exception):
+            cursor.create_empty_table(project_id, dataset_id, table_id,
+                                      view=view)
+
+    def test_create_view(self):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table_view'
+        view = {
+            'query': 'SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`',
+            "useLegacySql": False
+        }
+
+        mock_service = mock.Mock()
+        method = mock_service.tables.return_value.insert
+        cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+        cursor.create_empty_table(project_id, dataset_id, table_id,
+                                  view=view)
+        body = {
+            'tableReference': {
+                'tableId': table_id
+            },
+            'view': view
+        }
+        method.assert_called_once_with(projectId=project_id, datasetId=dataset_id, body=body)
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_patch_table(self, run_with_config):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        table_id = 'bq_table'
+
+        description_patched = 'Test description.'
+        expiration_time_patched = 2524608000000
+        friendly_name_patched = 'Test friendly name.'
+        labels_patched = {'label1': 'test1', 'label2': 'test2'}
+        schema_patched = [
+            {'name': 'id', 'type': 'STRING', 'mode': 'REQUIRED'},
+            {'name': 'name', 'type': 'STRING', 'mode': 'NULLABLE'},
+            {'name': 'balance', 'type': 'FLOAT', 'mode': 'NULLABLE'},
+            {'name': 'new_field', 'type': 'STRING', 'mode': 'NULLABLE'}
+        ]
+        time_partitioning_patched = {
+            'expirationMs': 10000000
+        }
+        require_partition_filter_patched = True
+
+        mock_service = mock.Mock()
+        method = (mock_service.tables.return_value.patch)
+        cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+        cursor.patch_table(
+            dataset_id, table_id, project_id,
+            description=description_patched,
+            expiration_time=expiration_time_patched,
+            friendly_name=friendly_name_patched,
+            labels=labels_patched, schema=schema_patched,
+            time_partitioning=time_partitioning_patched,
+            require_partition_filter=require_partition_filter_patched
+        )
+
+        body = {
+            "description": description_patched,
+            "expirationTime": expiration_time_patched,
+            "friendlyName": friendly_name_patched,
+            "labels": labels_patched,
+            "schema": {
+                "fields": schema_patched
+            },
+            "timePartitioning": time_partitioning_patched,
+            "requirePartitionFilter": require_partition_filter_patched
+        }
+        method.assert_called_once_with(
+            projectId=project_id,
+            datasetId=dataset_id,
+            tableId=table_id,
+            body=body
+        )
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_patch_view(self, run_with_config):
+        project_id = 'bq-project'
+        dataset_id = 'bq_dataset'
+        view_id = 'bq_view'
+        view_patched = {
+            'query': "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 500",
+            'useLegacySql': False
+        }
+
+        mock_service = mock.Mock()
+        method = (mock_service.tables.return_value.patch)
+        cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+        cursor.patch_table(dataset_id, view_id, project_id, view=view_patched)
+        body = {
+            'view': view_patched
+        }
+        method.assert_called_once_with(
+            projectId=project_id,
+            datasetId=dataset_id,
+            tableId=view_id,
+            body=body
+        )
+
+
+class TestBigQueryCursor(unittest.TestCase):
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_execute_with_parameters(self, mocked_rwc):
+        hook.BigQueryCursor("test", "test").execute(
+            "SELECT %(foo)s", {"foo": "bar"})
+        assert mocked_rwc.call_count == 1
+
+
+class TestLabelsInRunJob(unittest.TestCase):
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_query_with_arg(self, mocked_rwc):
+        project_id = 12345
+
+        def run_with_config(config):
+            self.assertEqual(
+                config['labels'], {'label1': 'test1', 'label2': 'test2'}
+            )
+        mocked_rwc.side_effect = run_with_config
+
+        bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
+        bq_hook.run_query(
+            sql='select 1',
+            destination_dataset_table='my_dataset.my_table',
+            labels={'label1': 'test1', 'label2': 'test2'}
+        )
+
+        assert mocked_rwc.call_count == 1
+
+
+class TestDatasetsOperations(unittest.TestCase):
+
+    def test_create_empty_dataset_no_dataset_id_err(self):
+
+        with self.assertRaises(ValueError):
+            hook.BigQueryBaseCursor(
+                mock.Mock(), "test_create_empty_dataset").create_empty_dataset(
+                dataset_id="", project_id="")
+
+    def test_create_empty_dataset_duplicates_call_err(self):
+        with self.assertRaises(ValueError):
+            hook.BigQueryBaseCursor(
+                mock.Mock(), "test_create_empty_dataset").create_empty_dataset(
+                dataset_id="", project_id="project_test",
+                dataset_reference={
+                    "datasetReference":
+                        {"datasetId": "test_dataset",
+                         "projectId": "project_test2"}})
+
+    def test_get_dataset_without_dataset_id(self):
+        with mock.patch.object(hook.BigQueryHook, 'get_service'):
+            with self.assertRaises(ValueError):
+                hook.BigQueryBaseCursor(
+                    mock.Mock(), "test_create_empty_dataset").get_dataset(
+                    dataset_id="", project_id="project_test")
+
+    def test_get_dataset(self):
+        expected_result = {
+            "kind": "bigquery#dataset",
+            "location": "US",
+            "id": "your-project:dataset_2_test",
+            "datasetReference": {
+                "projectId": "your-project",
+                "datasetId": "dataset_2_test"
+            }
+        }
+        dataset_id = "test_dataset"
+        project_id = "project_test"
+
+        bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
+        with mock.patch.object(bq_hook.service, 'datasets') as MockService:
+            MockService.return_value.get(datasetId=dataset_id,
+                                         projectId=project_id).execute.\
+                return_value = expected_result
+            result = bq_hook.get_dataset(dataset_id=dataset_id,
+                                         project_id=project_id)
+            self.assertEqual(result, expected_result)
+
+    def test_get_datasets_list(self):
+        expected_result = {'datasets': [
+            {
+                "kind": "bigquery#dataset",
+                "location": "US",
+                "id": "your-project:dataset_2_test",
+                "datasetReference": {
+                    "projectId": "your-project",
+                    "datasetId": "dataset_2_test"
+                }
+            },
+            {
+                "kind": "bigquery#dataset",
+                "location": "US",
+                "id": "your-project:dataset_1_test",
+                "datasetReference": {
+                    "projectId": "your-project",
+                    "datasetId": "dataset_1_test"
+                }
+            }
+        ]}
+        project_id = "project_test"''
+
+        mocked = mock.Mock()
+        with mock.patch.object(hook.BigQueryBaseCursor(mocked, project_id).service,
+                               'datasets') as MockService:
+            MockService.return_value.list(
+                projectId=project_id).execute.return_value = expected_result
+            result = hook.BigQueryBaseCursor(
+                mocked, "test_create_empty_dataset").get_datasets_list(
+                project_id=project_id)
+            self.assertEqual(result, expected_result['datasets'])
+
+
+class TestTimePartitioningInRunJob(unittest.TestCase):
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_load_default(self, mocked_rwc):
         project_id = 12345
 
         def run_with_config(config):
@@ -300,12 +629,18 @@ def run_with_config(config):
             source_uris=[],
         )
 
-        mocked_rwc.assert_called_once()
+        assert mocked_rwc.call_count == 1
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_with_auto_detect(self, run_with_config):
+        destination_project_dataset_table = "autodetect.table"
+        cursor = hook.BigQueryBaseCursor(mock.Mock(), "project_id")
+        cursor.run_load(destination_project_dataset_table, [], [], autodetect=True)
+        args, kwargs = run_with_config.call_args
+        self.assertIs(args[0]['load']['autodetect'], True)
 
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.LoggingMixin")
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.time")
     @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
-    def test_run_load_with_arg(self, mocked_rwc, mocked_time, mocked_logging):
+    def test_run_load_with_arg(self, mocked_rwc):
         project_id = 12345
 
         def run_with_config(config):
@@ -327,12 +662,10 @@ def run_with_config(config):
             time_partitioning={'type': 'DAY', 'field': 'test_field', 'expirationMs': 1000}
         )
 
-        mocked_rwc.assert_called_once()
+        assert mocked_rwc.call_count == 1
 
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.LoggingMixin")
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.time")
     @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
-    def test_run_query_default(self, mocked_rwc, mocked_time, mocked_logging):
+    def test_run_query_default(self, mocked_rwc):
         project_id = 12345
 
         def run_with_config(config):
@@ -342,12 +675,10 @@ def run_with_config(config):
         bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
         bq_hook.run_query(sql='select 1')
 
-        mocked_rwc.assert_called_once()
+        assert mocked_rwc.call_count == 1
 
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.LoggingMixin")
-    @mock.patch("airflow.contrib.hooks.bigquery_hook.time")
     @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
-    def test_run_query_with_arg(self, mocked_rwc, mocked_time, mocked_logging):
+    def test_run_query_with_arg(self, mocked_rwc):
         project_id = 12345
 
         def run_with_config(config):
@@ -365,10 +696,11 @@ def run_with_config(config):
         bq_hook.run_query(
             sql='select 1',
             destination_dataset_table='my_dataset.my_table',
-            time_partitioning={'type': 'DAY', 'field': 'test_field', 'expirationMs': 1000}
+            time_partitioning={'type': 'DAY',
+                               'field': 'test_field', 'expirationMs': 1000}
         )
 
-        mocked_rwc.assert_called_once()
+        assert mocked_rwc.call_count == 1
 
     def test_dollar_makes_partition(self):
         tp_out = _cleanse_time_partitioning('test.teast$20170101', {})
@@ -390,12 +722,85 @@ def test_extra_time_partitioning_options(self):
         }
         self.assertEqual(tp_out, expect)
 
-    def test_cant_add_dollar_and_field_name(self):
-        with self.assertRaises(AssertionError):
-            _cleanse_time_partitioning(
-                'test.teast$20170101',
-                {'type': 'DAY', 'field': 'test_field', 'expirationMs': 1000}
+
+class TestClusteringInRunJob(unittest.TestCase):
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_load_default(self, mocked_rwc):
+        project_id = 12345
+
+        def run_with_config(config):
+            self.assertIsNone(config['load'].get('clustering'))
+        mocked_rwc.side_effect = run_with_config
+
+        bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
+        bq_hook.run_load(
+            destination_project_dataset_table='my_dataset.my_table',
+            schema_fields=[],
+            source_uris=[],
+        )
+
+        assert mocked_rwc.call_count == 1
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_load_with_arg(self, mocked_rwc):
+        project_id = 12345
+
+        def run_with_config(config):
+            self.assertEqual(
+                config['load']['clustering'],
+                {
+                    'fields': ['field1', 'field2']
+                }
+            )
+        mocked_rwc.side_effect = run_with_config
+
+        bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
+        bq_hook.run_load(
+            destination_project_dataset_table='my_dataset.my_table',
+            schema_fields=[],
+            source_uris=[],
+            cluster_fields=['field1', 'field2'],
+            time_partitioning={'type': 'DAY'}
+        )
+
+        assert mocked_rwc.call_count == 1
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_query_default(self, mocked_rwc):
+        project_id = 12345
+
+        def run_with_config(config):
+            self.assertIsNone(config['query'].get('clustering'))
+        mocked_rwc.side_effect = run_with_config
+
+        bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
+        bq_hook.run_query(sql='select 1')
+
+        assert mocked_rwc.call_count == 1
+
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_run_query_with_arg(self, mocked_rwc):
+        project_id = 12345
+
+        def run_with_config(config):
+            self.assertEqual(
+                config['query']['clustering'],
+                {
+                    'fields': ['field1', 'field2']
+                }
             )
+        mocked_rwc.side_effect = run_with_config
+
+        bq_hook = hook.BigQueryBaseCursor(mock.Mock(), project_id)
+        bq_hook.run_query(
+            sql='select 1',
+            destination_dataset_table='my_dataset.my_table',
+            cluster_fields=['field1', 'field2'],
+            time_partitioning={'type': 'DAY'}
+        )
+
+        assert mocked_rwc.call_count == 1
 
 
 class TestBigQueryHookLegacySql(unittest.TestCase):
@@ -418,5 +823,21 @@ def test_legacy_sql_override_propagates_properly(self, run_with_config):
             self.assertIs(args[0]['query']['useLegacySql'], False)
 
 
+class TestBigQueryHookLocation(unittest.TestCase):
+    @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+    def test_location_propagates_properly(self, run_with_config):
+        with mock.patch.object(hook.BigQueryHook, 'get_service'):
+            bq_hook = hook.BigQueryHook(location=None)
+            self.assertIsNone(bq_hook.location)
+
+            bq_cursor = hook.BigQueryBaseCursor(mock.Mock(),
+                                                'test-project',
+                                                location=None)
+            self.assertIsNone(bq_cursor.location)
+            bq_cursor.run_query(sql='select 1', location='US')
+            assert run_with_config.call_count == 1
+            self.assertEqual(bq_cursor.location, 'US')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_cassandra_hook.py b/tests/contrib/hooks/test_cassandra_hook.py
index e420ec009583a..fb48961e53c20 100644
--- a/tests/contrib/hooks/test_cassandra_hook.py
+++ b/tests/contrib/hooks/test_cassandra_hook.py
@@ -18,8 +18,6 @@
 # under the License.
 
 import unittest
-import mock
-from mock import patch
 
 from airflow import configuration
 from airflow.contrib.hooks.cassandra_hook import CassandraHook
@@ -27,18 +25,38 @@
 from cassandra.policies import (
     TokenAwarePolicy, RoundRobinPolicy, DCAwareRoundRobinPolicy, WhiteListRoundRobinPolicy
 )
-from airflow import models
+from airflow.models.connection import Connection
 from airflow.utils import db
+from tests.compat import mock, patch
 
 
 class CassandraHookTest(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='cassandra_test', conn_type='cassandra',
                 host='host-1,host-2', port='9042', schema='test_keyspace',
                 extra='{"load_balancing_policy":"TokenAwarePolicy"}'))
+        db.merge_conn(
+            Connection(
+                conn_id='cassandra_default_with_schema', conn_type='cassandra',
+                host='cassandra', port='9042', schema='s'))
+
+        hook = CassandraHook("cassandra_default")
+        session = hook.get_conn()
+        cqls = [
+            "DROP SCHEMA IF EXISTS s",
+            """
+                CREATE SCHEMA s WITH REPLICATION =
+                    { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }
+            """,
+        ]
+        for cql in cqls:
+            session.execute(cql)
+
+        session.shutdown()
+        hook.shutdown_cluster()
 
     def test_get_conn(self):
         with mock.patch.object(Cluster, "connect") as mock_connect, \
@@ -46,7 +64,7 @@ def test_get_conn(self):
             mock_connect.return_value = 'session'
             hook = CassandraHook(cassandra_conn_id='cassandra_test')
             hook.get_conn()
-            mock_getaddrinfo.assert_called()
+            assert mock_getaddrinfo.called
             mock_connect.assert_called_once_with('test_keyspace')
 
             cluster = hook.get_cluster()
@@ -117,16 +135,10 @@ def _assert_get_lb_policy(self, policy_name, policy_args, expected_policy_type,
             thrown = True
         self.assertEqual(should_throw, thrown)
 
-    def test_record_exists(self):
-        hook = CassandraHook()
+    def test_record_exists_with_keyspace_from_cql(self):
+        hook = CassandraHook("cassandra_default")
         session = hook.get_conn()
-
         cqls = [
-            "DROP SCHEMA IF EXISTS s",
-            """
-                CREATE SCHEMA s WITH REPLICATION =
-                    { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }
-            """,
             "DROP TABLE IF EXISTS s.t",
             "CREATE TABLE s.t (pk1 text, pk2 text, c text, PRIMARY KEY (pk1, pk2))",
             "INSERT INTO s.t (pk1, pk2, c) VALUES ('foo', 'bar', 'baz')",
@@ -137,6 +149,58 @@ def test_record_exists(self):
         self.assertTrue(hook.record_exists("s.t", {"pk1": "foo", "pk2": "bar"}))
         self.assertFalse(hook.record_exists("s.t", {"pk1": "foo", "pk2": "baz"}))
 
+        session.shutdown()
+        hook.shutdown_cluster()
+
+    def test_record_exists_with_keyspace_from_session(self):
+        hook = CassandraHook("cassandra_default_with_schema")
+        session = hook.get_conn()
+        cqls = [
+            "DROP TABLE IF EXISTS t",
+            "CREATE TABLE t (pk1 text, pk2 text, c text, PRIMARY KEY (pk1, pk2))",
+            "INSERT INTO t (pk1, pk2, c) VALUES ('foo', 'bar', 'baz')",
+        ]
+        for cql in cqls:
+            session.execute(cql)
+
+        self.assertTrue(hook.record_exists("t", {"pk1": "foo", "pk2": "bar"}))
+        self.assertFalse(hook.record_exists("t", {"pk1": "foo", "pk2": "baz"}))
+
+        session.shutdown()
+        hook.shutdown_cluster()
+
+    def test_table_exists_with_keyspace_from_cql(self):
+        hook = CassandraHook("cassandra_default")
+        session = hook.get_conn()
+        cqls = [
+            "DROP TABLE IF EXISTS s.t",
+            "CREATE TABLE s.t (pk1 text PRIMARY KEY)",
+        ]
+        for cql in cqls:
+            session.execute(cql)
+
+        self.assertTrue(hook.table_exists("s.t"))
+        self.assertFalse(hook.table_exists("s.u"))
+
+        session.shutdown()
+        hook.shutdown_cluster()
+
+    def test_table_exists_with_keyspace_from_session(self):
+        hook = CassandraHook("cassandra_default_with_schema")
+        session = hook.get_conn()
+        cqls = [
+            "DROP TABLE IF EXISTS t",
+            "CREATE TABLE t (pk1 text PRIMARY KEY)",
+        ]
+        for cql in cqls:
+            session.execute(cql)
+
+        self.assertTrue(hook.table_exists("t"))
+        self.assertFalse(hook.table_exists("u"))
+
+        session.shutdown()
+        hook.shutdown_cluster()
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_databricks_hook.py b/tests/contrib/hooks/test_databricks_hook.py
index 6052a6d54f1f8..e335e21e61fca 100644
--- a/tests/contrib/hooks/test_databricks_hook.py
+++ b/tests/contrib/hooks/test_databricks_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,23 +18,22 @@
 # under the License.
 #
 
+import itertools
 import json
 import unittest
 
+from requests import exceptions as requests_exceptions
+
 from airflow import __version__
-from airflow.contrib.hooks.databricks_hook import DatabricksHook, RunState, SUBMIT_RUN_ENDPOINT, _TokenAuth
+from airflow.contrib.hooks.databricks_hook import (
+    DatabricksHook,
+    RunState,
+    SUBMIT_RUN_ENDPOINT
+)
 from airflow.exceptions import AirflowException
-from airflow.models import Connection
+from airflow.models.connection import Connection
 from airflow.utils import db
-from requests import exceptions as requests_exceptions
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'databricks-operator'
 DEFAULT_CONN_ID = 'databricks_default'
@@ -46,7 +45,9 @@
     'node_type_id': 'r3.xlarge',
     'num_workers': 1
 }
+CLUSTER_ID = 'cluster_id'
 RUN_ID = 1
+JOB_ID = 42
 HOST = 'xx.cloud.databricks.com'
 HOST_WITH_SCHEME = 'https://xx.cloud.databricks.com'
 LOGIN = 'login'
@@ -63,9 +64,21 @@
         'state_message': STATE_MESSAGE
     }
 }
+NOTEBOOK_PARAMS = {
+    "dry-run": "true",
+    "oldest-time-to-consider": "1457570074236"
+}
+JAR_PARAMS = ["param1", "param2"]
 RESULT_STATE = None
 
 
+def run_now_endpoint(host):
+    """
+    Utility function to generate the run now endpoint given the host.
+    """
+    return 'https://{}/api/2.0/jobs/run-now'.format(host)
+
+
 def submit_run_endpoint(host):
     """
     Utility function to generate the submit run endpoint given the host.
@@ -79,16 +92,72 @@ def get_run_endpoint(host):
     """
     return 'https://{}/api/2.0/jobs/runs/get'.format(host)
 
+
 def cancel_run_endpoint(host):
     """
     Utility function to generate the get run endpoint given the host.
     """
     return 'https://{}/api/2.0/jobs/runs/cancel'.format(host)
 
+
+def start_cluster_endpoint(host):
+    """
+    Utility function to generate the get run endpoint given the host.
+    """
+    return 'https://{}/api/2.0/clusters/start'.format(host)
+
+
+def restart_cluster_endpoint(host):
+    """
+    Utility function to generate the get run endpoint given the host.
+    """
+    return 'https://{}/api/2.0/clusters/restart'.format(host)
+
+
+def terminate_cluster_endpoint(host):
+    """
+    Utility function to generate the get run endpoint given the host.
+    """
+    return 'https://{}/api/2.0/clusters/delete'.format(host)
+
+
+def create_valid_response_mock(content):
+    response = mock.MagicMock()
+    response.json.return_value = content
+    return response
+
+
+def create_post_side_effect(exception, status_code=500):
+    if exception != requests_exceptions.HTTPError:
+        return exception()
+    else:
+        response = mock.MagicMock()
+        response.status_code = status_code
+        response.raise_for_status.side_effect = exception(response=response)
+        return response
+
+
+def setup_mock_requests(mock_requests,
+                        exception,
+                        status_code=500,
+                        error_count=None,
+                        response_content=None):
+    side_effect = create_post_side_effect(exception, status_code)
+
+    if error_count is None:
+        # POST requests will fail indefinitely
+        mock_requests.post.side_effect = itertools.repeat(side_effect)
+    else:
+        # POST requests will fail 'error_count' times, and then they will succeed (once)
+        mock_requests.post.side_effect = \
+            [side_effect] * error_count + [create_valid_response_mock(response_content)]
+
+
 class DatabricksHookTest(unittest.TestCase):
     """
     Tests for DatabricksHook.
     """
+
     @db.provide_session
     def setUp(self, session=None):
         conn = session.query(Connection) \
@@ -97,55 +166,101 @@ def setUp(self, session=None):
         conn.host = HOST
         conn.login = LOGIN
         conn.password = PASSWORD
+        conn.extra = None
         session.commit()
 
-        self.hook = DatabricksHook()
+        self.hook = DatabricksHook(retry_delay=0)
 
     def test_parse_host_with_proper_host(self):
         host = self.hook._parse_host(HOST)
-        self.assertEquals(host, HOST)
+        self.assertEqual(host, HOST)
 
     def test_parse_host_with_scheme(self):
         host = self.hook._parse_host(HOST_WITH_SCHEME)
-        self.assertEquals(host, HOST)
+        self.assertEqual(host, HOST)
 
     def test_init_bad_retry_limit(self):
-        with self.assertRaises(AssertionError):
-            DatabricksHook(retry_limit = 0)
+        with self.assertRaises(ValueError):
+            DatabricksHook(retry_limit=0)
 
-    @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
-    def test_do_api_call_with_error_retry(self, mock_requests):
-        for exception in [requests_exceptions.ConnectionError, requests_exceptions.Timeout]:
-            with mock.patch.object(self.hook.log, 'error') as mock_errors:
-                mock_requests.reset_mock()
-                mock_requests.post.side_effect = exception()
+    def test_do_api_call_retries_with_retryable_error(self):
+        for exception in [requests_exceptions.ConnectionError,
+                          requests_exceptions.SSLError,
+                          requests_exceptions.Timeout,
+                          requests_exceptions.ConnectTimeout,
+                          requests_exceptions.HTTPError]:
+            with mock.patch('airflow.contrib.hooks.databricks_hook.requests') as mock_requests:
+                with mock.patch.object(self.hook.log, 'error') as mock_errors:
+                    setup_mock_requests(mock_requests, exception)
 
-                with self.assertRaises(AirflowException):
-                    self.hook._do_api_call(SUBMIT_RUN_ENDPOINT, {})
+                    with self.assertRaises(AirflowException):
+                        self.hook._do_api_call(SUBMIT_RUN_ENDPOINT, {})
 
-                self.assertEquals(len(mock_errors.mock_calls), self.hook.retry_limit)
+                    self.assertEqual(mock_errors.call_count, self.hook.retry_limit)
 
     @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
-    def test_do_api_call_with_bad_status_code(self, mock_requests):
-        mock_requests.codes.ok = 200
-        status_code_mock = mock.PropertyMock(return_value=500)
-        type(mock_requests.post.return_value).status_code = status_code_mock
-        with self.assertRaises(AirflowException):
-            self.hook._do_api_call(SUBMIT_RUN_ENDPOINT, {})
+    def test_do_api_call_does_not_retry_with_non_retryable_error(self, mock_requests):
+        setup_mock_requests(
+            mock_requests, requests_exceptions.HTTPError, status_code=400
+        )
+
+        with mock.patch.object(self.hook.log, 'error') as mock_errors:
+            with self.assertRaises(AirflowException):
+                self.hook._do_api_call(SUBMIT_RUN_ENDPOINT, {})
+
+            mock_errors.assert_not_called()
+
+    def test_do_api_call_succeeds_after_retrying(self):
+        for exception in [requests_exceptions.ConnectionError,
+                          requests_exceptions.SSLError,
+                          requests_exceptions.Timeout,
+                          requests_exceptions.ConnectTimeout,
+                          requests_exceptions.HTTPError]:
+            with mock.patch('airflow.contrib.hooks.databricks_hook.requests') as mock_requests:
+                with mock.patch.object(self.hook.log, 'error') as mock_errors:
+                    setup_mock_requests(
+                        mock_requests,
+                        exception,
+                        error_count=2,
+                        response_content={'run_id': '1'}
+                    )
+
+                    response = self.hook._do_api_call(SUBMIT_RUN_ENDPOINT, {})
+
+                    self.assertEqual(mock_errors.call_count, 2)
+                    self.assertEqual(response, {'run_id': '1'})
+
+    @mock.patch('airflow.contrib.hooks.databricks_hook.sleep')
+    def test_do_api_call_waits_between_retries(self, mock_sleep):
+        retry_delay = 5
+        self.hook = DatabricksHook(retry_delay=retry_delay)
+
+        for exception in [requests_exceptions.ConnectionError,
+                          requests_exceptions.SSLError,
+                          requests_exceptions.Timeout,
+                          requests_exceptions.ConnectTimeout,
+                          requests_exceptions.HTTPError]:
+            with mock.patch('airflow.contrib.hooks.databricks_hook.requests') as mock_requests:
+                with mock.patch.object(self.hook.log, 'error'):
+                    mock_sleep.reset_mock()
+                    setup_mock_requests(mock_requests, exception)
+
+                    with self.assertRaises(AirflowException):
+                        self.hook._do_api_call(SUBMIT_RUN_ENDPOINT, {})
+
+                    self.assertEqual(len(mock_sleep.mock_calls), self.hook.retry_limit - 1)
+                    mock_sleep.assert_called_with(retry_delay)
 
     @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
     def test_submit_run(self, mock_requests):
-        mock_requests.codes.ok = 200
         mock_requests.post.return_value.json.return_value = {'run_id': '1'}
-        status_code_mock = mock.PropertyMock(return_value=200)
-        type(mock_requests.post.return_value).status_code = status_code_mock
         json = {
-          'notebook_task': NOTEBOOK_TASK,
-          'new_cluster': NEW_CLUSTER
+            'notebook_task': NOTEBOOK_TASK,
+            'new_cluster': NEW_CLUSTER
         }
         run_id = self.hook.submit_run(json)
 
-        self.assertEquals(run_id, '1')
+        self.assertEqual(run_id, '1')
         mock_requests.post.assert_called_once_with(
             submit_run_endpoint(HOST),
             json={
@@ -157,15 +272,38 @@ def test_submit_run(self, mock_requests):
             timeout=self.hook.timeout_seconds)
 
     @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
-    def test_get_run_page_url(self, mock_requests):
+    def test_run_now(self, mock_requests):
         mock_requests.codes.ok = 200
-        mock_requests.get.return_value.json.return_value = GET_RUN_RESPONSE
+        mock_requests.post.return_value.json.return_value = {'run_id': '1'}
         status_code_mock = mock.PropertyMock(return_value=200)
-        type(mock_requests.get.return_value).status_code = status_code_mock
+        type(mock_requests.post.return_value).status_code = status_code_mock
+        json = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'jar_params': JAR_PARAMS,
+            'job_id': JOB_ID
+        }
+        run_id = self.hook.run_now(json)
+
+        self.assertEqual(run_id, '1')
+
+        mock_requests.post.assert_called_once_with(
+            run_now_endpoint(HOST),
+            json={
+                'notebook_params': NOTEBOOK_PARAMS,
+                'jar_params': JAR_PARAMS,
+                'job_id': JOB_ID
+            },
+            auth=(LOGIN, PASSWORD),
+            headers=USER_AGENT_HEADER,
+            timeout=self.hook.timeout_seconds)
+
+    @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
+    def test_get_run_page_url(self, mock_requests):
+        mock_requests.get.return_value.json.return_value = GET_RUN_RESPONSE
 
         run_page_url = self.hook.get_run_page_url(RUN_ID)
 
-        self.assertEquals(run_page_url, RUN_PAGE_URL)
+        self.assertEqual(run_page_url, RUN_PAGE_URL)
         mock_requests.get.assert_called_once_with(
             get_run_endpoint(HOST),
             json={'run_id': RUN_ID},
@@ -175,14 +313,11 @@ def test_get_run_page_url(self, mock_requests):
 
     @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
     def test_get_run_state(self, mock_requests):
-        mock_requests.codes.ok = 200
         mock_requests.get.return_value.json.return_value = GET_RUN_RESPONSE
-        status_code_mock = mock.PropertyMock(return_value=200)
-        type(mock_requests.get.return_value).status_code = status_code_mock
 
         run_state = self.hook.get_run_state(RUN_ID)
 
-        self.assertEquals(run_state, RunState(
+        self.assertEqual(run_state, RunState(
             LIFE_CYCLE_STATE,
             RESULT_STATE,
             STATE_MESSAGE))
@@ -195,10 +330,7 @@ def test_get_run_state(self, mock_requests):
 
     @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
     def test_cancel_run(self, mock_requests):
-        mock_requests.codes.ok = 200
         mock_requests.post.return_value.json.return_value = GET_RUN_RESPONSE
-        status_code_mock = mock.PropertyMock(return_value=200)
-        type(mock_requests.post.return_value).status_code = status_code_mock
 
         self.hook.cancel_run(RUN_ID)
 
@@ -209,11 +341,60 @@ def test_cancel_run(self, mock_requests):
             headers=USER_AGENT_HEADER,
             timeout=self.hook.timeout_seconds)
 
+    @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
+    def test_start_cluster(self, mock_requests):
+        mock_requests.codes.ok = 200
+        mock_requests.post.return_value.json.return_value = {}
+        status_code_mock = mock.PropertyMock(return_value=200)
+        type(mock_requests.post.return_value).status_code = status_code_mock
+
+        self.hook.start_cluster({"cluster_id": CLUSTER_ID})
+
+        mock_requests.post.assert_called_once_with(
+            start_cluster_endpoint(HOST),
+            json={'cluster_id': CLUSTER_ID},
+            auth=(LOGIN, PASSWORD),
+            headers=USER_AGENT_HEADER,
+            timeout=self.hook.timeout_seconds)
+
+    @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
+    def test_restart_cluster(self, mock_requests):
+        mock_requests.codes.ok = 200
+        mock_requests.post.return_value.json.return_value = {}
+        status_code_mock = mock.PropertyMock(return_value=200)
+        type(mock_requests.post.return_value).status_code = status_code_mock
+
+        self.hook.restart_cluster({"cluster_id": CLUSTER_ID})
+
+        mock_requests.post.assert_called_once_with(
+            restart_cluster_endpoint(HOST),
+            json={'cluster_id': CLUSTER_ID},
+            auth=(LOGIN, PASSWORD),
+            headers=USER_AGENT_HEADER,
+            timeout=self.hook.timeout_seconds)
+
+    @mock.patch('airflow.contrib.hooks.databricks_hook.requests')
+    def test_terminate_cluster(self, mock_requests):
+        mock_requests.codes.ok = 200
+        mock_requests.post.return_value.json.return_value = {}
+        status_code_mock = mock.PropertyMock(return_value=200)
+        type(mock_requests.post.return_value).status_code = status_code_mock
+
+        self.hook.terminate_cluster({"cluster_id": CLUSTER_ID})
+
+        mock_requests.post.assert_called_once_with(
+            terminate_cluster_endpoint(HOST),
+            json={'cluster_id': CLUSTER_ID},
+            auth=(LOGIN, PASSWORD),
+            headers=USER_AGENT_HEADER,
+            timeout=self.hook.timeout_seconds)
+
 
 class DatabricksHookTokenTest(unittest.TestCase):
     """
     Tests for DatabricksHook when auth is done with token.
     """
+
     @db.provide_session
     def setUp(self, session=None):
         conn = session.query(Connection) \
@@ -231,15 +412,15 @@ def test_submit_run(self, mock_requests):
         status_code_mock = mock.PropertyMock(return_value=200)
         type(mock_requests.post.return_value).status_code = status_code_mock
         json = {
-          'notebook_task': NOTEBOOK_TASK,
-          'new_cluster': NEW_CLUSTER
+            'notebook_task': NOTEBOOK_TASK,
+            'new_cluster': NEW_CLUSTER
         }
         run_id = self.hook.submit_run(json)
 
-        self.assertEquals(run_id, '1')
+        self.assertEqual(run_id, '1')
         args = mock_requests.post.call_args
         kwargs = args[1]
-        self.assertEquals(kwargs['auth'].token, TOKEN)
+        self.assertEqual(kwargs['auth'].token, TOKEN)
 
 
 class RunStateTest(unittest.TestCase):
diff --git a/tests/contrib/hooks/test_datadog_hook.py b/tests/contrib/hooks/test_datadog_hook.py
new file mode 100644
index 0000000000000..09e19d700d862
--- /dev/null
+++ b/tests/contrib/hooks/test_datadog_hook.py
@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import mock
+import unittest
+
+from airflow.exceptions import AirflowException
+from airflow.models.connection import Connection
+
+from airflow.contrib.hooks.datadog_hook import DatadogHook
+
+
+APP_KEY = 'app_key'
+API_KEY = 'api_key'
+METRIC_NAME = 'metric'
+DATAPOINT = 7
+TAGS = ['tag']
+TYPE = 'rate'
+INTERVAL = 30
+TITLE = 'title'
+TEXT = 'text'
+AGGREGATION_KEY = 'aggregation-key'
+ALERT_TYPE = 'warning'
+DATE_HAPPENED = 12345
+HANDLE = 'handle'
+PRIORITY = 'normal'
+RELATED_EVENT_ID = 7
+DEVICE_NAME = 'device-name'
+
+
+class TestDatadogHook(unittest.TestCase):
+
+    @mock.patch('airflow.contrib.hooks.datadog_hook.initialize')
+    @mock.patch('airflow.contrib.hooks.datadog_hook.DatadogHook.get_connection')
+    def setUp(self, mock_get_connection, mock_initialize):
+        mock_get_connection.return_value = Connection(extra=json.dumps({
+            'app_key': APP_KEY,
+            'api_key': API_KEY,
+        }))
+        self.hook = DatadogHook()
+
+    @mock.patch('airflow.contrib.hooks.datadog_hook.initialize')
+    @mock.patch('airflow.contrib.hooks.datadog_hook.DatadogHook.get_connection')
+    def test_api_key_required(self, mock_get_connection, mock_initialize):
+        mock_get_connection.return_value = Connection()
+        with self.assertRaises(AirflowException) as ctx:
+            DatadogHook()
+        self.assertEqual(str(ctx.exception),
+                         'api_key must be specified in the Datadog connection details')
+
+    def test_validate_response_valid(self):
+        try:
+            self.hook.validate_response({'status': 'ok'})
+        except AirflowException:
+            self.fail('Unexpected AirflowException raised')
+
+    def test_validate_response_invalid(self):
+        with self.assertRaises(AirflowException):
+            self.hook.validate_response({'status': 'error'})
+
+    @mock.patch('airflow.contrib.hooks.datadog_hook.api.Metric.send')
+    def test_send_metric(self, mock_send):
+        mock_send.return_value = {'status': 'ok'}
+        self.hook.send_metric(
+            METRIC_NAME,
+            DATAPOINT,
+            tags=TAGS,
+            type_=TYPE,
+            interval=INTERVAL,
+        )
+        mock_send.assert_called_with(
+            metric=METRIC_NAME,
+            points=DATAPOINT,
+            host=self.hook.host,
+            tags=TAGS,
+            type=TYPE,
+            interval=INTERVAL,
+        )
+
+    @mock.patch('airflow.contrib.hooks.datadog_hook.api.Metric.query')
+    @mock.patch('airflow.contrib.hooks.datadog_hook.time.time')
+    def test_query_metric(self, mock_time, mock_query):
+        now = 12345
+        mock_time.return_value = now
+        mock_query.return_value = {'status': 'ok'}
+        self.hook.query_metric('query', 60, 30)
+        mock_query.assert_called_with(
+            start=now - 60,
+            end=now - 30,
+            query='query',
+        )
+
+    @mock.patch('airflow.contrib.hooks.datadog_hook.api.Event.create')
+    def test_post_event(self, mock_create):
+        mock_create.return_value = {'status': 'ok'}
+        self.hook.post_event(
+            TITLE,
+            TEXT,
+            aggregation_key=AGGREGATION_KEY,
+            alert_type=ALERT_TYPE,
+            date_happened=DATE_HAPPENED,
+            handle=HANDLE,
+            priority=PRIORITY,
+            related_event_id=RELATED_EVENT_ID,
+            tags=TAGS,
+            device_name=DEVICE_NAME,
+        )
+        mock_create.assert_called_with(
+            title=TITLE,
+            text=TEXT,
+            aggregation_key=AGGREGATION_KEY,
+            alert_type=ALERT_TYPE,
+            date_happened=DATE_HAPPENED,
+            handle=HANDLE,
+            priority=PRIORITY,
+            related_event_id=RELATED_EVENT_ID,
+            tags=TAGS,
+            host=self.hook.host,
+            device_name=DEVICE_NAME,
+            source_type_name=self.hook.source_type_name,
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_dingding_hook.py b/tests/contrib/hooks/test_dingding_hook.py
new file mode 100644
index 0000000000000..5afa786f34553
--- /dev/null
+++ b/tests/contrib/hooks/test_dingding_hook.py
@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import unittest
+
+from airflow.utils import db
+
+from airflow import configuration
+
+from airflow.contrib.hooks.dingding_hook import DingdingHook
+from airflow.models.connection import Connection
+
+
+class TestDingdingHook(unittest.TestCase):
+    conn_id = 'dingding_conn_id_test'
+
+    def setUp(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id=self.conn_id,
+                conn_type='http',
+                host='https://oapi.dingtalk.com',
+                password='you_token_here'))
+
+    def test_get_endpoint_conn_id(self):
+        hook = DingdingHook(dingding_conn_id=self.conn_id)
+        endpoint = hook._get_endpoint()
+        self.assertEqual('robot/send?access_token=you_token_here', endpoint)
+
+    def test_build_text_message_not_remind(self):
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'text',
+            'message': 'Airflow dingding text message remind no one',
+            'at_mobiles': False,
+            'at_all': False,
+        }
+        expect = {
+            'msgtype': 'text',
+            'text': {
+                'content': 'Airflow dingding text message remind no one'
+            },
+            'at': {
+                'atMobiles': False,
+                'isAtAll': False
+            }
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_text_message_remind_specific(self):
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'text',
+            'message': 'Airflow dingding text message remind specific users',
+            'at_mobiles': ['1234', '5768'],
+            'at_all': False,
+        }
+        expect = {
+            'msgtype': 'text',
+            'text': {
+                'content': 'Airflow dingding text message remind specific users'
+            },
+            'at': {
+                'atMobiles': ['1234', '5768'],
+                'isAtAll': False
+            }
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_text_message_remind_all(self):
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'text',
+            'message': 'Airflow dingding text message remind all user in group',
+            'at_all': True,
+        }
+        expect = {
+            'msgtype': 'text',
+            'text': {
+                'content': 'Airflow dingding text message remind all user in group'
+            },
+            'at': {
+                'atMobiles': None,
+                'isAtAll': True
+            }
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_markdown_message_remind_specific(self):
+        msg = {
+            'title': 'Airflow dingding markdown message',
+            'text': '# Markdown message title\ncontent content .. \n### sub-title\n'
+                    '![logo](http://airflow.apache.org/_images/pin_large.png)'
+        }
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'markdown',
+            'message': msg,
+            'at_mobiles': ['1234', '5678'],
+            'at_all': False,
+        }
+        expect = {
+            'msgtype': 'markdown',
+            'markdown': msg,
+            'at': {
+                'atMobiles': ['1234', '5678'],
+                'isAtAll': False
+            }
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_markdown_message_remind_all(self):
+        msg = {
+            'title': 'Airflow dingding markdown message',
+            'text': '# Markdown message title\ncontent content .. \n### sub-title\n'
+                    '![logo](http://airflow.apache.org/_images/pin_large.png)'
+        }
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'markdown',
+            'message': msg,
+            'at_all': True,
+        }
+        expect = {
+            'msgtype': 'markdown',
+            'markdown': msg,
+            'at': {
+                'atMobiles': None,
+                'isAtAll': True
+            }
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_link_message(self):
+        msg = {
+            'title': 'Airflow dingding link message',
+            'text': 'Airflow official documentation link',
+            'messageUrl': 'http://airflow.apache.org',
+            'picURL': 'http://airflow.apache.org/_images/pin_large.png'
+        }
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'link',
+            'message': msg
+        }
+        expect = {
+            'msgtype': 'link',
+            'link': msg
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_single_action_card_message(self):
+        msg = {
+            'title': 'Airflow dingding single actionCard message',
+            'text': 'Airflow dingding single actionCard message\n'
+                    '![logo](http://airflow.apache.org/_images/pin_large.png)\n'
+                    'This is a official logo in Airflow website.',
+            'hideAvatar': '0',
+            'btnOrientation': '0',
+            'singleTitle': 'read more',
+            'singleURL': 'http://airflow.apache.org'
+        }
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'actionCard',
+            'message': msg
+        }
+        expect = {
+            'msgtype': 'actionCard',
+            'actionCard': msg
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_multi_action_card_message(self):
+        msg = {
+            'title': 'Airflow dingding multi actionCard message',
+            'text': 'Airflow dingding multi actionCard message\n'
+                    '![logo](http://airflow.apache.org/_images/pin_large.png)\n'
+                    'Airflow documentation and github',
+            'hideAvatar': '0',
+            'btnOrientation': '0',
+            'btns': [
+                {
+                    'title': 'Airflow Documentation',
+                    'actionURL': 'http://airflow.apache.org'
+                },
+                {
+                    'title': 'Airflow Github',
+                    'actionURL': 'https://github.com/apache/airflow'
+                }
+            ]
+        }
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'actionCard',
+            'message': msg
+        }
+        expect = {
+            'msgtype': 'actionCard',
+            'actionCard': msg
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_build_feed_card_message(self):
+        msg = {
+            "links": [
+                {
+                    "title": "Airflow DAG feed card",
+                    "messageURL": "https://airflow.readthedocs.io/en/latest/ui.html",
+                    "picURL": "http://airflow.apache.org/_images/dags.png"
+                },
+                {
+                    "title": "Airflow tree feed card",
+                    "messageURL": "https://airflow.readthedocs.io/en/latest/ui.html",
+                    "picURL": "http://airflow.apache.org/_images/tree.png"
+                },
+                {
+                    "title": "Airflow graph feed card",
+                    "messageURL": "https://airflow.readthedocs.io/en/latest/ui.html",
+                    "picURL": "http://airflow.apache.org/_images/graph.png"
+                }
+            ]
+        }
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'feedCard',
+            'message': msg
+        }
+        expect = {
+            'msgtype': 'feedCard',
+            'feedCard': msg
+        }
+        hook = DingdingHook(**config)
+        message = hook._build_message()
+        self.assertEqual(json.dumps(expect), message)
+
+    def test_send_not_support_type(self):
+        config = {
+            'dingding_conn_id': self.conn_id,
+            'message_type': 'not_support_type',
+            'message': 'Airflow dingding text message remind no one'
+        }
+        hook = DingdingHook(**config)
+        self.assertRaises(ValueError, hook.send)
diff --git a/tests/contrib/hooks/test_discord_webhook_hook.py b/tests/contrib/hooks/test_discord_webhook_hook.py
index c92b198fc20e1..e41d88cf45b1f 100644
--- a/tests/contrib/hooks/test_discord_webhook_hook.py
+++ b/tests/contrib/hooks/test_discord_webhook_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,7 +20,8 @@
 import json
 import unittest
 
-from airflow import configuration, models, AirflowException
+from airflow import configuration, AirflowException
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 from airflow.contrib.hooks.discord_webhook_hook import DiscordWebhookHook
@@ -50,7 +51,7 @@ class TestDiscordWebhookHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='default-discord-webhook',
                 host='https://discordapp.com/api/',
                 extra='{"webhook_endpoint": "webhooks/00000/some-discord-token_000"}')
diff --git a/tests/contrib/hooks/test_emr_hook.py b/tests/contrib/hooks/test_emr_hook.py
index dc981ba093b3c..03f23f69062ca 100644
--- a/tests/contrib/hooks/test_emr_hook.py
+++ b/tests/contrib/hooks/test_emr_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,35 +24,56 @@
 from airflow import configuration
 from airflow.contrib.hooks.emr_hook import EmrHook
 
-
 try:
     from moto import mock_emr
 except ImportError:
     mock_emr = None
 
 
+@unittest.skipIf(mock_emr is None, 'moto package not present')
 class TestEmrHook(unittest.TestCase):
     @mock_emr
     def setUp(self):
         configuration.load_test_config()
 
-    @unittest.skipIf(mock_emr is None, 'mock_emr package not present')
     @mock_emr
     def test_get_conn_returns_a_boto3_connection(self):
-        hook = EmrHook(aws_conn_id='aws_default')
+        hook = EmrHook(aws_conn_id='aws_default', region_name='ap-southeast-2')
         self.assertIsNotNone(hook.get_conn().list_clusters())
 
-    @unittest.skipIf(mock_emr is None, 'mock_emr package not present')
     @mock_emr
     def test_create_job_flow_uses_the_emr_config_to_create_a_cluster(self):
         client = boto3.client('emr', region_name='us-east-1')
-        if len(client.list_clusters()['Clusters']):
-            raise ValueError('AWS not properly mocked')
 
         hook = EmrHook(aws_conn_id='aws_default', emr_conn_id='emr_default')
         cluster = hook.create_job_flow({'Name': 'test_cluster'})
 
-        self.assertEqual(client.list_clusters()['Clusters'][0]['Id'], cluster['JobFlowId'])
+        self.assertEqual(client.list_clusters()['Clusters'][0]['Id'],
+                         cluster['JobFlowId'])
+
+    @mock_emr
+    def test_create_job_flow_extra_args(self):
+        """
+        Test that we can add extra arguments to the launch call.
+
+        This is useful for when AWS add new options, such as
+        "SecurityConfiguration" so that we don't have to change our code
+        """
+        client = boto3.client('emr', region_name='us-east-1')
+
+        hook = EmrHook(aws_conn_id='aws_default', emr_conn_id='emr_default')
+        # AmiVersion is really old and almost no one will use it anymore, but
+        # it's one of the "optional" request params that moto supports - it's
+        # coverage of EMR isn't 100% it turns out.
+        cluster = hook.create_job_flow({'Name': 'test_cluster',
+                                        'ReleaseLabel': '',
+                                        'AmiVersion': '3.2'})
+
+        cluster = client.describe_cluster(ClusterId=cluster['JobFlowId'])['Cluster']
+
+        # The AmiVersion comes back as {Requested,Running}AmiVersion fields.
+        self.assertEqual(cluster['RequestedAmiVersion'], '3.2')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_ftp_hook.py b/tests/contrib/hooks/test_ftp_hook.py
index ac5db561c32f6..1274990827096 100644
--- a/tests/contrib/hooks/test_ftp_hook.py
+++ b/tests/contrib/hooks/test_ftp_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,6 +19,7 @@
 #
 
 import mock
+import six
 import unittest
 
 from airflow.contrib.hooks import ftp_hook as fh
@@ -82,25 +83,47 @@ def test_rename(self):
 
         self.conn_mock.rename.assert_called_once_with(from_path, to_path)
         self.conn_mock.quit.assert_called_once_with()
-        
+
     def test_mod_time(self):
         self.conn_mock.sendcmd.return_value = '213 20170428010138'
-        
+
         path = '/path/file'
         with fh.FTPHook() as ftp_hook:
             ftp_hook.get_mod_time(path)
 
         self.conn_mock.sendcmd.assert_called_once_with('MDTM ' + path)
-        
+
     def test_mod_time_micro(self):
         self.conn_mock.sendcmd.return_value = '213 20170428010138.003'
-        
+
         path = '/path/file'
         with fh.FTPHook() as ftp_hook:
             ftp_hook.get_mod_time(path)
 
         self.conn_mock.sendcmd.assert_called_once_with('MDTM ' + path)
 
+    def test_get_size(self):
+        self.conn_mock.size.return_value = 1942
+
+        path = '/path/file'
+        with fh.FTPHook() as ftp_hook:
+            ftp_hook.get_size(path)
+
+        self.conn_mock.size.assert_called_once_with(path)
+
+    def test_retrieve_file(self):
+        _buffer = six.StringIO('buffer')
+        with fh.FTPHook() as ftp_hook:
+            ftp_hook.retrieve_file(self.path, _buffer)
+        self.conn_mock.retrbinary.assert_called_once_with('RETR path', _buffer.write)
+
+    def test_retrieve_file_with_callback(self):
+        func = mock.Mock()
+        _buffer = six.StringIO('buffer')
+        with fh.FTPHook() as ftp_hook:
+            ftp_hook.retrieve_file(self.path, _buffer, callback=func)
+        self.conn_mock.retrbinary.assert_called_once_with('RETR path', func)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_gcp_api_base_hook.py b/tests/contrib/hooks/test_gcp_api_base_hook.py
index f4509d9111ff6..19123d1f01712 100644
--- a/tests/contrib/hooks/test_gcp_api_base_hook.py
+++ b/tests/contrib/hooks/test_gcp_api_base_hook.py
@@ -18,13 +18,29 @@
 # under the License.
 #
 
+import os
 import unittest
 
+from parameterized import parameterized
+from google.api_core.exceptions import RetryError, AlreadyExists
+from google.cloud.exceptions import MovedPermanently
+
+from airflow import AirflowException, LoggingMixin
+from googleapiclient.errors import HttpError
+
 from airflow.contrib.hooks import gcp_api_base_hook as hook
 
 import google.auth
 from google.auth.exceptions import GoogleAuthError
 
+from airflow.hooks.base_hook import BaseHook
+from tests.compat import mock
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 
 default_creds_available = True
 default_project = None
@@ -34,21 +50,84 @@
     default_creds_available = False
 
 
+class TestCatchHttpException(unittest.TestCase):
+    def test_no_exception(self):
+        self.called = False
+
+        class FixtureClass(LoggingMixin):
+            @hook.GoogleCloudBaseHook.catch_http_exception
+            def text_fixture(*args, **kwargs):
+                self.called = True
+
+        FixtureClass().text_fixture()
+
+        self.assertTrue(self.called)
+
+    @parameterized.expand(
+        [
+            (MovedPermanently("MESSAGE"),),
+            (RetryError("MESSAGE", cause=Exception("MESSAGE")),),
+            (ValueError("MESSAGE"),),
+        ]
+    )
+    def test_raise_airflowexception(self, ex_obj):
+        self.called = False
+
+        class FixtureClass(LoggingMixin):
+            @hook.GoogleCloudBaseHook.catch_http_exception
+            def test_fixutre(*args, **kwargs):
+                self.called = True
+                raise ex_obj
+
+        with self.assertRaises(AirflowException):
+            FixtureClass().test_fixutre()
+
+        self.assertTrue(self.called)
+
+    def test_raise_alreadyexists(self):
+        self.called = False
+
+        class FixtureClass(LoggingMixin):
+            @hook.GoogleCloudBaseHook.catch_http_exception
+            def test_fixutre(*args, **kwargs):
+                self.called = True
+                raise AlreadyExists("MESSAGE")
+
+        with self.assertRaises(AlreadyExists):
+            FixtureClass().test_fixutre()
+
+        self.assertTrue(self.called)
+
+    def test_raise_http_error(self):
+        self.called = False
+
+        class FixtureClass(BaseHook):
+            @hook.GoogleCloudBaseHook.catch_http_exception
+            def test_fixtue(*args, **kwargs):
+                self.called = True
+                raise HttpError(mock.Mock(**{"reason.return_value": None}), b"CONTENT")
+
+        with self.assertRaises(AirflowException):
+            FixtureClass(None).test_fixtue()
+
+        self.assertTrue(self.called)
+
+
 class TestGoogleCloudBaseHook(unittest.TestCase):
     def setUp(self):
         self.instance = hook.GoogleCloudBaseHook()
 
-    @unittest.skipIf(
-        not default_creds_available,
-        'Default GCP credentials not available to run tests')
+    @unittest.skipIf(not default_creds_available, 'Default GCP credentials not available to run tests')
     def test_default_creds_with_scopes(self):
         self.instance.extras = {
             'extra__google_cloud_platform__project': default_project,
             'extra__google_cloud_platform__scope': (
-                ','.join((
-                    'https://www.googleapis.com/auth/bigquery',
-                    'https://www.googleapis.com/auth/devstorage.read_only',
-                ))
+                ','.join(
+                    (
+                        'https://www.googleapis.com/auth/bigquery',
+                        'https://www.googleapis.com/auth/devstorage.read_only',
+                    )
+                )
             ),
         }
 
@@ -69,7 +148,7 @@ def test_default_creds_with_scopes(self):
         'Default GCP credentials not available to run tests')
     def test_default_creds_no_scopes(self):
         self.instance.extras = {
-            'extra__google_cloud_platform__project': default_project,
+            'extra__google_cloud_platform__project': default_project
         }
 
         credentials = self.instance._get_credentials()
@@ -81,3 +160,34 @@ def test_default_creds_no_scopes(self):
 
         scopes = credentials.scopes
         self.assertEqual(tuple(hook._DEFAULT_SCOPES), tuple(scopes))
+
+    def test_provide_gcp_credential_file_decorator_key_path(self):
+        key_path = '/test/key-path'
+        self.instance.extras = {'extra__google_cloud_platform__key_path': key_path}
+
+        @hook.GoogleCloudBaseHook._Decorators.provide_gcp_credential_file
+        def assert_gcp_credential_file_in_env(hook_instance):
+            self.assertEqual(os.environ[hook._G_APP_CRED_ENV_VAR],
+                             key_path)
+
+        assert_gcp_credential_file_in_env(self.instance)
+
+    @mock.patch('tempfile.NamedTemporaryFile')
+    def test_provide_gcp_credential_file_decorator_key_content(self, mock_file):
+        string_file = StringIO()
+        file_content = '{"foo": "bar"}'
+        file_name = '/test/mock-file'
+        self.instance.extras = {
+            'extra__google_cloud_platform__keyfile_dict': file_content
+        }
+        mock_file_handler = mock_file.return_value.__enter__.return_value
+        mock_file_handler.name = file_name
+        mock_file_handler.write = string_file.write
+
+        @hook.GoogleCloudBaseHook._Decorators.provide_gcp_credential_file
+        def assert_gcp_credential_file_in_env(hook_instance):
+            self.assertEqual(os.environ[hook._G_APP_CRED_ENV_VAR],
+                             file_name)
+            self.assertEqual(file_content, string_file.getvalue())
+
+        assert_gcp_credential_file_in_env(self.instance)
diff --git a/tests/contrib/hooks/test_gcp_bigtable_hook.py b/tests/contrib/hooks/test_gcp_bigtable_hook.py
new file mode 100644
index 0000000000000..876879d588352
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_bigtable_hook.py
@@ -0,0 +1,365 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import google
+
+import unittest
+
+from google.cloud.bigtable import Client
+from google.cloud.bigtable.instance import Instance
+
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id, \
+    mock_base_gcp_hook_default_project_id, GCP_PROJECT_ID_HOOK_UNIT_TEST
+from tests.compat import mock
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_bigtable_hook import BigtableHook
+
+CBT_INSTANCE = 'instance'
+CBT_CLUSTER = 'cluster'
+CBT_ZONE = 'zone'
+CBT_TABLE = 'table'
+
+
+class TestBigtableHookNoDefaultProjectId(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_no_default_project_id):
+            self.bigtable_hook_no_default_project_id = BigtableHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_get_instance_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        with self.assertRaises(AirflowException) as cm:
+            self.bigtable_hook_no_default_project_id.get_instance(instance_id=CBT_INSTANCE)
+        instance_exists_method.assert_not_called()
+        instance_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_get_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        res = self.bigtable_hook_no_default_project_id.get_instance(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='example-project')
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_instance_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        delete_method = instance_method.return_value.delete
+        instance_exists_method.return_value = True
+        with self.assertRaises(AirflowException) as cm:
+            self.bigtable_hook_no_default_project_id.delete_instance(instance_id=CBT_INSTANCE)
+        instance_exists_method.assert_not_called()
+        instance_method.assert_not_called()
+        delete_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        res = self.bigtable_hook_no_default_project_id.delete_instance(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST, instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        delete_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='example-project')
+        self.assertIsNone(res)
+
+    @mock.patch('google.cloud.bigtable.instance.Instance.create')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_create_instance_missing_project_id(self, get_client, instance_create):
+        operation = mock.Mock()
+        operation.result_return_value = Instance(instance_id=CBT_INSTANCE, client=get_client)
+        instance_create.return_value = operation
+        with self.assertRaises(AirflowException) as cm:
+            self.bigtable_hook_no_default_project_id.create_instance(
+                instance_id=CBT_INSTANCE,
+                main_cluster_id=CBT_CLUSTER,
+                main_cluster_zone=CBT_ZONE)
+        get_client.assert_not_called()
+        instance_create.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('google.cloud.bigtable.instance.Instance.create')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_create_instance_overridden_project_id(self, get_client, instance_create):
+        operation = mock.Mock()
+        operation.result_return_value = Instance(instance_id=CBT_INSTANCE, client=get_client)
+        instance_create.return_value = operation
+        res = self.bigtable_hook_no_default_project_id.create_instance(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=CBT_INSTANCE,
+            main_cluster_id=CBT_CLUSTER,
+            main_cluster_zone=CBT_ZONE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_create.assert_called_once_with(clusters=mock.ANY)
+        self.assertEqual(res.instance_id, 'instance')
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_table_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        table_delete_method = instance_method.return_value.table.return_value.delete
+        instance_exists_method.return_value = True
+        with self.assertRaises(AirflowException) as cm:
+            self.bigtable_hook_no_default_project_id.delete_table(
+                instance_id=CBT_INSTANCE,
+                table_id=CBT_TABLE)
+        get_client.assert_not_called()
+        instance_exists_method.assert_not_called()
+        table_delete_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_table_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        table_delete_method = instance_method.return_value.table.return_value.delete
+        instance_exists_method.return_value = True
+        self.bigtable_hook_no_default_project_id.delete_table(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=CBT_INSTANCE,
+            table_id=CBT_TABLE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_exists_method.assert_called_once_with()
+        table_delete_method.assert_called_once_with()
+
+
+class TestBigtableHookDefaultProjectId(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_default_project_id):
+            self.bigtable_hook_default_project_id = BigtableHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_get_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        res = self.bigtable_hook_default_project_id.get_instance(
+            instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='example-project')
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_get_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        res = self.bigtable_hook_default_project_id.get_instance(
+            project_id='new-project',
+            instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='new-project')
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_get_instance_no_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = False
+        res = self.bigtable_hook_default_project_id.get_instance(
+            instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='example-project')
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        res = self.bigtable_hook_default_project_id.delete_instance(
+            instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        delete_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='example-project')
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        res = self.bigtable_hook_default_project_id.delete_instance(
+            project_id='new-project', instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        delete_method.assert_called_once_with()
+        get_client.assert_called_once_with(project_id='new-project')
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_instance_no_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = False
+        delete_method = instance_method.return_value.delete
+        self.bigtable_hook_default_project_id.delete_instance(
+            instance_id=CBT_INSTANCE)
+        instance_method.assert_called_once_with('instance')
+        instance_exists_method.assert_called_once_with()
+        delete_method.assert_not_called()
+        get_client.assert_called_once_with(project_id='example-project')
+
+    @mock.patch('google.cloud.bigtable.instance.Instance.create')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_create_instance(self, get_client, instance_create):
+        operation = mock.Mock()
+        operation.result_return_value = Instance(instance_id=CBT_INSTANCE, client=get_client)
+        instance_create.return_value = operation
+        res = self.bigtable_hook_default_project_id.create_instance(
+            instance_id=CBT_INSTANCE,
+            main_cluster_id=CBT_CLUSTER,
+            main_cluster_zone=CBT_ZONE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_create.assert_called_once_with(clusters=mock.ANY)
+        self.assertEqual(res.instance_id, 'instance')
+
+    @mock.patch('google.cloud.bigtable.instance.Instance.create')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_create_instance_overridden_project_id(self, get_client, instance_create):
+        operation = mock.Mock()
+        operation.result_return_value = Instance(instance_id=CBT_INSTANCE, client=get_client)
+        instance_create.return_value = operation
+        res = self.bigtable_hook_default_project_id.create_instance(
+            project_id='new-project',
+            instance_id=CBT_INSTANCE,
+            main_cluster_id=CBT_CLUSTER,
+            main_cluster_zone=CBT_ZONE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_create.assert_called_once_with(clusters=mock.ANY)
+        self.assertEqual(res.instance_id, 'instance')
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_table(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        table_delete_method = instance_method.return_value.table.return_value.delete
+        instance_exists_method.return_value = True
+        self.bigtable_hook_default_project_id.delete_table(
+            instance_id=CBT_INSTANCE,
+            table_id=CBT_TABLE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_exists_method.assert_called_once_with()
+        table_delete_method.assert_called_once_with()
+
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_delete_table_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        table_delete_method = instance_method.return_value.table.return_value.delete
+        instance_exists_method.return_value = True
+        self.bigtable_hook_default_project_id.delete_table(
+            project_id='new-project',
+            instance_id=CBT_INSTANCE,
+            table_id=CBT_TABLE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_exists_method.assert_called_once_with()
+        table_delete_method.assert_called_once_with()
+
+    @mock.patch('google.cloud.bigtable.table.Table.create')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_create_table(self, get_client, create):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        client = mock.Mock(Client)
+        instance = google.cloud.bigtable.instance.Instance(
+            instance_id=CBT_INSTANCE,
+            client=client)
+        self.bigtable_hook_default_project_id.create_table(
+            instance=instance,
+            table_id=CBT_TABLE)
+        get_client.assert_not_called()
+        create.assert_called_once_with([], {})
+
+    @mock.patch('google.cloud.bigtable.cluster.Cluster.update')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_update_cluster(self, get_client, update):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        client = mock.Mock(Client)
+        instance = google.cloud.bigtable.instance.Instance(
+            instance_id=CBT_INSTANCE,
+            client=client)
+        self.bigtable_hook_default_project_id.update_cluster(
+            instance=instance,
+            cluster_id=CBT_CLUSTER,
+            nodes=4)
+        get_client.assert_not_called()
+        update.assert_called_once_with()
+
+    @mock.patch('google.cloud.bigtable.table.Table.list_column_families')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_list_column_families(self, get_client, list_column_families):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        client = mock.Mock(Client)
+        get_client.return_value = client
+        instance = google.cloud.bigtable.instance.Instance(
+            instance_id=CBT_INSTANCE,
+            client=client)
+        self.bigtable_hook_default_project_id.get_column_families_for_table(
+            instance=instance, table_id=CBT_TABLE)
+        get_client.assert_not_called()
+        list_column_families.assert_called_once_with()
+
+    @mock.patch('google.cloud.bigtable.table.Table.get_cluster_states')
+    @mock.patch('airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook._get_client')
+    def test_get_cluster_states(self, get_client, get_cluster_states):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        client = mock.Mock(Client)
+        instance = google.cloud.bigtable.instance.Instance(
+            instance_id=CBT_INSTANCE,
+            client=client)
+        self.bigtable_hook_default_project_id.get_cluster_states_for_table(
+            instance=instance, table_id=CBT_TABLE)
+        get_client.assert_not_called()
+        get_cluster_states.assert_called_once_with()
diff --git a/tests/contrib/hooks/test_gcp_compute_hook.py b/tests/contrib/hooks/test_gcp_compute_hook.py
new file mode 100644
index 0000000000000..76af96fe40ad2
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_compute_hook.py
@@ -0,0 +1,569 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id, \
+    mock_base_gcp_hook_default_project_id, GCP_PROJECT_ID_HOOK_UNIT_TEST
+from tests.compat import mock
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_compute_hook import GceHook
+
+GCE_ZONE = 'zone'
+GCE_INSTANCE = 'instance'
+GCE_INSTANCE_TEMPLATE = 'instance-template'
+GCE_REQUEST_ID = 'request_id'
+GCE_INSTANCE_GROUP_MANAGER = 'instance_group_manager'
+
+
+class TestGcpComputeHookNoDefaultProjectId(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_no_default_project_id):
+            self.gce_hook_no_project_id = GceHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_start_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        start_method = get_conn.return_value.instances.return_value.start
+        execute_method = start_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.start_instance(
+            project_id='example-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        start_method.assert_called_once_with(instance='instance', project='example-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_start_instance_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        start_method = get_conn.return_value.instances.return_value.start
+        execute_method = start_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.start_instance(
+                zone=GCE_ZONE,
+                resource_id=GCE_INSTANCE)
+        start_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_stop_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        stop_method = get_conn.return_value.instances.return_value.stop
+        execute_method = stop_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.stop_instance(
+            project_id='example-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        stop_method.assert_called_once_with(instance='instance', project='example-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_stop_instance_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        stop_method = get_conn.return_value.instances.return_value.stop
+        execute_method = stop_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.stop_instance(
+                zone=GCE_ZONE,
+                resource_id=GCE_INSTANCE)
+        stop_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_set_machine_type_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        set_machine_type_method = get_conn.return_value.instances.return_value.setMachineType
+        execute_method = set_machine_type_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.set_machine_type(
+            body={},
+            project_id='example-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        set_machine_type_method.assert_called_once_with(body={}, instance='instance',
+                                                        project='example-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_set_machine_type_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        set_machine_type_method = get_conn.return_value.instances.return_value.setMachineType
+        execute_method = set_machine_type_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.set_machine_type(
+                body={},
+                zone=GCE_ZONE,
+                resource_id=GCE_INSTANCE)
+        set_machine_type_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_template_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceTemplates.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.get_instance_template(
+            resource_id=GCE_INSTANCE_TEMPLATE,
+            project_id='example-project'
+        )
+        self.assertIsNotNone(res)
+        get_method.assert_called_once_with(instanceTemplate='instance-template', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_template_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceTemplates.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.get_instance_template(
+                resource_id=GCE_INSTANCE_TEMPLATE
+            )
+        get_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_insert_instance_template_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instanceTemplates.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.insert_instance_template(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            body={},
+            request_id=GCE_REQUEST_ID
+        )
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, project='example-project', requestId='request_id')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_insert_instance_template_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instanceTemplates.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.insert_instance_template(
+                body={},
+                request_id=GCE_REQUEST_ID
+            )
+        insert_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_group_manager_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceGroupManagers.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.get_instance_group_manager(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER
+        )
+        self.assertIsNotNone(res)
+        get_method.assert_called_once_with(instanceGroupManager='instance_group_manager',
+                                           project='example-project',
+                                           zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_group_manager_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceGroupManagers.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.get_instance_group_manager(
+                zone=GCE_ZONE,
+                resource_id=GCE_INSTANCE_GROUP_MANAGER
+            )
+        get_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_patch_instance_group_manager_overridden_project_id(self,
+                                                                wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instanceGroupManagers.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook_no_project_id.patch_instance_group_manager(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER,
+            body={},
+            request_id=GCE_REQUEST_ID
+        )
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(
+            body={},
+            instanceGroupManager='instance_group_manager',
+            project='example-project',
+            requestId='request_id',
+            zone='zone'
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id',
+                                                               project_id='example-project',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_patch_instance_group_manager_no_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instanceGroupManagers.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gce_hook_no_project_id.patch_instance_group_manager(
+                zone=GCE_ZONE,
+                resource_id=GCE_INSTANCE_GROUP_MANAGER,
+                body={},
+                request_id=GCE_REQUEST_ID
+            )
+        patch_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+
+class TestGcpComputeHookDefaultProjectId(unittest.TestCase):
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_default_project_id):
+            self.gce_hook = GceHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_start_instance(self, wait_for_operation_to_complete, get_conn):
+        start_method = get_conn.return_value.instances.return_value.start
+        execute_method = start_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.start_instance(
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        start_method.assert_called_once_with(instance='instance', project='example-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_start_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        start_method = get_conn.return_value.instances.return_value.start
+        execute_method = start_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.start_instance(
+            project_id='new-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        start_method.assert_called_once_with(instance='instance', project='new-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='new-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_stop_instance(self, wait_for_operation_to_complete, get_conn):
+        stop_method = get_conn.return_value.instances.return_value.stop
+        execute_method = stop_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.stop_instance(
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        stop_method.assert_called_once_with(instance='instance', project='example-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_stop_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        stop_method = get_conn.return_value.instances.return_value.stop
+        execute_method = stop_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.stop_instance(
+            project_id='new-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        stop_method.assert_called_once_with(instance='instance', project='new-project', zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='new-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_set_machine_type_instance(self, wait_for_operation_to_complete, get_conn):
+        execute_method = get_conn.return_value.instances.return_value.setMachineType.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.set_machine_type(
+            body={},
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_set_machine_type_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        execute_method = get_conn.return_value.instances.return_value.setMachineType.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.set_machine_type(
+            project_id='new-project',
+            body={},
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE)
+        self.assertIsNone(res)
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='new-project',
+                                                               operation_name='operation_id',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_template(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceTemplates.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.get_instance_template(
+            resource_id=GCE_INSTANCE_TEMPLATE)
+        self.assertIsNotNone(res)
+        get_method.assert_called_once_with(instanceTemplate='instance-template', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_template_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceTemplates.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.get_instance_template(
+            project_id='new-project',
+            resource_id=GCE_INSTANCE_TEMPLATE)
+        self.assertIsNotNone(res)
+        get_method.assert_called_once_with(instanceTemplate='instance-template', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_insert_instance_template(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instanceTemplates.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.insert_instance_template(
+            body={},
+            request_id=GCE_REQUEST_ID
+        )
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, project='example-project', requestId='request_id')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_insert_instance_template_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instanceTemplates.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.insert_instance_template(
+            project_id='new-project',
+            body={},
+            request_id=GCE_REQUEST_ID
+        )
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, project='new-project', requestId='request_id')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='new-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_group_manager(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceGroupManagers.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.get_instance_group_manager(
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER
+        )
+        self.assertIsNotNone(res)
+        get_method.assert_called_once_with(instanceGroupManager='instance_group_manager',
+                                           project='example-project',
+                                           zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_get_instance_group_manager_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instanceGroupManagers.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.get_instance_group_manager(
+            project_id='new-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER
+        )
+        self.assertIsNotNone(res)
+        get_method.assert_called_once_with(instanceGroupManager='instance_group_manager',
+                                           project='new-project',
+                                           zone='zone')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_patch_instance_group_manager(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instanceGroupManagers.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.patch_instance_group_manager(
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER,
+            body={},
+            request_id=GCE_REQUEST_ID
+        )
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(
+            body={},
+            instanceGroupManager='instance_group_manager',
+            project='example-project',
+            requestId='request_id',
+            zone='zone'
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id',
+                                                               project_id='example-project',
+                                                               zone='zone')
+
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_compute_hook.GceHook._wait_for_operation_to_complete')
+    def test_patch_instance_group_manager_overridden_project_id(self,
+                                                                wait_for_operation_to_complete,
+                                                                get_conn):
+        patch_method = get_conn.return_value.instanceGroupManagers.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gce_hook.patch_instance_group_manager(
+            project_id='new-project',
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER,
+            body={},
+            request_id=GCE_REQUEST_ID
+        )
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(
+            body={},
+            instanceGroupManager='instance_group_manager',
+            project='new-project',
+            requestId='request_id',
+            zone='zone'
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id',
+                                                               project_id='new-project',
+                                                               zone='zone')
diff --git a/tests/contrib/hooks/test_gcp_container_hook.py b/tests/contrib/hooks/test_gcp_container_hook.py
index f3705ea4ce7d4..f4687cfa041a0 100644
--- a/tests/contrib/hooks/test_gcp_container_hook.py
+++ b/tests/contrib/hooks/test_gcp_container_hook.py
@@ -21,28 +21,19 @@
 
 from airflow import AirflowException
 from airflow.contrib.hooks.gcp_container_hook import GKEClusterHook
+from tests.compat import mock
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
 
 TASK_ID = 'test-gke-cluster-operator'
 CLUSTER_NAME = 'test-cluster'
-TEST_PROJECT_ID = 'test-project'
-ZONE = 'test-zone'
+TEST_GCP_PROJECT_ID = 'test-project'
+GKE_ZONE = 'test-zone'
 
 
 class GKEClusterHookDeleteTest(unittest.TestCase):
     def setUp(self):
-        with mock.patch.object(GKEClusterHook, "__init__", return_value=None):
-            self.gke_hook = GKEClusterHook(None, None, None)
-            self.gke_hook.project_id = TEST_PROJECT_ID
-            self.gke_hook.location = ZONE
-            self.gke_hook.client = mock.Mock()
+        self.gke_hook = GKEClusterHook(location=GKE_ZONE)
+        self.gke_hook._client = mock.Mock()
 
     @mock.patch("airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._dict_to_proto")
     @mock.patch(
@@ -50,37 +41,53 @@ def setUp(self):
     def test_delete_cluster(self, wait_mock, convert_mock):
         retry_mock, timeout_mock = mock.Mock(), mock.Mock()
 
-        client_delete = self.gke_hook.client.delete_cluster = mock.Mock()
+        client_delete = self.gke_hook._client.delete_cluster = mock.Mock()
 
-        self.gke_hook.delete_cluster(name=CLUSTER_NAME, retry=retry_mock,
+        self.gke_hook.delete_cluster(name=CLUSTER_NAME, project_id=TEST_GCP_PROJECT_ID,
+                                     retry=retry_mock,
                                      timeout=timeout_mock)
 
-        client_delete.assert_called_with(project_id=TEST_PROJECT_ID, zone=ZONE,
+        client_delete.assert_called_with(project_id=TEST_GCP_PROJECT_ID,
+                                         zone=GKE_ZONE,
                                          cluster_id=CLUSTER_NAME,
-                                         retry=retry_mock, timeout=timeout_mock)
+                                         retry=retry_mock,
+                                         timeout=timeout_mock)
         wait_mock.assert_called_with(client_delete.return_value)
         convert_mock.assert_not_called()
 
+    @mock.patch(
+        "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.log")
+    @mock.patch("airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._dict_to_proto")
+    @mock.patch(
+        "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.wait_for_operation")
+    def test_delete_cluster_not_found(self, wait_mock, convert_mock, log_mock):
+        from google.api_core.exceptions import NotFound
+        # To force an error
+        message = 'Not Found'
+        self.gke_hook._client.delete_cluster.side_effect = NotFound(message=message)
+
+        self.gke_hook.delete_cluster('not-existing')
+        wait_mock.assert_not_called()
+        convert_mock.assert_not_called()
+        log_mock.info.assert_any_call("Assuming Success: %s", message)
+
     @mock.patch("airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._dict_to_proto")
     @mock.patch(
         "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.wait_for_operation")
     def test_delete_cluster_error(self, wait_mock, convert_mock):
         # To force an error
-        self.gke_hook.client.delete_cluster.side_effect = AirflowException('400')
+        self.gke_hook._client.delete_cluster.side_effect = AirflowException('400')
 
         with self.assertRaises(AirflowException):
-            self.gke_hook.delete_cluster(None)
+            self.gke_hook.delete_cluster('a-cluster')
             wait_mock.assert_not_called()
             convert_mock.assert_not_called()
 
 
 class GKEClusterHookCreateTest(unittest.TestCase):
     def setUp(self):
-        with mock.patch.object(GKEClusterHook, "__init__", return_value=None):
-            self.gke_hook = GKEClusterHook(None, None, None)
-            self.gke_hook.project_id = TEST_PROJECT_ID
-            self.gke_hook.location = ZONE
-            self.gke_hook.client = mock.Mock()
+        self.gke_hook = GKEClusterHook(location=GKE_ZONE)
+        self.gke_hook._client = mock.Mock()
 
     @mock.patch("airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._dict_to_proto")
     @mock.patch(
@@ -93,12 +100,15 @@ def test_create_cluster_proto(self, wait_mock, convert_mock):
 
         retry_mock, timeout_mock = mock.Mock(), mock.Mock()
 
-        client_create = self.gke_hook.client.create_cluster = mock.Mock()
+        client_create = self.gke_hook._client.create_cluster = mock.Mock()
 
-        self.gke_hook.create_cluster(mock_cluster_proto, retry=retry_mock,
+        self.gke_hook.create_cluster(mock_cluster_proto,
+                                     project_id=TEST_GCP_PROJECT_ID,
+                                     retry=retry_mock,
                                      timeout=timeout_mock)
 
-        client_create.assert_called_with(project_id=TEST_PROJECT_ID, zone=ZONE,
+        client_create.assert_called_with(project_id=TEST_GCP_PROJECT_ID,
+                                         zone=GKE_ZONE,
                                          cluster=mock_cluster_proto,
                                          retry=retry_mock, timeout=timeout_mock)
         wait_mock.assert_called_with(client_create.return_value)
@@ -107,17 +117,20 @@ def test_create_cluster_proto(self, wait_mock, convert_mock):
     @mock.patch("airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._dict_to_proto")
     @mock.patch(
         "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.wait_for_operation")
-    def test_delete_cluster_dict(self, wait_mock, convert_mock):
+    def test_create_cluster_dict(self, wait_mock, convert_mock):
         mock_cluster_dict = {'name': CLUSTER_NAME}
         retry_mock, timeout_mock = mock.Mock(), mock.Mock()
 
-        client_create = self.gke_hook.client.create_cluster = mock.Mock()
+        client_create = self.gke_hook._client.create_cluster = mock.Mock()
         proto_mock = convert_mock.return_value = mock.Mock()
 
-        self.gke_hook.create_cluster(mock_cluster_dict, retry=retry_mock,
+        self.gke_hook.create_cluster(mock_cluster_dict,
+                                     project_id=TEST_GCP_PROJECT_ID,
+                                     retry=retry_mock,
                                      timeout=timeout_mock)
 
-        client_create.assert_called_with(project_id=TEST_PROJECT_ID, zone=ZONE,
+        client_create.assert_called_with(project_id=TEST_GCP_PROJECT_ID,
+                                         zone=GKE_ZONE,
                                          cluster=proto_mock,
                                          retry=retry_mock, timeout=timeout_mock)
         wait_mock.assert_called_with(client_create.return_value)
@@ -135,24 +148,40 @@ def test_create_cluster_error(self, wait_mock, convert_mock):
             wait_mock.assert_not_called()
             convert_mock.assert_not_called()
 
+    @mock.patch(
+        "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.log")
+    @mock.patch("airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._dict_to_proto")
+    @mock.patch(
+        "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook.wait_for_operation")
+    def test_create_cluster_already_exists(self, wait_mock, convert_mock, log_mock):
+        from google.api_core.exceptions import AlreadyExists
+        # To force an error
+        message = 'Already Exists'
+        self.gke_hook._client.create_cluster.side_effect = AlreadyExists(message=message)
+
+        self.gke_hook.create_cluster({})
+        wait_mock.assert_not_called()
+        self.assertEqual(convert_mock.call_count, 1)
+        log_mock.info.assert_any_call("Assuming Success: %s", message)
+
 
 class GKEClusterHookGetTest(unittest.TestCase):
     def setUp(self):
-        with mock.patch.object(GKEClusterHook, "__init__", return_value=None):
-            self.gke_hook = GKEClusterHook(None, None, None)
-            self.gke_hook.project_id = TEST_PROJECT_ID
-            self.gke_hook.location = ZONE
-            self.gke_hook.client = mock.Mock()
+        self.gke_hook = GKEClusterHook(location=GKE_ZONE)
+        self.gke_hook._client = mock.Mock()
 
     def test_get_cluster(self):
         retry_mock, timeout_mock = mock.Mock(), mock.Mock()
 
-        client_get = self.gke_hook.client.get_cluster = mock.Mock()
+        client_get = self.gke_hook._client.get_cluster = mock.Mock()
 
-        self.gke_hook.get_cluster(name=CLUSTER_NAME, retry=retry_mock,
+        self.gke_hook.get_cluster(name=CLUSTER_NAME,
+                                  project_id=TEST_GCP_PROJECT_ID,
+                                  retry=retry_mock,
                                   timeout=timeout_mock)
 
-        client_get.assert_called_with(project_id=TEST_PROJECT_ID, zone=ZONE,
+        client_get.assert_called_with(project_id=TEST_GCP_PROJECT_ID,
+                                      zone=GKE_ZONE,
                                       cluster_id=CLUSTER_NAME,
                                       retry=retry_mock, timeout=timeout_mock)
 
@@ -160,18 +189,26 @@ def test_get_cluster(self):
 class GKEClusterHookTest(unittest.TestCase):
 
     def setUp(self):
-        with mock.patch.object(GKEClusterHook, "__init__", return_value=None):
-            self.gke_hook = GKEClusterHook(None, None, None)
-            self.gke_hook.project_id = TEST_PROJECT_ID
-            self.gke_hook.location = ZONE
-            self.gke_hook.client = mock.Mock()
+        self.gke_hook = GKEClusterHook(location=GKE_ZONE)
+        self.gke_hook._client = mock.Mock()
+
+    @mock.patch('airflow.contrib.hooks.gcp_container_hook.container_v1.'
+                'ClusterManagerClient')
+    @mock.patch('airflow.contrib.hooks.gcp_container_hook.ClientInfo')
+    @mock.patch('airflow.contrib.hooks.gcp_container_hook.GKEClusterHook._get_credentials')
+    def test_get_client(self, mock_get_credentials, mock_client_info, mock_client):
+        self.gke_hook._client = None
+        self.gke_hook.get_client()
+        assert mock_get_credentials.called
+        mock_client.assert_called_with(
+            credentials=mock_get_credentials.return_value,
+            client_info=mock_client_info.return_value)
 
     def test_get_operation(self):
-        self.gke_hook.client.get_operation = mock.Mock()
-        self.gke_hook.get_operation('TEST_OP')
-        self.gke_hook.client.get_operation.assert_called_with(project_id=TEST_PROJECT_ID,
-                                                              zone=ZONE,
-                                                              operation_id='TEST_OP')
+        self.gke_hook._client.get_operation = mock.Mock()
+        self.gke_hook.get_operation('TEST_OP', project_id=TEST_GCP_PROJECT_ID)
+        self.gke_hook._client.get_operation.assert_called_with(
+            project_id=TEST_GCP_PROJECT_ID, zone=GKE_ZONE, operation_id='TEST_OP')
 
     def test_append_label(self):
         key = 'test-key'
@@ -219,11 +256,11 @@ def test_wait_for_response_running(self, time_mock, operation_mock):
 
         # Status goes from Running -> Pending -> Done
         operation_mock.side_effect = [pending_op, done_op]
-        self.gke_hook.wait_for_operation(running_op)
+        self.gke_hook.wait_for_operation(running_op, project_id=TEST_GCP_PROJECT_ID)
 
         self.assertEqual(time_mock.call_count, 3)
-        operation_mock.assert_any_call(running_op.name)
-        operation_mock.assert_any_call(pending_op.name)
+        operation_mock.assert_any_call(running_op.name, project_id=TEST_GCP_PROJECT_ID)
+        operation_mock.assert_any_call(pending_op.name, project_id=TEST_GCP_PROJECT_ID)
         self.assertEqual(operation_mock.call_count, 2)
 
     @mock.patch("google.protobuf.json_format.Parse")
diff --git a/tests/contrib/hooks/test_gcp_dataflow_hook.py b/tests/contrib/hooks/test_gcp_dataflow_hook.py
index f16dcdfcbc1e1..7e97ea94d3f2e 100644
--- a/tests/contrib/hooks/test_gcp_dataflow_hook.py
+++ b/tests/contrib/hooks/test_gcp_dataflow_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,22 +19,14 @@
 #
 
 import unittest
-from mock import call
-from mock import MagicMock
 
-from airflow.contrib.hooks.gcp_dataflow_hook import DataFlowHook
-from airflow.contrib.hooks.gcp_dataflow_hook import _Dataflow
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from airflow.contrib.hooks.gcp_dataflow_hook import DataFlowHook,\
+    _Dataflow, _DataflowJob
+from tests.compat import mock, MagicMock
 
 
 TASK_ID = 'test-dataflow-operator'
+JOB_NAME = 'test-dataflow-pipeline'
 TEMPLATE = 'gs://dataflow-templates/wordcount/template_file'
 PARAMETERS = {
     'inputFile': 'gs://dataflow-samples/shakespeare/kinglear.txt',
@@ -62,6 +54,10 @@
 BASE_STRING = 'airflow.contrib.hooks.gcp_api_base_hook.{}'
 DATAFLOW_STRING = 'airflow.contrib.hooks.gcp_dataflow_hook.{}'
 MOCK_UUID = '12345678'
+TEST_PROJECT = 'test-project'
+TEST_JOB_NAME = 'test-job-name'
+TEST_JOB_ID = 'test-job-id'
+TEST_LOCATION = 'us-central1'
 
 
 def mock_init(self, gcp_conn_id, delegate_to=None):
@@ -75,7 +71,7 @@ def setUp(self):
                         new=mock_init):
             self.dataflow_hook = DataFlowHook(gcp_conn_id='test')
 
-    @mock.patch(DATAFLOW_STRING.format('uuid.uuid1'))
+    @mock.patch(DATAFLOW_STRING.format('uuid.uuid4'))
     @mock.patch(DATAFLOW_STRING.format('_DataflowJob'))
     @mock.patch(DATAFLOW_STRING.format('_Dataflow'))
     @mock.patch(DATAFLOW_STRING.format('DataFlowHook.get_conn'))
@@ -88,18 +84,18 @@ def test_start_python_dataflow(self, mock_conn,
         dataflowjob_instance = mock_dataflowjob.return_value
         dataflowjob_instance.wait_for_done.return_value = None
         self.dataflow_hook.start_python_dataflow(
-            task_id=TASK_ID, variables=DATAFLOW_OPTIONS_PY,
+            job_name=JOB_NAME, variables=DATAFLOW_OPTIONS_PY,
             dataflow=PY_FILE, py_options=PY_OPTIONS)
-        EXPECTED_CMD = ['python', '-m', PY_FILE,
+        EXPECTED_CMD = ['python2', '-m', PY_FILE,
                         '--region=us-central1',
                         '--runner=DataflowRunner', '--project=test',
                         '--labels=foo=bar',
                         '--staging_location=gs://test/staging',
-                        '--job_name={}-{}'.format(TASK_ID, MOCK_UUID)]
+                        '--job_name={}-{}'.format(JOB_NAME, MOCK_UUID)]
         self.assertListEqual(sorted(mock_dataflow.call_args[0][0]),
                              sorted(EXPECTED_CMD))
 
-    @mock.patch(DATAFLOW_STRING.format('uuid.uuid1'))
+    @mock.patch(DATAFLOW_STRING.format('uuid.uuid4'))
     @mock.patch(DATAFLOW_STRING.format('_DataflowJob'))
     @mock.patch(DATAFLOW_STRING.format('_Dataflow'))
     @mock.patch(DATAFLOW_STRING.format('DataFlowHook.get_conn'))
@@ -112,18 +108,18 @@ def test_start_java_dataflow(self, mock_conn,
         dataflowjob_instance = mock_dataflowjob.return_value
         dataflowjob_instance.wait_for_done.return_value = None
         self.dataflow_hook.start_java_dataflow(
-            task_id=TASK_ID, variables=DATAFLOW_OPTIONS_JAVA,
+            job_name=JOB_NAME, variables=DATAFLOW_OPTIONS_JAVA,
             dataflow=JAR_FILE)
         EXPECTED_CMD = ['java', '-jar', JAR_FILE,
                         '--region=us-central1',
                         '--runner=DataflowRunner', '--project=test',
                         '--stagingLocation=gs://test/staging',
                         '--labels={"foo":"bar"}',
-                        '--jobName={}-{}'.format(TASK_ID, MOCK_UUID)]
+                        '--jobName={}-{}'.format(JOB_NAME, MOCK_UUID)]
         self.assertListEqual(sorted(mock_dataflow.call_args[0][0]),
                              sorted(EXPECTED_CMD))
 
-    @mock.patch(DATAFLOW_STRING.format('uuid.uuid1'))
+    @mock.patch(DATAFLOW_STRING.format('uuid.uuid4'))
     @mock.patch(DATAFLOW_STRING.format('_DataflowJob'))
     @mock.patch(DATAFLOW_STRING.format('_Dataflow'))
     @mock.patch(DATAFLOW_STRING.format('DataFlowHook.get_conn'))
@@ -136,41 +132,107 @@ def test_start_java_dataflow_with_job_class(
         dataflowjob_instance = mock_dataflowjob.return_value
         dataflowjob_instance.wait_for_done.return_value = None
         self.dataflow_hook.start_java_dataflow(
-            task_id=TASK_ID, variables=DATAFLOW_OPTIONS_JAVA,
+            job_name=JOB_NAME, variables=DATAFLOW_OPTIONS_JAVA,
             dataflow=JAR_FILE, job_class=JOB_CLASS)
         EXPECTED_CMD = ['java', '-cp', JAR_FILE, JOB_CLASS,
                         '--region=us-central1',
                         '--runner=DataflowRunner', '--project=test',
                         '--stagingLocation=gs://test/staging',
                         '--labels={"foo":"bar"}',
-                        '--jobName={}-{}'.format(TASK_ID, MOCK_UUID)]
+                        '--jobName={}-{}'.format(JOB_NAME, MOCK_UUID)]
         self.assertListEqual(sorted(mock_dataflow.call_args[0][0]),
                              sorted(EXPECTED_CMD))
 
-
     @mock.patch('airflow.contrib.hooks.gcp_dataflow_hook._Dataflow.log')
     @mock.patch('subprocess.Popen')
     @mock.patch('select.select')
     def test_dataflow_wait_for_done_logging(self, mock_select, mock_popen, mock_logging):
-      mock_logging.info = MagicMock()
-      mock_logging.warning = MagicMock()
-      mock_proc = MagicMock()
-      mock_proc.stderr = MagicMock()
-      mock_proc.stderr.readlines = MagicMock(return_value=['test\n','error\n'])
-      mock_stderr_fd = MagicMock()
-      mock_proc.stderr.fileno = MagicMock(return_value=mock_stderr_fd)
-      mock_proc_poll = MagicMock()
-      mock_select.return_value = [[mock_stderr_fd]]
-      def poll_resp_error():
-        mock_proc.return_code = 1
-        return True
-      mock_proc_poll.side_effect=[None, poll_resp_error]
-      mock_proc.poll = mock_proc_poll
-      mock_popen.return_value = mock_proc
-      dataflow = _Dataflow(['test', 'cmd'])
-      mock_logging.info.assert_called_with('Running command: %s', 'test cmd')
-      self.assertRaises(Exception, dataflow.wait_for_done)
-      mock_logging.warning.assert_has_calls([call('test'), call('error')])
+        mock_logging.info = MagicMock()
+        mock_logging.warning = MagicMock()
+        mock_proc = MagicMock()
+        mock_proc.stderr = MagicMock()
+        mock_proc.stderr.readlines = MagicMock(return_value=['test\n', 'error\n'])
+        mock_stderr_fd = MagicMock()
+        mock_proc.stderr.fileno = MagicMock(return_value=mock_stderr_fd)
+        mock_proc_poll = MagicMock()
+        mock_select.return_value = [[mock_stderr_fd]]
+
+        def poll_resp_error():
+            mock_proc.return_code = 1
+            return True
+        mock_proc_poll.side_effect = [None, poll_resp_error]
+        mock_proc.poll = mock_proc_poll
+        mock_popen.return_value = mock_proc
+        dataflow = _Dataflow(['test', 'cmd'])
+        mock_logging.info.assert_called_with('Running command: %s', 'test cmd')
+        self.assertRaises(Exception, dataflow.wait_for_done)
+
+    def test_valid_dataflow_job_name(self):
+        job_name = self.dataflow_hook._build_dataflow_job_name(
+            job_name=JOB_NAME, append_job_name=False
+        )
+
+        self.assertEqual(job_name, JOB_NAME)
+
+    def test_fix_underscore_in_job_name(self):
+        job_name_with_underscore = 'test_example'
+        fixed_job_name = job_name_with_underscore.replace(
+            '_', '-'
+        )
+        job_name = self.dataflow_hook._build_dataflow_job_name(
+            job_name=job_name_with_underscore, append_job_name=False
+        )
+
+        self.assertEqual(job_name, fixed_job_name)
+
+    def test_invalid_dataflow_job_name(self):
+        invalid_job_name = '9test_invalid_name'
+        fixed_name = invalid_job_name.replace(
+            '_', '-')
+
+        with self.assertRaises(ValueError) as e:
+            self.dataflow_hook._build_dataflow_job_name(
+                job_name=invalid_job_name, append_job_name=False
+            )
+        #   Test whether the job_name is present in the Error msg
+        self.assertIn('Invalid job_name ({})'.format(fixed_name),
+                      str(e.exception))
+
+    def test_dataflow_job_regex_check(self):
+
+        self.assertEqual(self.dataflow_hook._build_dataflow_job_name(
+            job_name='df-job-1', append_job_name=False
+        ), 'df-job-1')
+
+        self.assertEqual(self.dataflow_hook._build_dataflow_job_name(
+            job_name='df-job', append_job_name=False
+        ), 'df-job')
+
+        self.assertEqual(self.dataflow_hook._build_dataflow_job_name(
+            job_name='dfjob', append_job_name=False
+        ), 'dfjob')
+
+        self.assertEqual(self.dataflow_hook._build_dataflow_job_name(
+            job_name='dfjob1', append_job_name=False
+        ), 'dfjob1')
+
+        self.assertRaises(
+            ValueError,
+            self.dataflow_hook._build_dataflow_job_name,
+            job_name='1dfjob', append_job_name=False
+        )
+
+        self.assertRaises(
+            ValueError,
+            self.dataflow_hook._build_dataflow_job_name,
+            job_name='dfjob@', append_job_name=False
+        )
+
+        self.assertRaises(
+            ValueError,
+            self.dataflow_hook._build_dataflow_job_name,
+            job_name='df^jo', append_job_name=False
+        )
 
 
 class DataFlowTemplateHookTest(unittest.TestCase):
@@ -183,7 +245,47 @@ def setUp(self):
     @mock.patch(DATAFLOW_STRING.format('DataFlowHook._start_template_dataflow'))
     def test_start_template_dataflow(self, internal_dataflow_mock):
         self.dataflow_hook.start_template_dataflow(
-            task_id=TASK_ID, variables=DATAFLOW_OPTIONS_TEMPLATE, parameters=PARAMETERS,
+            job_name=JOB_NAME, variables=DATAFLOW_OPTIONS_TEMPLATE, parameters=PARAMETERS,
             dataflow_template=TEMPLATE)
+        options_with_region = {'region': 'us-central1'}
+        options_with_region.update(DATAFLOW_OPTIONS_TEMPLATE)
         internal_dataflow_mock.assert_called_once_with(
-            mock.ANY, DATAFLOW_OPTIONS_TEMPLATE, PARAMETERS, TEMPLATE)
+            mock.ANY, options_with_region, PARAMETERS, TEMPLATE)
+
+
+class DataFlowJobTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mock_dataflow = MagicMock()
+
+    def test_dataflow_job_init_with_job_id(self):
+        mock_jobs = MagicMock()
+        self.mock_dataflow.projects.return_value.locations.return_value.\
+            jobs.return_value = mock_jobs
+        _DataflowJob(self.mock_dataflow, TEST_PROJECT, TEST_JOB_NAME,
+                     TEST_LOCATION, 10, TEST_JOB_ID)
+        mock_jobs.get.assert_called_with(projectId=TEST_PROJECT, location=TEST_LOCATION,
+                                         jobId=TEST_JOB_ID)
+
+    def test_dataflow_job_init_without_job_id(self):
+        mock_jobs = MagicMock()
+        self.mock_dataflow.projects.return_value.locations.return_value.\
+            jobs.return_value = mock_jobs
+        _DataflowJob(self.mock_dataflow, TEST_PROJECT, TEST_JOB_NAME,
+                     TEST_LOCATION, 10)
+        mock_jobs.list.assert_called_with(projectId=TEST_PROJECT,
+                                          location=TEST_LOCATION)
+
+
+class DataflowTest(unittest.TestCase):
+
+    def test_data_flow_valid_job_id(self):
+        cmd = ['echo', 'additional unit test lines.\n' +
+               'INFO: the Dataflow monitoring console, please navigate to' +
+               'https://console.cloud.google.com/dataflow/jobsDetail/locations/' +
+               '{}/jobs/{}?project={}'.format(TEST_LOCATION, TEST_JOB_ID, TEST_PROJECT)]
+        self.assertEqual(_Dataflow(cmd).wait_for_done(), TEST_JOB_ID)
+
+    def test_data_flow_missing_job_id(self):
+        cmd = ['echo', 'unit testing']
+        self.assertEqual(_Dataflow(cmd).wait_for_done(), None)
diff --git a/tests/contrib/hooks/test_gcp_dataproc_hook.py b/tests/contrib/hooks/test_gcp_dataproc_hook.py
index f2629ff148ce1..d5b5e96af421a 100644
--- a/tests/contrib/hooks/test_gcp_dataproc_hook.py
+++ b/tests/contrib/hooks/test_gcp_dataproc_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,27 +19,23 @@
 #
 
 import unittest
+from airflow.contrib.hooks.gcp_dataproc_hook import _DataProcJob
 from airflow.contrib.hooks.gcp_dataproc_hook import DataProcHook
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.contrib.utils.base_gcp_mock import GCP_PROJECT_ID_HOOK_UNIT_TEST
+from tests.compat import mock
 
 JOB = 'test-job'
-PROJECT_ID = 'test-project-id'
-REGION = 'global'
+GCP_REGION = 'global'
 TASK_ID = 'test-task-id'
 
 BASE_STRING = 'airflow.contrib.hooks.gcp_api_base_hook.{}'
 DATAPROC_STRING = 'airflow.contrib.hooks.gcp_dataproc_hook.{}'
 
+
 def mock_init(self, gcp_conn_id, delegate_to=None):
     pass
 
+
 class DataProcHookTest(unittest.TestCase):
     def setUp(self):
         with mock.patch(BASE_STRING.format('GoogleCloudBaseHook.__init__'),
@@ -48,6 +44,48 @@ def setUp(self):
 
     @mock.patch(DATAPROC_STRING.format('_DataProcJob'))
     def test_submit(self, job_mock):
-      with mock.patch(DATAPROC_STRING.format('DataProcHook.get_conn', return_value=None)):
-        self.dataproc_hook.submit(PROJECT_ID, JOB)
-        job_mock.assert_called_once_with(mock.ANY, PROJECT_ID, JOB, REGION)
+        with mock.patch(DATAPROC_STRING.format('DataProcHook.get_conn',
+                                               return_value=None)):
+            self.dataproc_hook.submit(GCP_PROJECT_ID_HOOK_UNIT_TEST, JOB)
+            job_mock.assert_called_once_with(mock.ANY, GCP_PROJECT_ID_HOOK_UNIT_TEST, JOB, GCP_REGION,
+                                             job_error_states=mock.ANY)
+
+
+class DataProcJobTest(unittest.TestCase):
+    @mock.patch(DATAPROC_STRING.format('_DataProcJob.__init__'), return_value=None)
+    def test_raise_error_default_job_error_states(self, mock_init):
+        job = _DataProcJob()
+        job.job = {'status': {'state': 'ERROR'}}
+        job.job_error_states = None
+        with self.assertRaises(Exception) as cm:
+            job.raise_error()
+        self.assertIn('ERROR', str(cm.exception))
+
+    @mock.patch(DATAPROC_STRING.format('_DataProcJob.__init__'), return_value=None)
+    def test_raise_error_custom_job_error_states(self, mock_init):
+        job = _DataProcJob()
+        job.job = {'status': {'state': 'CANCELLED'}}
+        job.job_error_states = ['ERROR', 'CANCELLED']
+        with self.assertRaises(Exception) as cm:
+            job.raise_error()
+        self.assertIn('CANCELLED', str(cm.exception))
+
+    @mock.patch(DATAPROC_STRING.format('_DataProcJob.__init__'), return_value=None)
+    def test_raise_error_fallback_job_error_states(self, mock_init):
+        job = _DataProcJob()
+        job.job = {'status': {'state': 'ERROR'}}
+        job.job_error_states = ['CANCELLED']
+        with self.assertRaises(Exception) as cm:
+            job.raise_error()
+        self.assertIn('ERROR', str(cm.exception))
+
+    @mock.patch(DATAPROC_STRING.format('_DataProcJob.__init__'), return_value=None)
+    def test_raise_error_with_state_done(self, mock_init):
+        job = _DataProcJob()
+        job.job = {'status': {'state': 'DONE'}}
+        job.job_error_states = None
+        try:
+            job.raise_error()
+            # Pass test
+        except Exception:
+            self.fail("raise_error() should not raise Exception when job=%s" % job.job)
diff --git a/tests/contrib/hooks/test_gcp_function_hook.py b/tests/contrib/hooks/test_gcp_function_hook.py
new file mode 100644
index 0000000000000..0ba1956805619
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_function_hook.py
@@ -0,0 +1,269 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import AirflowException
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id, \
+    mock_base_gcp_hook_default_project_id, GCP_PROJECT_ID_HOOK_UNIT_TEST, get_open_mock
+from tests.compat import mock
+
+from airflow.contrib.hooks.gcp_function_hook import GcfHook
+
+GCF_LOCATION = 'location'
+GCF_FUNCTION = 'function'
+
+
+class TestFunctionHookNoDefaultProjectId(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_no_default_project_id):
+            self.gcf_function_hook_no_project_id = GcfHook(gcp_conn_id='test', api_version='v1')
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook._wait_for_operation_to_complete')
+    def test_create_new_function_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        create_method = get_conn.return_value.projects.return_value.locations. \
+            return_value.functions.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.gcf_function_hook_no_project_id.create_new_function(
+                location=GCF_LOCATION,
+                body={}
+            )
+        create_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook._wait_for_operation_to_complete')
+    def test_create_new_function_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        create_method = get_conn.return_value.projects.return_value.locations. \
+            return_value.functions.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gcf_function_hook_no_project_id.create_new_function(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            location=GCF_LOCATION,
+            body={}
+        )
+        self.assertIsNone(res)
+        create_method.assert_called_once_with(body={},
+                                              location='projects/example-project/locations/location')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id')
+
+    @mock.patch('requests.put')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    def test_upload_function_zip_missing_project_id(self, get_conn, requests_put):
+        m = mock.mock_open()
+        with mock.patch('builtins.open', m):
+            generate_upload_url_method = get_conn.return_value.projects.return_value.locations. \
+                return_value.functions.return_value.generateUploadUrl
+            execute_method = generate_upload_url_method.return_value.execute
+            execute_method.return_value = {"uploadUrl": "http://uploadHere"}
+            requests_put.return_value = None
+            with self.assertRaises(AirflowException) as cm:
+                self.gcf_function_hook_no_project_id.upload_function_zip(
+                    location=GCF_LOCATION,
+                    zip_path="/tmp/path.zip"
+                )
+                generate_upload_url_method.assert_not_called()
+                execute_method.assert_not_called()
+                m.assert_not_called()
+                err = cm.exception
+                self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('requests.put')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    def test_upload_function_zip_overridden_project_id(self, get_conn, requests_put):
+        m, open_module = get_open_mock()
+        with mock.patch('{}.open'.format(open_module), m):
+            generate_upload_url_method = get_conn.return_value.projects.return_value.locations. \
+                return_value.functions.return_value.generateUploadUrl
+            execute_method = generate_upload_url_method.return_value.execute
+            execute_method.return_value = {"uploadUrl": "http://uploadHere"}
+            requests_put.return_value = None
+            res = self.gcf_function_hook_no_project_id.upload_function_zip(
+                project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+                location=GCF_LOCATION,
+                zip_path="/tmp/path.zip"
+            )
+            self.assertEqual("http://uploadHere", res)
+            generate_upload_url_method.assert_called_with(
+                parent='projects/example-project/locations/location')
+            execute_method.assert_called_once_with(num_retries=5)
+            requests_put.assert_called_once_with(
+                data=mock.ANY,
+                headers={'Content-type': 'application/zip',
+                         'x-goog-content-length-range': '0,104857600'},
+                url='http://uploadHere'
+            )
+
+
+class TestFunctionHookDefaultProjectId(unittest.TestCase):
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_default_project_id):
+            self.gcf_function_hook = GcfHook(gcp_conn_id='test', api_version='v1')
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook._wait_for_operation_to_complete')
+    def test_create_new_function(self, wait_for_operation_to_complete, get_conn):
+        create_method = get_conn.return_value.projects.return_value.locations.\
+            return_value.functions.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gcf_function_hook.create_new_function(
+            location=GCF_LOCATION,
+            body={}
+        )
+        self.assertIsNone(res)
+        create_method.assert_called_once_with(body={},
+                                              location='projects/example-project/locations/location')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook._wait_for_operation_to_complete')
+    def test_create_new_function_override_project_id(self, wait_for_operation_to_complete, get_conn):
+        create_method = get_conn.return_value.projects.return_value.locations. \
+            return_value.functions.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gcf_function_hook.create_new_function(
+            project_id='new-project',
+            location=GCF_LOCATION,
+            body={}
+        )
+        self.assertIsNone(res)
+        create_method.assert_called_once_with(body={},
+                                              location='projects/new-project/locations/location')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    def test_get_function(self, get_conn):
+        get_method = get_conn.return_value.projects.return_value.locations. \
+            return_value.functions.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "function"}
+        res = self.gcf_function_hook.get_function(
+            name=GCF_FUNCTION
+        )
+        self.assertIsNotNone(res)
+        self.assertEqual('function', res['name'])
+        get_method.assert_called_once_with(name='function')
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook._wait_for_operation_to_complete')
+    def test_delete_function(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.projects.return_value.locations. \
+            return_value.functions.return_value.delete
+        execute_method = delete_method.return_value.execute
+        wait_for_operation_to_complete.return_value = None
+        execute_method.return_value = {"name": "operation_id"}
+        res = self.gcf_function_hook.delete_function(
+            name=GCF_FUNCTION
+        )
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(name='function')
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook._wait_for_operation_to_complete')
+    def test_update_function(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.projects.return_value.locations. \
+            return_value.functions.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.gcf_function_hook.update_function(
+            update_mask=['a', 'b', 'c'],
+            name=GCF_FUNCTION,
+            body={}
+        )
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(
+            body={},
+            name='function',
+            updateMask='a,b,c'
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(operation_name='operation_id')
+
+    @mock.patch('requests.put')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    def test_upload_function_zip(self, get_conn, requests_put):
+        m, open_module = get_open_mock()
+        with mock.patch('{}.open'.format(open_module), m):
+            generate_upload_url_method = get_conn.return_value.projects.return_value.locations. \
+                return_value.functions.return_value.generateUploadUrl
+            execute_method = generate_upload_url_method.return_value.execute
+            execute_method.return_value = {"uploadUrl": "http://uploadHere"}
+            requests_put.return_value = None
+            res = self.gcf_function_hook.upload_function_zip(
+                location=GCF_LOCATION,
+                zip_path="/tmp/path.zip"
+            )
+            self.assertEqual("http://uploadHere", res)
+            generate_upload_url_method.assert_called_with(
+                parent='projects/example-project/locations/location')
+            execute_method.assert_called_once_with(num_retries=5)
+            requests_put.assert_called_once_with(
+                data=mock.ANY,
+                headers={'Content-type': 'application/zip',
+                         'x-goog-content-length-range': '0,104857600'},
+                url='http://uploadHere'
+            )
+
+    @mock.patch('requests.put')
+    @mock.patch('airflow.contrib.hooks.gcp_function_hook.GcfHook.get_conn')
+    def test_upload_function_zip_overridden_project_id(self, get_conn, requests_put):
+        m, open_module = get_open_mock()
+        with mock.patch('{}.open'.format(open_module), m):
+            generate_upload_url_method = get_conn.return_value.projects.return_value.locations. \
+                return_value.functions.return_value.generateUploadUrl
+            execute_method = generate_upload_url_method.return_value.execute
+            execute_method.return_value = {"uploadUrl": "http://uploadHere"}
+            requests_put.return_value = None
+            res = self.gcf_function_hook.upload_function_zip(
+                project_id='new-project',
+                location=GCF_LOCATION,
+                zip_path="/tmp/path.zip"
+            )
+            self.assertEqual("http://uploadHere", res)
+            generate_upload_url_method.assert_called_with(
+                parent='projects/new-project/locations/location')
+            execute_method.assert_called_once_with(num_retries=5)
+            requests_put.assert_called_once_with(
+                data=mock.ANY,
+                headers={'Content-type': 'application/zip',
+                         'x-goog-content-length-range': '0,104857600'},
+                url='http://uploadHere'
+            )
diff --git a/tests/contrib/hooks/test_gcp_kms_hook.py b/tests/contrib/hooks/test_gcp_kms_hook.py
new file mode 100644
index 0000000000000..e8ad400e74361
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_kms_hook.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from __future__ import unicode_literals
+
+import unittest
+from base64 import b64encode
+
+from airflow.contrib.hooks.gcp_kms_hook import GoogleCloudKMSHook
+from tests.compat import mock
+
+BASE_STRING = 'airflow.contrib.hooks.gcp_api_base_hook.{}'
+KMS_STRING = 'airflow.contrib.hooks.gcp_kms_hook.{}'
+
+
+TEST_PROJECT = 'test-project'
+TEST_LOCATION = 'global'
+TEST_KEY_RING = 'test-key-ring'
+TEST_KEY = 'test-key'
+TEST_KEY_ID = 'projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}'.format(
+    TEST_PROJECT, TEST_LOCATION, TEST_KEY_RING, TEST_KEY)
+
+
+def mock_init(self, gcp_conn_id, delegate_to=None):
+    pass
+
+
+class GoogleCloudKMSHookTest(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(BASE_STRING.format('GoogleCloudBaseHook.__init__'),
+                        new=mock_init):
+            self.kms_hook = GoogleCloudKMSHook(gcp_conn_id='test')
+
+    @mock.patch(KMS_STRING.format('GoogleCloudKMSHook.get_conn'))
+    def test_encrypt(self, mock_service):
+        plaintext = b'Test plaintext'
+        ciphertext = 'Test ciphertext'
+        plaintext_b64 = b64encode(plaintext).decode('ascii')
+        body = {'plaintext': plaintext_b64}
+        response = {'ciphertext': ciphertext}
+
+        encrypt_method = (mock_service.return_value
+                          .projects.return_value
+                          .locations.return_value
+                          .keyRings.return_value
+                          .cryptoKeys.return_value
+                          .encrypt)
+        execute_method = encrypt_method.return_value.execute
+        execute_method.return_value = response
+
+        ret_val = self.kms_hook.encrypt(TEST_KEY_ID, plaintext)
+        encrypt_method.assert_called_with(name=TEST_KEY_ID,
+                                          body=body)
+        execute_method.assert_called_with()
+        self.assertEqual(ciphertext, ret_val)
+
+    @mock.patch(KMS_STRING.format('GoogleCloudKMSHook.get_conn'))
+    def test_encrypt_authdata(self, mock_service):
+        plaintext = b'Test plaintext'
+        auth_data = b'Test authdata'
+        ciphertext = 'Test ciphertext'
+        plaintext_b64 = b64encode(plaintext).decode('ascii')
+        auth_data_b64 = b64encode(auth_data).decode('ascii')
+        body = {
+            'plaintext': plaintext_b64,
+            'additionalAuthenticatedData': auth_data_b64
+        }
+        response = {'ciphertext': ciphertext}
+
+        encrypt_method = (mock_service.return_value
+                          .projects.return_value
+                          .locations.return_value
+                          .keyRings.return_value
+                          .cryptoKeys.return_value
+                          .encrypt)
+        execute_method = encrypt_method.return_value.execute
+        execute_method.return_value = response
+
+        ret_val = self.kms_hook.encrypt(TEST_KEY_ID, plaintext,
+                                        authenticated_data=auth_data)
+        encrypt_method.assert_called_with(name=TEST_KEY_ID,
+                                          body=body)
+        execute_method.assert_called_with()
+        self.assertEqual(ciphertext, ret_val)
+
+    @mock.patch(KMS_STRING.format('GoogleCloudKMSHook.get_conn'))
+    def test_decrypt(self, mock_service):
+        plaintext = b'Test plaintext'
+        ciphertext = 'Test ciphertext'
+        plaintext_b64 = b64encode(plaintext).decode('ascii')
+        body = {'ciphertext': ciphertext}
+        response = {'plaintext': plaintext_b64}
+
+        decrypt_method = (mock_service.return_value
+                          .projects.return_value
+                          .locations.return_value
+                          .keyRings.return_value
+                          .cryptoKeys.return_value
+                          .decrypt)
+        execute_method = decrypt_method.return_value.execute
+        execute_method.return_value = response
+
+        ret_val = self.kms_hook.decrypt(TEST_KEY_ID, ciphertext)
+        decrypt_method.assert_called_with(name=TEST_KEY_ID,
+                                          body=body)
+        execute_method.assert_called_with()
+        self.assertEqual(plaintext, ret_val)
+
+    @mock.patch(KMS_STRING.format('GoogleCloudKMSHook.get_conn'))
+    def test_decrypt_authdata(self, mock_service):
+        plaintext = b'Test plaintext'
+        auth_data = b'Test authdata'
+        ciphertext = 'Test ciphertext'
+        plaintext_b64 = b64encode(plaintext).decode('ascii')
+        auth_data_b64 = b64encode(auth_data).decode('ascii')
+        body = {
+            'ciphertext': ciphertext,
+            'additionalAuthenticatedData': auth_data_b64
+        }
+        response = {'plaintext': plaintext_b64}
+
+        decrypt_method = (mock_service.return_value
+                          .projects.return_value
+                          .locations.return_value
+                          .keyRings.return_value
+                          .cryptoKeys.return_value
+                          .decrypt)
+        execute_method = decrypt_method.return_value.execute
+        execute_method.return_value = response
+
+        ret_val = self.kms_hook.decrypt(TEST_KEY_ID, ciphertext,
+                                        authenticated_data=auth_data)
+        decrypt_method.assert_called_with(name=TEST_KEY_ID,
+                                          body=body)
+        execute_method.assert_called_with()
+        self.assertEqual(plaintext, ret_val)
diff --git a/tests/contrib/hooks/test_gcp_mlengine_hook.py b/tests/contrib/hooks/test_gcp_mlengine_hook.py
index bb3c5b62586db..be06619d53e84 100644
--- a/tests/contrib/hooks/test_gcp_mlengine_hook.py
+++ b/tests/contrib/hooks/test_gcp_mlengine_hook.py
@@ -25,9 +25,9 @@
     from urllib.parse import urlparse, parse_qsl
 
 from airflow.contrib.hooks import gcp_mlengine_hook as hook
-from apiclient import errors
-from apiclient.discovery import build_from_document
-from apiclient.http import HttpMockSequence
+from googleapiclient.errors import HttpError
+from googleapiclient.discovery import build_from_document
+from googleapiclient.http import HttpMockSequence
 from google.auth.exceptions import GoogleAuthError
 import requests
 
@@ -61,7 +61,8 @@ def __init__(self, test_cls, responses, expected_requests):
             for x in expected_requests]
         self._actual_requests = []
 
-    def _normalize_requests_for_comparison(self, uri, http_method, body):
+    @staticmethod
+    def _normalize_requests_for_comparison(uri, http_method, body):
         parts = urlparse(uri)
         return (
             parts._replace(query=set(parse_qsl(parts.query))),
@@ -89,7 +90,7 @@ def __exit__(self, *args):
         # Propogating exceptions here since assert will silence them.
         if any(args):
             return None
-        self._test_cls.assertEquals(
+        self._test_cls.assertEqual(
             [self._normalize_requests_for_comparison(x[0], x[1], x[2])
                 for x in self._actual_requests],
             self._expected_requests)
@@ -131,7 +132,7 @@ def test_create_version(self):
             create_version_response = cml_hook.create_version(
                 project_id=project, model_name=model_name,
                 version_spec=version)
-            self.assertEquals(create_version_response, response_body)
+            self.assertEqual(create_version_response, response_body)
 
     @_SKIP_IF
     def test_set_default_version(self):
@@ -157,7 +158,7 @@ def test_set_default_version(self):
             set_default_version_response = cml_hook.set_default_version(
                 project_id=project, model_name=model_name,
                 version_name=version)
-            self.assertEquals(set_default_version_response, response_body)
+            self.assertEqual(set_default_version_response, response_body)
 
     @_SKIP_IF
     def test_list_versions(self):
@@ -184,8 +185,7 @@ def test_list_versions(self):
                 self._SERVICE_URI_PREFIX, project, model_name), 'GET',
              None),
         ] + [
-            ('{}projects/{}/models/{}/versions?alt=json&pageToken={}'
-             '&pageSize=100'.format(
+            ('{}projects/{}/models/{}/versions?alt=json&pageToken={}&pageSize=100'.format(
                 self._SERVICE_URI_PREFIX, project, model_name, ix), 'GET',
              None) for ix in range(len(versions) - 1)
         ]
@@ -196,7 +196,7 @@ def test_list_versions(self):
                 expected_requests=expected_requests) as cml_hook:
             list_versions_response = cml_hook.list_versions(
                 project_id=project, model_name=model_name)
-            self.assertEquals(list_versions_response, versions)
+            self.assertEqual(list_versions_response, versions)
 
     @_SKIP_IF
     def test_delete_version(self):
@@ -230,7 +230,7 @@ def test_delete_version(self):
             delete_version_response = cml_hook.delete_version(
                 project_id=project, model_name=model_name,
                 version_name=version)
-            self.assertEquals(delete_version_response, done_response_body)
+            self.assertEqual(delete_version_response, done_response_body)
 
     @_SKIP_IF
     def test_create_model(self):
@@ -254,7 +254,7 @@ def test_create_model(self):
                 expected_requests=expected_requests) as cml_hook:
             create_model_response = cml_hook.create_model(
                 project_id=project, model=model)
-            self.assertEquals(create_model_response, response_body)
+            self.assertEqual(create_model_response, response_body)
 
     @_SKIP_IF
     def test_get_model(self):
@@ -275,7 +275,7 @@ def test_get_model(self):
                 expected_requests=expected_requests) as cml_hook:
             get_model_response = cml_hook.get_model(
                 project_id=project, model_name=model_name)
-            self.assertEquals(get_model_response, response_body)
+            self.assertEqual(get_model_response, response_body)
 
     @_SKIP_IF
     def test_create_mlengine_job(self):
@@ -311,7 +311,7 @@ def test_create_mlengine_job(self):
                 expected_requests=expected_requests) as cml_hook:
             create_job_response = cml_hook.create_job(
                 project_id=project, job=my_job)
-            self.assertEquals(create_job_response, my_job)
+            self.assertEqual(create_job_response, my_job)
 
     @_SKIP_IF
     def test_create_mlengine_job_reuse_existing_job_by_default(self):
@@ -343,7 +343,7 @@ def test_create_mlengine_job_reuse_existing_job_by_default(self):
                 expected_requests=expected_requests) as cml_hook:
             create_job_response = cml_hook.create_job(
                 project_id=project, job=my_job)
-            self.assertEquals(create_job_response, my_job)
+            self.assertEqual(create_job_response, my_job)
 
     @_SKIP_IF
     def test_create_mlengine_job_check_existing_job(self):
@@ -392,7 +392,7 @@ def check_input(existing_job):
                 self,
                 responses=responses,
                 expected_requests=expected_requests) as cml_hook:
-            with self.assertRaises(errors.HttpError):
+            with self.assertRaises(HttpError):
                 cml_hook.create_job(
                     project_id=project, job=my_job,
                     use_existing_job_fn=check_input)
@@ -414,7 +414,7 @@ def check_input(existing_job):
             create_job_response = cml_hook.create_job(
                 project_id=project, job=my_job,
                 use_existing_job_fn=check_input)
-            self.assertEquals(create_job_response, my_job)
+            self.assertEqual(create_job_response, my_job)
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/hooks/test_gcp_natural_language_hook.py b/tests/contrib/hooks/test_gcp_natural_language_hook.py
new file mode 100644
index 0000000000000..33b03dea96cac
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_natural_language_hook.py
@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import unittest
+from typing import Dict, Any
+
+from google.cloud.language_v1.proto.language_service_pb2 import Document
+
+from airflow.contrib.hooks.gcp_natural_language_hook import CloudNaturalLanguageHook
+from tests.compat import mock
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id
+
+
+API_RESPONSE = {}  # type: Dict[Any, Any]
+DOCUMENT = Document(
+    content="Airflow is a platform to programmatically author, schedule and monitor workflows."
+)
+ENCODING_TYPE = "UTF32"
+
+
+class TestCloudNaturalLanguageHook(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(
+            "airflow.contrib.hooks." "gcp_api_base_hook.GoogleCloudBaseHook.__init__",
+            new=mock_base_gcp_hook_no_default_project_id,
+        ):
+            self.hook = CloudNaturalLanguageHook(gcp_conn_id="test")
+
+    @mock.patch(  # type: ignore
+        "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook.get_conn",
+        **{"return_value.analyze_entities.return_value": API_RESPONSE}  # type: ignore
+    )
+    def test_analyze_entities(self, get_conn):
+        result = self.hook.analyze_entities(document=DOCUMENT, encoding_type=ENCODING_TYPE)
+
+        self.assertIs(result, API_RESPONSE)
+
+        get_conn.return_value.analyze_entities.assert_called_once_with(
+            document=DOCUMENT, encoding_type=ENCODING_TYPE, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch(  # type: ignore
+        "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook.get_conn",
+        **{"return_value.analyze_entity_sentiment.return_value": API_RESPONSE}
+    )
+    def test_analyze_entity_sentiment(self, get_conn):
+        result = self.hook.analyze_entity_sentiment(document=DOCUMENT, encoding_type=ENCODING_TYPE)
+
+        self.assertIs(result, API_RESPONSE)
+
+        get_conn.return_value.analyze_entity_sentiment.assert_called_once_with(
+            document=DOCUMENT, encoding_type=ENCODING_TYPE, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch(  # type: ignore
+        "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook.get_conn",
+        **{"return_value.analyze_sentiment.return_value": API_RESPONSE}
+    )
+    def test_analyze_sentiment(self, get_conn):
+        result = self.hook.analyze_sentiment(document=DOCUMENT, encoding_type=ENCODING_TYPE)
+
+        self.assertIs(result, API_RESPONSE)
+
+        get_conn.return_value.analyze_sentiment.assert_called_once_with(
+            document=DOCUMENT, encoding_type=ENCODING_TYPE, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch(  # type: ignore
+        "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook.get_conn",
+        **{"return_value.analyze_syntax.return_value": API_RESPONSE}
+    )
+    def test_analyze_syntax(self, get_conn):
+        result = self.hook.analyze_syntax(document=DOCUMENT, encoding_type=ENCODING_TYPE)
+
+        self.assertIs(result, API_RESPONSE)
+
+        get_conn.return_value.analyze_syntax.assert_called_once_with(
+            document=DOCUMENT, encoding_type=ENCODING_TYPE, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch(  # type: ignore
+        "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook.get_conn",
+        **{"return_value.annotate_text.return_value": API_RESPONSE}
+    )
+    def test_annotate_text(self, get_conn):
+        result = self.hook.annotate_text(document=DOCUMENT, encoding_type=ENCODING_TYPE, features=None)
+
+        self.assertIs(result, API_RESPONSE)
+
+        get_conn.return_value.annotate_text.assert_called_once_with(
+            document=DOCUMENT,
+            encoding_type=ENCODING_TYPE,
+            features=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch(  # type: ignore
+        "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook.get_conn",
+        **{"return_value.classify_text.return_value": API_RESPONSE}
+    )
+    def test_classify_text(self, get_conn):
+        result = self.hook.classify_text(document=DOCUMENT)
+
+        self.assertIs(result, API_RESPONSE)
+
+        get_conn.return_value.classify_text.assert_called_once_with(
+            document=DOCUMENT, retry=None, timeout=None, metadata=None
+        )
diff --git a/tests/contrib/hooks/test_gcp_pubsub_hook.py b/tests/contrib/hooks/test_gcp_pubsub_hook.py
index 98f1b1316392c..29231866f7f2e 100644
--- a/tests/contrib/hooks/test_gcp_pubsub_hook.py
+++ b/tests/contrib/hooks/test_gcp_pubsub_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,17 +23,10 @@
 from base64 import b64encode as b64e
 import unittest
 
-from apiclient.errors import HttpError
+from googleapiclient.errors import HttpError
 
 from airflow.contrib.hooks.gcp_pubsub_hook import PubSubException, PubSubHook
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 BASE_STRING = 'airflow.contrib.hooks.gcp_api_base_hook.{}'
 PUBSUB_STRING = 'airflow.contrib.hooks.gcp_pubsub_hook.{}'
@@ -93,8 +86,8 @@ def test_delete_nonexisting_topic_failifnotexists(self, mock_service):
         with self.assertRaises(PubSubException) as e:
             self.pubsub_hook.delete_topic(TEST_PROJECT, TEST_TOPIC, True)
 
-        self.assertEquals(str(e.exception),
-                          'Topic does not exist: %s' % EXPANDED_TOPIC)
+        self.assertEqual(str(e.exception),
+                         'Topic does not exist: %s' % EXPANDED_TOPIC)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_create_preexisting_topic_failifexists(self, mock_service):
@@ -104,8 +97,8 @@ def test_create_preexisting_topic_failifexists(self, mock_service):
 
         with self.assertRaises(PubSubException) as e:
             self.pubsub_hook.create_topic(TEST_PROJECT, TEST_TOPIC, True)
-        self.assertEquals(str(e.exception),
-                          'Topic already exists: %s' % EXPANDED_TOPIC)
+        self.assertEqual(str(e.exception),
+                         'Topic already exists: %s' % EXPANDED_TOPIC)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_create_preexisting_topic_nofailifexists(self, mock_service):
@@ -130,7 +123,7 @@ def test_create_nonexistent_subscription(self, mock_service):
         create_method.assert_called_with(name=EXPANDED_SUBSCRIPTION,
                                          body=expected_body)
         create_method.return_value.execute.assert_called_with()
-        self.assertEquals(TEST_SUBSCRIPTION, response)
+        self.assertEqual(TEST_SUBSCRIPTION, response)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_create_subscription_different_project_topic(self, mock_service):
@@ -150,7 +143,7 @@ def test_create_subscription_different_project_topic(self, mock_service):
         create_method.assert_called_with(name=expected_subscription,
                                          body=expected_body)
         create_method.return_value.execute.assert_called_with()
-        self.assertEquals(TEST_SUBSCRIPTION, response)
+        self.assertEqual(TEST_SUBSCRIPTION, response)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_delete_subscription(self, mock_service):
@@ -172,9 +165,9 @@ def test_delete_nonexisting_subscription_failifnotexists(self,
             self.pubsub_hook.delete_subscription(
                 TEST_PROJECT, TEST_SUBSCRIPTION, fail_if_not_exists=True)
 
-        self.assertEquals(str(e.exception),
-                          'Subscription does not exist: %s' %
-                          EXPANDED_SUBSCRIPTION)
+        self.assertEqual(str(e.exception),
+                         'Subscription does not exist: %s' %
+                         EXPANDED_SUBSCRIPTION)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     @mock.patch(PUBSUB_STRING.format('uuid4'),
@@ -194,7 +187,7 @@ def test_create_subscription_without_name(self, mock_uuid, mock_service):
         create_method.assert_called_with(name=expected_name,
                                          body=expected_body)
         create_method.return_value.execute.assert_called_with()
-        self.assertEquals('sub-%s' % TEST_UUID, response)
+        self.assertEqual('sub-%s' % TEST_UUID, response)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_create_subscription_with_ack_deadline(self, mock_service):
@@ -211,7 +204,7 @@ def test_create_subscription_with_ack_deadline(self, mock_service):
         create_method.assert_called_with(name=EXPANDED_SUBSCRIPTION,
                                          body=expected_body)
         create_method.return_value.execute.assert_called_with()
-        self.assertEquals(TEST_SUBSCRIPTION, response)
+        self.assertEqual(TEST_SUBSCRIPTION, response)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_create_subscription_failifexists(self, mock_service):
@@ -225,9 +218,9 @@ def test_create_subscription_failifexists(self, mock_service):
                 TEST_PROJECT, TEST_TOPIC, TEST_SUBSCRIPTION,
                 fail_if_exists=True)
 
-        self.assertEquals(str(e.exception),
-                          'Subscription already exists: %s' %
-                          EXPANDED_SUBSCRIPTION)
+        self.assertEqual(str(e.exception),
+                         'Subscription already exists: %s' %
+                         EXPANDED_SUBSCRIPTION)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_create_subscription_nofailifexists(self, mock_service):
@@ -238,7 +231,7 @@ def test_create_subscription_nofailifexists(self, mock_service):
         response = self.pubsub_hook.create_subscription(
             TEST_PROJECT, TEST_TOPIC, TEST_SUBSCRIPTION
         )
-        self.assertEquals(TEST_SUBSCRIPTION, response)
+        self.assertEqual(TEST_SUBSCRIPTION, response)
 
     @mock.patch(PUBSUB_STRING.format('PubSubHook.get_conn'))
     def test_publish(self, mock_service):
diff --git a/tests/contrib/hooks/test_gcp_spanner_hook.py b/tests/contrib/hooks/test_gcp_spanner_hook.py
new file mode 100644
index 0000000000000..601e111ca5749
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_spanner_hook.py
@@ -0,0 +1,723 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_spanner_hook import CloudSpannerHook
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id, \
+    GCP_PROJECT_ID_HOOK_UNIT_TEST, mock_base_gcp_hook_default_project_id
+from tests.compat import mock
+
+SPANNER_INSTANCE = 'instance'
+SPANNER_CONFIGURATION = 'configuration'
+SPANNER_DATABASE = 'database-name'
+
+
+class TestGcpSpannerHookDefaultProjectId(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_default_project_id):
+            self.spanner_hook_default_project_id = CloudSpannerHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_existing_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        res = self.spanner_hook_default_project_id.get_instance(instance_id=SPANNER_INSTANCE,
+                                                                project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_existing_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        res = self.spanner_hook_default_project_id.get_instance(instance_id=SPANNER_INSTANCE,
+                                                                project_id='new-project')
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        create_method = instance_method.return_value.create
+        create_method.return_value = False
+        res = self.spanner_hook_default_project_id.create_instance(
+            instance_id=SPANNER_INSTANCE,
+            configuration_name=SPANNER_CONFIGURATION,
+            node_count=1,
+            display_name=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(
+            instance_id='instance',
+            configuration_name='configuration',
+            display_name='database-name',
+            node_count=1)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        create_method = instance_method.return_value.create
+        create_method.return_value = False
+        res = self.spanner_hook_default_project_id.create_instance(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            configuration_name=SPANNER_CONFIGURATION,
+            node_count=1,
+            display_name=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(
+            instance_id='instance',
+            configuration_name='configuration',
+            display_name='database-name',
+            node_count=1)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        update_method = instance_method.return_value.update
+        update_method.return_value = False
+        res = self.spanner_hook_default_project_id.update_instance(
+            instance_id=SPANNER_INSTANCE,
+            configuration_name=SPANNER_CONFIGURATION,
+            node_count=2,
+            display_name=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(
+            instance_id='instance', configuration_name='configuration', display_name='database-name',
+            node_count=2)
+        update_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        update_method = instance_method.return_value.update
+        update_method.return_value = False
+        res = self.spanner_hook_default_project_id.update_instance(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            configuration_name=SPANNER_CONFIGURATION,
+            node_count=2,
+            display_name=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(
+            instance_id='instance', configuration_name='configuration', display_name='database-name',
+            node_count=2)
+        update_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        delete_method.return_value = False
+        res = self.spanner_hook_default_project_id.delete_instance(
+            instance_id=SPANNER_INSTANCE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(
+            'instance')
+        delete_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        delete_method.return_value = False
+        res = self.spanner_hook_default_project_id.delete_instance(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(
+            'instance')
+        delete_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_database(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_exists_method = instance_method.return_value.exists
+        database_exists_method.return_value = True
+        res = self.spanner_hook_default_project_id.get_database(
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_exists_method = instance_method.return_value.exists
+        database_exists_method.return_value = True
+        res = self.spanner_hook_default_project_id.get_database(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_database(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_create_method = database_method.return_value.create
+        res = self.spanner_hook_default_project_id.create_database(
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name', ddl_statements=[])
+        database_create_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_create_method = database_method.return_value.create
+        res = self.spanner_hook_default_project_id.create_database(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name', ddl_statements=[])
+        database_create_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_database(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_update_ddl_method = database_method.return_value.update_ddl
+        res = self.spanner_hook_default_project_id.update_database(
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_update_ddl_method.assert_called_with(ddl_statements=[], operation_id=None)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_update_ddl_method = database_method.return_value.update_ddl
+        res = self.spanner_hook_default_project_id.update_database(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_update_ddl_method.assert_called_with(ddl_statements=[], operation_id=None)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_database(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_drop_method = database_method.return_value.drop
+        database_exists_method = database_method.return_value.exists
+        database_exists_method.return_value = True
+        res = self.spanner_hook_default_project_id.delete_database(
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        database_drop_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_drop_method = database_method.return_value.drop
+        database_exists_method = database_method.return_value.exists
+        database_exists_method.return_value = True
+        res = self.spanner_hook_default_project_id.delete_database(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        database_drop_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_execute_dml(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        run_in_transaction_method = database_method.return_value.run_in_transaction
+        res = self.spanner_hook_default_project_id.execute_dml(
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            queries='')
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        run_in_transaction_method.assert_called_with(mock.ANY)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_execute_dml_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        run_in_transaction_method = database_method.return_value.run_in_transaction
+        res = self.spanner_hook_default_project_id.execute_dml(
+            project_id='new-project',
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            queries='')
+        get_client.assert_called_once_with(project_id='new-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        run_in_transaction_method.assert_called_with(mock.ANY)
+        self.assertIsNone(res)
+
+
+class TestGcpSpannerHookNoDefaultProjectID(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_no_default_project_id):
+            self.spanner_hook_no_default_project_id = CloudSpannerHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_existing_instance_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.get_instance(instance_id=SPANNER_INSTANCE)
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        instance_exists_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_existing_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        res = self.spanner_hook_no_default_project_id.get_instance(instance_id=SPANNER_INSTANCE,
+                                                                   project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_non_existing_instance(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = False
+        res = self.spanner_hook_no_default_project_id.get_instance(instance_id=SPANNER_INSTANCE,
+                                                                   project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_instance_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        create_method = instance_method.return_value.create
+        create_method.return_value = False
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.create_instance(
+                instance_id=SPANNER_INSTANCE,
+                configuration_name=SPANNER_CONFIGURATION,
+                node_count=1,
+                display_name=SPANNER_DATABASE)
+            get_client.assert_called_once_with(project_id='example-project')
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        create_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        create_method = instance_method.return_value.create
+        create_method.return_value = False
+        res = self.spanner_hook_no_default_project_id.create_instance(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            configuration_name=SPANNER_CONFIGURATION,
+            node_count=1,
+            display_name=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(
+            instance_id='instance',
+            configuration_name='configuration',
+            display_name='database-name',
+            node_count=1)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_instance_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        update_method = instance_method.return_value.update
+        update_method.return_value = False
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.update_instance(
+                instance_id=SPANNER_INSTANCE,
+                configuration_name=SPANNER_CONFIGURATION,
+                node_count=2,
+                display_name=SPANNER_DATABASE)
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        update_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        update_method = instance_method.return_value.update
+        update_method.return_value = False
+        res = self.spanner_hook_no_default_project_id.update_instance(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            configuration_name=SPANNER_CONFIGURATION,
+            node_count=2,
+            display_name=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(
+            instance_id='instance', configuration_name='configuration', display_name='database-name',
+            node_count=2)
+        update_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_instance_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        delete_method.return_value = False
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.delete_instance(
+                instance_id=SPANNER_INSTANCE)
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        delete_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_instance_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        delete_method = instance_method.return_value.delete
+        delete_method.return_value = False
+        res = self.spanner_hook_no_default_project_id.delete_instance(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(
+            'instance')
+        delete_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_database_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_exists_method = instance_method.return_value.exists
+        database_exists_method.return_value = True
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.get_database(
+                instance_id=SPANNER_INSTANCE,
+                database_id=SPANNER_DATABASE)
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        database_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_get_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_exists_method = instance_method.return_value.exists
+        database_exists_method.return_value = True
+        res = self.spanner_hook_no_default_project_id.get_database(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        self.assertIsNotNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_database_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_create_method = database_method.return_value.create
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.create_database(
+                instance_id=SPANNER_INSTANCE,
+                database_id=SPANNER_DATABASE,
+                ddl_statements=[])
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        database_method.assert_not_called()
+        database_create_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_create_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_create_method = database_method.return_value.create
+        res = self.spanner_hook_no_default_project_id.create_database(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name', ddl_statements=[])
+        database_create_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_database_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_update_method = database_method.return_value.update
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.update_database(
+                instance_id=SPANNER_INSTANCE,
+                database_id=SPANNER_DATABASE,
+                ddl_statements=[])
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        database_method.assert_not_called()
+        database_update_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_update_ddl_method = database_method.return_value.update_ddl
+        res = self.spanner_hook_no_default_project_id.update_database(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_update_ddl_method.assert_called_with(ddl_statements=[], operation_id=None)
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_update_database_overridden_project_id_and_operation(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_update_ddl_method = database_method.return_value.update_ddl
+        res = self.spanner_hook_no_default_project_id.update_database(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            operation_id="operation",
+            ddl_statements=[])
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_update_ddl_method.assert_called_with(ddl_statements=[], operation_id="operation")
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_database_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_drop_method = database_method.return_value.drop
+        database_exists_method = database_method.return_value.exists
+        database_exists_method.return_value = True
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.delete_database(
+                instance_id=SPANNER_INSTANCE,
+                database_id=SPANNER_DATABASE)
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        database_method.assert_not_called()
+        database_exists_method.assert_not_called()
+        database_drop_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_database_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_drop_method = database_method.return_value.drop
+        database_exists_method = database_method.return_value.exists
+        database_exists_method.return_value = True
+        res = self.spanner_hook_no_default_project_id.delete_database(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        database_drop_method.assert_called_with()
+        self.assertIsNone(res)
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_delete_database_missing_database(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        database_drop_method = database_method.return_value.drop
+        database_exists_method = database_method.return_value.exists
+        database_exists_method.return_value = False
+        self.spanner_hook_no_default_project_id.delete_database(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE)
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        database_exists_method.assert_called_with()
+        database_drop_method.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_execute_dml_missing_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        run_in_transaction_method = database_method.return_value.run_in_transaction
+        with self.assertRaises(AirflowException) as cm:
+            self.spanner_hook_no_default_project_id.execute_dml(
+                instance_id=SPANNER_INSTANCE,
+                database_id=SPANNER_DATABASE,
+                queries='')
+        get_client.assert_not_called()
+        instance_method.assert_not_called()
+        database_method.assert_not_called()
+        run_in_transaction_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook._get_client')
+    def test_execute_dml_overridden_project_id(self, get_client):
+        instance_method = get_client.return_value.instance
+        instance_exists_method = instance_method.return_value.exists
+        instance_exists_method.return_value = True
+        database_method = instance_method.return_value.database
+        run_in_transaction_method = database_method.return_value.run_in_transaction
+        res = self.spanner_hook_no_default_project_id.execute_dml(
+            project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
+            instance_id=SPANNER_INSTANCE,
+            database_id=SPANNER_DATABASE,
+            queries='')
+        get_client.assert_called_once_with(project_id='example-project')
+        instance_method.assert_called_once_with(instance_id='instance')
+        database_method.assert_called_with(database_id='database-name')
+        run_in_transaction_method.assert_called_with(mock.ANY)
+        self.assertIsNone(res)
diff --git a/tests/contrib/hooks/test_gcp_sql_hook.py b/tests/contrib/hooks/test_gcp_sql_hook.py
new file mode 100644
index 0000000000000..a7d548e1467f3
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_sql_hook.py
@@ -0,0 +1,1001 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import unittest
+
+from googleapiclient.errors import HttpError
+
+from airflow.contrib.hooks.gcp_sql_hook import CloudSqlHook, CloudSqlDatabaseHook
+from airflow.exceptions import AirflowException
+from airflow.models import Connection
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id, \
+    mock_base_gcp_hook_no_default_project_id
+from tests.compat import mock
+
+from parameterized import parameterized
+
+
+class TestGcpSqlHookDefaultProjectId(unittest.TestCase):
+
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_default_project_id):
+            self.cloudsql_hook = CloudSqlHook(api_version='v1', gcp_conn_id='test')
+
+    def test_instance_import_exception(self):
+        self.cloudsql_hook.get_conn = mock.Mock(
+            side_effect=HttpError(resp={'status': '400'},
+                                  content='Error content'.encode('utf-8'))
+        )
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook.import_instance(
+                instance='instance',
+                body={})
+        err = cm.exception
+        self.assertIn("Importing instance ", str(err))
+
+    def test_instance_export_exception(self):
+        self.cloudsql_hook.get_conn = mock.Mock(
+            side_effect=HttpError(resp={'status': '400'},
+                                  content='Error content'.encode('utf-8'))
+        )
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook.export_instance(
+                instance='instance',
+                body={})
+        err = cm.exception
+        self.assertIn("Exporting instance ", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_import(self, wait_for_operation_to_complete, get_conn):
+        import_method = get_conn.return_value.instances.return_value.import_
+        execute_method = import_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.import_instance(
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        import_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_import_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        import_method = get_conn.return_value.instances.return_value.import_
+        execute_method = import_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.import_instance(
+            project_id='new-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        import_method.assert_called_once_with(body={}, instance='instance', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='new-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_export(self, wait_for_operation_to_complete, get_conn):
+        export_method = get_conn.return_value.instances.return_value.export
+        execute_method = export_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.export_instance(
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        export_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_export_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        export_method = get_conn.return_value.instances.return_value.export
+        execute_method = export_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.export_instance(
+            project_id='new-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        export_method.assert_called_once_with(body={}, instance='instance', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='new-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_instance(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instances.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "instance"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.get_instance(
+            instance='instance')
+        self.assertIsNotNone(res)
+        self.assertEqual('instance', res['name'])
+        get_method.assert_called_once_with(instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instances.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "instance"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.get_instance(
+            project_id='new-project',
+            instance='instance')
+        self.assertIsNotNone(res)
+        self.assertEqual('instance', res['name'])
+        get_method.assert_called_once_with(instance='instance', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_instance(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instances.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.create_instance(
+            body={})
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instances.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.create_instance(
+            project_id='new-project',
+            body={})
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='new-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_instance(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instances.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.patch_instance(
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instances.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.patch_instance(
+            project_id='new-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(body={}, instance='instance', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='new-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_instance(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.instances.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.delete_instance(
+            instance='instance')
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.instances.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.delete_instance(
+            project_id='new-project',
+            instance='instance')
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(instance='instance', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='new-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_database(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.databases.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "database"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.get_database(
+            database='database',
+            instance='instance')
+        self.assertIsNotNone(res)
+        self.assertEqual('database', res['name'])
+        get_method.assert_called_once_with(instance='instance',
+                                           database='database',
+                                           project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.databases.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "database"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.get_database(
+            project_id='new-project',
+            database='database',
+            instance='instance')
+        self.assertIsNotNone(res)
+        self.assertEqual('database', res['name'])
+        get_method.assert_called_once_with(instance='instance',
+                                           database='database',
+                                           project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_database(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.databases.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.create_database(
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.databases.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.create_database(
+            project_id='new-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, instance='instance', project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='new-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_database(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.databases.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.patch_database(
+            instance='instance',
+            database='database',
+            body={})
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(body={},
+                                             database='database',
+                                             instance='instance',
+                                             project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.databases.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.patch_database(
+            project_id='new-project',
+            instance='instance',
+            database='database',
+            body={})
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(body={},
+                                             database='database',
+                                             instance='instance',
+                                             project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='new-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_database(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.databases.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.delete_database(
+            instance='instance',
+            database='database')
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(database='database',
+                                              instance='instance',
+                                              project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.databases.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook.delete_database(
+            project_id='new-project',
+            instance='instance',
+            database='database')
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(database='database',
+                                              instance='instance',
+                                              project='new-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='new-project'
+        )
+
+
+class TestGcpSqlHookNoDefaultProjectID(unittest.TestCase):
+    def setUp(self):
+        with mock.patch('airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+                        new=mock_base_gcp_hook_no_default_project_id):
+            self.cloudsql_hook_no_default_project_id = CloudSqlHook(api_version='v1', gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_import_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        import_method = get_conn.return_value.instances.return_value.import_
+        execute_method = import_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.import_instance(
+            project_id='example-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        import_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_import_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        import_method = get_conn.return_value.instances.return_value.import_
+        execute_method = import_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.import_instance(
+                instance='instance',
+                body={})
+        import_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_export_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        export_method = get_conn.return_value.instances.return_value.export
+        execute_method = export_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.export_instance(
+            project_id='example-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        export_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(project_id='example-project',
+                                                               operation_name='operation_id')
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_instance_export_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        export_method = get_conn.return_value.instances.return_value.export
+        execute_method = export_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.export_instance(
+                instance='instance',
+                body={})
+        export_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instances.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "instance"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.get_instance(
+            project_id='example-project',
+            instance='instance')
+        self.assertIsNotNone(res)
+        self.assertEqual('instance', res['name'])
+        get_method.assert_called_once_with(instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_instance_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.instances.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "instance"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.get_instance(
+                instance='instance')
+        get_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instances.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.create_instance(
+            project_id='example-project',
+            body={})
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_instance_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.instances.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.create_instance(
+                body={})
+        insert_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instances.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.patch_instance(
+            project_id='example-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_instance_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.instances.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.patch_instance(
+                instance='instance',
+                body={})
+        patch_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_instance_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.instances.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.delete_instance(
+            project_id='example-project',
+            instance='instance')
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_instance_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.instances.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.delete_instance(
+                instance='instance')
+        delete_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.databases.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "database"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.get_database(
+            project_id='example-project',
+            database='database',
+            instance='instance')
+        self.assertIsNotNone(res)
+        self.assertEqual('database', res['name'])
+        get_method.assert_called_once_with(instance='instance',
+                                           database='database',
+                                           project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_get_database_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        get_method = get_conn.return_value.databases.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = {"name": "database"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.get_database(
+                database='database',
+                instance='instance')
+        get_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.databases.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.create_database(
+            project_id='example-project',
+            instance='instance',
+            body={})
+        self.assertIsNone(res)
+        insert_method.assert_called_once_with(body={}, instance='instance', project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_create_database_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        insert_method = get_conn.return_value.databases.return_value.insert
+        execute_method = insert_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.create_database(
+                instance='instance',
+                body={})
+        insert_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.databases.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.patch_database(
+            project_id='example-project',
+            instance='instance',
+            database='database',
+            body={})
+        self.assertIsNone(res)
+        patch_method.assert_called_once_with(body={},
+                                             database='database',
+                                             instance='instance',
+                                             project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_patch_database_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        patch_method = get_conn.return_value.databases.return_value.patch
+        execute_method = patch_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.patch_database(
+                instance='instance',
+                database='database',
+                body={})
+        patch_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_database_overridden_project_id(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.databases.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        res = self.cloudsql_hook_no_default_project_id.delete_database(
+            project_id='example-project',
+            instance='instance',
+            database='database')
+        self.assertIsNone(res)
+        delete_method.assert_called_once_with(database='database',
+                                              instance='instance',
+                                              project='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+        wait_for_operation_to_complete.assert_called_once_with(
+            operation_name='operation_id', project_id='example-project'
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook.get_conn')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook._wait_for_operation_to_complete')
+    def test_delete_database_missing_project_id(self, wait_for_operation_to_complete, get_conn):
+        delete_method = get_conn.return_value.databases.return_value.delete
+        execute_method = delete_method.return_value.execute
+        execute_method.return_value = {"name": "operation_id"}
+        wait_for_operation_to_complete.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            self.cloudsql_hook_no_default_project_id.delete_database(
+                instance='instance',
+                database='database')
+        delete_method.assert_not_called()
+        execute_method.assert_not_called()
+        err = cm.exception
+        self.assertIn("The project id must be passed", str(err))
+        wait_for_operation_to_complete.assert_not_called()
+
+
+class TestCloudsqlDatabaseHook(unittest.TestCase):
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_validate_ssl_certs_no_ssl(self, get_connection):
+        connection = Connection()
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres"
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.validate_ssl_certs()
+
+    @parameterized.expand([
+        [{}],
+        [{"sslcert": "cert_file.pem"}],
+        [{"sslkey": "cert_key.pem"}],
+        [{"sslrootcert": "root_cert_file.pem"}],
+        [{"sslcert": "cert_file.pem", "sslkey": "cert_key.pem"}],
+        [{"sslrootcert": "root_cert_file.pem", "sslkey": "cert_key.pem"}],
+        [{"sslrootcert": "root_cert_file.pem", "sslcert": "cert_file.pem"}],
+    ])
+    @mock.patch('os.path.isfile')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_validate_ssl_certs_missing_cert_params(
+            self, cert_dict, get_connection, mock_is_file):
+        mock_is_file.side_effects = True
+        connection = Connection()
+        extras = {
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+            "use_ssl": "True"
+        }
+        extras.update(cert_dict)
+        connection.set_extra(json.dumps(extras))
+
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        with self.assertRaises(AirflowException) as cm:
+            hook.validate_ssl_certs()
+        err = cm.exception
+        self.assertIn("SSL connections requires", str(err))
+
+    @mock.patch('os.path.isfile')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_validate_ssl_certs_with_ssl(self, get_connection, mock_is_file):
+        connection = Connection()
+        mock_is_file.return_value = True
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+            "use_ssl": "True",
+            "sslcert": "cert_file.pem",
+            "sslrootcert": "rootcert_file.pem",
+            "sslkey": "key_file.pem",
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.validate_ssl_certs()
+
+    @mock.patch('os.path.isfile')
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_validate_ssl_certs_with_ssl_files_not_readable(
+            self, get_connection, mock_is_file):
+        connection = Connection()
+        mock_is_file.return_value = False
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+            "use_ssl": "True",
+            "sslcert": "cert_file.pem",
+            "sslrootcert": "rootcert_file.pem",
+            "sslkey": "key_file.pem",
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        with self.assertRaises(AirflowException) as cm:
+            hook.validate_ssl_certs()
+        err = cm.exception
+        self.assertIn("must be a readable file", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_validate_socket_path_length_too_long(self, get_connection):
+        connection = Connection()
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "very_long_instance_name_that_will_be_too_long_to_build_socket_length",
+            "database_type": "postgres",
+            "use_proxy": "True",
+            "use_tcp": "False"
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        with self.assertRaises(AirflowException) as cm:
+            hook.validate_socket_path_length()
+        err = cm.exception
+        self.assertIn("The UNIX socket path length cannot exceed", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_validate_socket_path_length_not_too_long(self, get_connection):
+        connection = Connection()
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "short_instance_name",
+            "database_type": "postgres",
+            "use_proxy": "True",
+            "use_tcp": "False"
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.validate_socket_path_length()
+
+    @parameterized.expand([
+        ["http://:password@host:80/database"],
+        ["http://user:@host:80/database"],
+        ["http://user:password@/database"],
+        ["http://user:password@host:80/"],
+        ["http://user:password@/"],
+        ["http://host:80/database"],
+        ["http://host:80/"],
+    ])
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_create_connection_missing_fields(self, uri, get_connection):
+        connection = Connection()
+        connection.parse_from_uri(uri)
+        params = {
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+            'use_proxy': "True",
+            'use_tcp': "False"
+        }
+        connection.set_extra(json.dumps(params))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        with self.assertRaises(AirflowException) as cm:
+            hook.create_connection()
+        err = cm.exception
+        self.assertIn("needs to be set in connection", str(err))
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_create_delete_connection(self, get_connection):
+        connection = Connection()
+        connection.parse_from_uri("http://user:password@host:80/database")
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres"
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.create_connection()
+        self.assertIsNotNone(hook.retrieve_connection())
+        hook.delete_connection()
+        self.assertIsNone(hook.retrieve_connection())
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_get_sqlproxy_runner_no_proxy(self, get_connection):
+        connection = Connection()
+        connection.parse_from_uri("http://user:password@host:80/database")
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.create_connection()
+        try:
+            with self.assertRaises(AirflowException) as cm:
+                hook.get_sqlproxy_runner()
+            err = cm.exception
+            self.assertIn('Proxy runner can only be retrieved in case of use_proxy = True', str(err))
+        finally:
+            hook.delete_connection()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_get_sqlproxy_runner(self, get_connection):
+        connection = Connection()
+        connection.parse_from_uri("http://user:password@host:80/database")
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+            'use_proxy': "True",
+            'use_tcp': "False"
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.create_connection()
+        try:
+            proxy_runner = hook.get_sqlproxy_runner()
+            self.assertIsNotNone(proxy_runner)
+        finally:
+            hook.delete_connection()
+
+    @mock.patch('airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook.get_connection')
+    def test_cloudsql_database_hook_get_database_hook(self, get_connection):
+        connection = Connection()
+        connection.parse_from_uri("http://user:password@host:80/database")
+        connection.set_extra(json.dumps({
+            "location": "test",
+            "instance": "instance",
+            "database_type": "postgres",
+        }))
+        get_connection.return_value = connection
+        hook = CloudSqlDatabaseHook(gcp_cloudsql_conn_id='cloudsql_connection',
+                                    default_gcp_project_id='google_connection')
+        hook.create_connection()
+        try:
+            db_hook = hook.get_database_hook()
+            self.assertIsNotNone(db_hook)
+        finally:
+            hook.delete_connection()
diff --git a/tests/contrib/hooks/test_gcp_transfer_hook.py b/tests/contrib/hooks/test_gcp_transfer_hook.py
new file mode 100644
index 0000000000000..ab0cfbf4f0454
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_transfer_hook.py
@@ -0,0 +1,594 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import unittest
+from copy import deepcopy
+
+import six
+from parameterized import parameterized
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_transfer_hook import (
+    GCPTransferServiceHook,
+    TIME_TO_SLEEP_IN_SECONDS,
+    GcpTransferOperationStatus,
+    GcpTransferJobsStatus,
+    PROJECT_ID,
+    STATUS,
+    DESCRIPTION,
+    FILTER_PROJECT_ID,
+    FILTER_JOB_NAMES,
+    TRANSFER_JOBS,
+    OPERATIONS,
+    METADATA,
+    TRANSFER_JOB,
+    TRANSFER_JOB_FIELD_MASK,
+)
+from tests.contrib.utils.base_gcp_mock import (
+    mock_base_gcp_hook_no_default_project_id,
+    mock_base_gcp_hook_default_project_id,
+)
+from tests.compat import mock
+
+NAME = "name"
+
+TEST_PROJECT_ID = 'project-id'
+TEST_BODY = {DESCRIPTION: 'AAA', PROJECT_ID: TEST_PROJECT_ID}
+
+TEST_TRANSFER_JOB_NAME = "transfer-job"
+TEST_TRANSFER_OPERATION_NAME = "transfer-operation"
+
+TEST_TRANSFER_JOB = {NAME: TEST_TRANSFER_JOB_NAME}
+TEST_TRANSFER_OPERATION = {NAME: TEST_TRANSFER_OPERATION_NAME}
+
+TEST_TRANSFER_JOB_FILTER = {FILTER_PROJECT_ID: TEST_PROJECT_ID, FILTER_JOB_NAMES: [TEST_TRANSFER_JOB_NAME]}
+TEST_TRANSFER_OPERATION_FILTER = {
+    FILTER_PROJECT_ID: TEST_PROJECT_ID,
+    FILTER_JOB_NAMES: [TEST_TRANSFER_JOB_NAME],
+}
+TEST_UPDATE_TRANSFER_JOB_BODY = {
+    TRANSFER_JOB: {DESCRIPTION: 'description-1'},
+    PROJECT_ID: TEST_PROJECT_ID,
+    TRANSFER_JOB_FIELD_MASK: 'description',
+}
+
+
+def _without_key(body, key):
+    obj = deepcopy(body)
+    del obj[key]
+    return obj
+
+
+class TestGCPTransferServiceHookWithPassedProjectId(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(
+            'airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+            new=mock_base_gcp_hook_no_default_project_id,
+        ):
+            self.gct_hook = GCPTransferServiceHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_create_transfer_job(self, get_conn):
+        create_method = get_conn.return_value.transferJobs.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        res = self.gct_hook.create_transfer_job(body=TEST_BODY)
+        self.assertEqual(res, TEST_TRANSFER_JOB)
+        create_method.assert_called_once_with(body=TEST_BODY)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_get_transfer_job(self, get_conn):
+        get_method = get_conn.return_value.transferJobs.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        res = self.gct_hook.get_transfer_job(job_name=TEST_TRANSFER_JOB_NAME, project_id=TEST_PROJECT_ID)
+        self.assertIsNotNone(res)
+        self.assertEqual(TEST_TRANSFER_JOB_NAME, res[NAME])
+        get_method.assert_called_once_with(jobName=TEST_TRANSFER_JOB_NAME, projectId=TEST_PROJECT_ID)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_job(self, get_conn):
+        list_method = get_conn.return_value.transferJobs.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {TRANSFER_JOBS: [TEST_TRANSFER_JOB]}
+
+        list_next = get_conn.return_value.transferJobs.return_value.list_next
+        list_next.return_value = None
+
+        res = self.gct_hook.list_transfer_job(filter=TEST_TRANSFER_JOB_FILTER)
+        self.assertIsNotNone(res)
+        self.assertEqual(res, [TEST_TRANSFER_JOB])
+        list_method.assert_called_once_with(filter=mock.ANY)
+        args, kwargs = list_method.call_args_list[0]
+        self.assertEqual(
+            json.loads(kwargs['filter']),
+            {FILTER_PROJECT_ID: TEST_PROJECT_ID, FILTER_JOB_NAMES: [TEST_TRANSFER_JOB_NAME]},
+        )
+        list_execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_update_transfer_job(self, get_conn):
+        update_method = get_conn.return_value.transferJobs.return_value.patch
+        execute_method = update_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        res = self.gct_hook.update_transfer_job(
+            job_name=TEST_TRANSFER_JOB_NAME, body=TEST_UPDATE_TRANSFER_JOB_BODY
+        )
+        self.assertIsNotNone(res)
+        update_method.assert_called_once_with(
+            jobName=TEST_TRANSFER_JOB_NAME, body=TEST_UPDATE_TRANSFER_JOB_BODY
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_delete_transfer_job(self, get_conn):
+        update_method = get_conn.return_value.transferJobs.return_value.patch
+        execute_method = update_method.return_value.execute
+
+        self.gct_hook.delete_transfer_job(job_name=TEST_TRANSFER_JOB_NAME, project_id=TEST_PROJECT_ID)
+
+        update_method.assert_called_once_with(
+            jobName=TEST_TRANSFER_JOB_NAME,
+            body={
+                PROJECT_ID: TEST_PROJECT_ID,
+                TRANSFER_JOB: {STATUS: GcpTransferJobsStatus.DELETED},
+                TRANSFER_JOB_FIELD_MASK: STATUS,
+            },
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_cancel_transfer_operation(self, get_conn):
+        cancel_method = get_conn.return_value.transferOperations.return_value.cancel
+        execute_method = cancel_method.return_value.execute
+
+        self.gct_hook.cancel_transfer_operation(operation_name=TEST_TRANSFER_OPERATION_NAME)
+        cancel_method.assert_called_once_with(name=TEST_TRANSFER_OPERATION_NAME)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_get_transfer_operation(self, get_conn):
+        get_method = get_conn.return_value.transferOperations.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_OPERATION
+        res = self.gct_hook.get_transfer_operation(operation_name=TEST_TRANSFER_OPERATION_NAME)
+        self.assertEqual(res, TEST_TRANSFER_OPERATION)
+        get_method.assert_called_once_with(name=TEST_TRANSFER_OPERATION_NAME)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_operation(self, get_conn):
+        list_method = get_conn.return_value.transferOperations.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {OPERATIONS: [TEST_TRANSFER_OPERATION]}
+
+        list_next = get_conn.return_value.transferOperations.return_value.list_next
+        list_next.return_value = None
+
+        res = self.gct_hook.list_transfer_operations(filter=TEST_TRANSFER_OPERATION_FILTER)
+        self.assertIsNotNone(res)
+        self.assertEqual(res, [TEST_TRANSFER_OPERATION])
+        list_method.assert_called_once_with(filter=mock.ANY, name='transferOperations')
+        args, kwargs = list_method.call_args_list[0]
+        self.assertEqual(
+            json.loads(kwargs['filter']),
+            {FILTER_PROJECT_ID: TEST_PROJECT_ID, FILTER_JOB_NAMES: [TEST_TRANSFER_JOB_NAME]},
+        )
+        list_execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_pause_transfer_operation(self, get_conn):
+        pause_method = get_conn.return_value.transferOperations.return_value.pause
+        execute_method = pause_method.return_value.execute
+
+        self.gct_hook.pause_transfer_operation(operation_name=TEST_TRANSFER_OPERATION_NAME)
+        pause_method.assert_called_once_with(name=TEST_TRANSFER_OPERATION_NAME)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_resume_transfer_operation(self, get_conn):
+        resume_method = get_conn.return_value.transferOperations.return_value.resume
+        execute_method = resume_method.return_value.execute
+
+        self.gct_hook.resume_transfer_operation(operation_name=TEST_TRANSFER_OPERATION_NAME)
+        resume_method.assert_called_once_with(name=TEST_TRANSFER_OPERATION_NAME)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('time.sleep')
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_wait_for_transfer_job(self, get_conn, mock_sleep):
+        list_method = get_conn.return_value.transferOperations.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.side_effect = [
+            {OPERATIONS: [{METADATA: {STATUS: GcpTransferOperationStatus.IN_PROGRESS}}]},
+            {OPERATIONS: [{METADATA: {STATUS: GcpTransferOperationStatus.SUCCESS}}]},
+        ]
+        get_conn.return_value.transferOperations.return_value.list_next.return_value = None
+
+        self.gct_hook.wait_for_transfer_job({PROJECT_ID: TEST_PROJECT_ID, 'name': 'transferJobs/test-job'})
+        self.assertTrue(list_method.called)
+
+        list_method.assert_called_with(name='transferOperations', filter=mock.ANY)
+        args, kwargs = list_method.call_args_list[0]
+        self.assertEqual(
+            json.loads(kwargs['filter']),
+            {FILTER_PROJECT_ID: TEST_PROJECT_ID, FILTER_JOB_NAMES: ["transferJobs/test-job"]},
+        )
+
+        mock_sleep.assert_called_once_with(TIME_TO_SLEEP_IN_SECONDS)
+
+    @mock.patch('time.sleep')
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_wait_for_transfer_job_failed(self, get_conn, mock_sleep):
+        list_method = get_conn.return_value.transferOperations.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {
+            OPERATIONS: [
+                {NAME: TEST_TRANSFER_OPERATION_NAME, METADATA: {STATUS: GcpTransferOperationStatus.FAILED}}
+            ]
+        }
+
+        get_conn.return_value.transferOperations.return_value.list_next.return_value = None
+
+        with self.assertRaises(AirflowException):
+            self.gct_hook.wait_for_transfer_job({PROJECT_ID: TEST_PROJECT_ID, NAME: 'transferJobs/test-job'})
+            self.assertTrue(list_method.called)
+
+    @mock.patch('time.sleep')
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_wait_for_transfer_job_expect_failed(self, get_conn, mock_sleep):
+        list_method = get_conn.return_value.transferOperations.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {
+            OPERATIONS: [
+                {NAME: TEST_TRANSFER_OPERATION_NAME, METADATA: {STATUS: GcpTransferOperationStatus.FAILED}}
+            ]
+        }
+
+        get_conn.return_value.transferOperations.return_value.list_next.return_value = None
+        with six.assertRaisesRegex(
+            self, AirflowException, "An unexpected operation status was encountered. Expected: SUCCESS"
+        ):
+            self.gct_hook.wait_for_transfer_job(
+                job={PROJECT_ID: 'test-project', NAME: 'transferJobs/test-job'},
+                expected_statuses=GcpTransferOperationStatus.SUCCESS,
+            )
+
+    @parameterized.expand(
+        [
+            ([GcpTransferOperationStatus.ABORTED], (GcpTransferOperationStatus.IN_PROGRESS,)),
+            (
+                [GcpTransferOperationStatus.SUCCESS, GcpTransferOperationStatus.ABORTED],
+                (GcpTransferOperationStatus.IN_PROGRESS,),
+            ),
+            (
+                [GcpTransferOperationStatus.PAUSED, GcpTransferOperationStatus.ABORTED],
+                (GcpTransferOperationStatus.IN_PROGRESS,),
+            ),
+            ([GcpTransferOperationStatus.ABORTED], (GcpTransferOperationStatus.IN_PROGRESS,)),
+            (
+                [GcpTransferOperationStatus.SUCCESS, GcpTransferOperationStatus.ABORTED],
+                (GcpTransferOperationStatus.IN_PROGRESS,),
+            ),
+            (
+                [GcpTransferOperationStatus.PAUSED, GcpTransferOperationStatus.ABORTED],
+                (GcpTransferOperationStatus.IN_PROGRESS,),
+            ),
+        ]
+    )
+    def test_operations_contain_expected_statuses_red_path(self, statuses, expected_statuses):
+        operations = [{NAME: TEST_TRANSFER_OPERATION_NAME, METADATA: {STATUS: status}} for status in statuses]
+
+        with six.assertRaisesRegex(
+            self,
+            AirflowException,
+            "An unexpected operation status was encountered. Expected: {}".format(
+                ", ".join(expected_statuses)
+            ),
+        ):
+            GCPTransferServiceHook.operations_contain_expected_statuses(
+                operations, GcpTransferOperationStatus.IN_PROGRESS
+            )
+
+    @parameterized.expand(
+        [
+            ([GcpTransferOperationStatus.ABORTED], GcpTransferOperationStatus.ABORTED),
+            (
+                [GcpTransferOperationStatus.SUCCESS, GcpTransferOperationStatus.ABORTED],
+                GcpTransferOperationStatus.ABORTED,
+            ),
+            (
+                [GcpTransferOperationStatus.PAUSED, GcpTransferOperationStatus.ABORTED],
+                GcpTransferOperationStatus.ABORTED,
+            ),
+            ([GcpTransferOperationStatus.ABORTED], (GcpTransferOperationStatus.ABORTED,)),
+            (
+                [GcpTransferOperationStatus.SUCCESS, GcpTransferOperationStatus.ABORTED],
+                (GcpTransferOperationStatus.ABORTED,),
+            ),
+            (
+                [GcpTransferOperationStatus.PAUSED, GcpTransferOperationStatus.ABORTED],
+                (GcpTransferOperationStatus.ABORTED,),
+            ),
+        ]
+    )
+    def test_operations_contain_expected_statuses_green_path(self, statuses, expected_statuses):
+        operations = [{NAME: TEST_TRANSFER_OPERATION_NAME, METADATA: {STATUS: status}} for status in statuses]
+
+        result = GCPTransferServiceHook.operations_contain_expected_statuses(operations, expected_statuses)
+
+        self.assertTrue(result)
+
+
+class TestGCPTransferServiceHookWithProjectIdFromConnection(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(
+            'airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+            new=mock_base_gcp_hook_default_project_id,
+        ):
+            self.gct_hook = GCPTransferServiceHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_create_transfer_job(self, get_conn):
+        create_method = get_conn.return_value.transferJobs.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = deepcopy(TEST_TRANSFER_JOB)
+        res = self.gct_hook.create_transfer_job(body=self._without_project_id(TEST_BODY))
+        self.assertEqual(res, TEST_TRANSFER_JOB)
+        create_method.assert_called_once_with(body=self._with_project_id(TEST_BODY, 'example-project'))
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_get_transfer_job(self, get_conn):
+        get_method = get_conn.return_value.transferJobs.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        res = self.gct_hook.get_transfer_job(job_name=TEST_TRANSFER_JOB_NAME)
+        self.assertIsNotNone(res)
+        self.assertEqual(TEST_TRANSFER_JOB_NAME, res[NAME])
+        get_method.assert_called_once_with(jobName=TEST_TRANSFER_JOB_NAME, projectId='example-project')
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_job(self, get_conn):
+        list_method = get_conn.return_value.transferJobs.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {TRANSFER_JOBS: [TEST_TRANSFER_JOB]}
+
+        list_next = get_conn.return_value.transferJobs.return_value.list_next
+        list_next.return_value = None
+
+        res = self.gct_hook.list_transfer_job(
+            filter=_without_key(TEST_TRANSFER_JOB_FILTER, FILTER_PROJECT_ID)
+        )
+        self.assertIsNotNone(res)
+        self.assertEqual(res, [TEST_TRANSFER_JOB])
+
+        list_method.assert_called_with(filter=mock.ANY)
+        args, kwargs = list_method.call_args_list[0]
+        self.assertEqual(
+            json.loads(kwargs['filter']),
+            {FILTER_PROJECT_ID: 'example-project', FILTER_JOB_NAMES: [TEST_TRANSFER_JOB_NAME]},
+        )
+        list_execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_update_transfer_job(self, get_conn):
+        update_method = get_conn.return_value.transferJobs.return_value.patch
+        execute_method = update_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        res = self.gct_hook.update_transfer_job(
+            job_name=TEST_TRANSFER_JOB_NAME, body=self._without_project_id(TEST_UPDATE_TRANSFER_JOB_BODY)
+        )
+        self.assertIsNotNone(res)
+        update_method.assert_called_once_with(
+            jobName=TEST_TRANSFER_JOB_NAME,
+            body=self._with_project_id(TEST_UPDATE_TRANSFER_JOB_BODY, 'example-project'),
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_delete_transfer_job(self, get_conn):
+        update_method = get_conn.return_value.transferJobs.return_value.patch
+        execute_method = update_method.return_value.execute
+
+        self.gct_hook.delete_transfer_job(job_name=TEST_TRANSFER_JOB_NAME, project_id=TEST_PROJECT_ID)
+
+        update_method.assert_called_once_with(
+            jobName=TEST_TRANSFER_JOB_NAME,
+            body={
+                PROJECT_ID: TEST_PROJECT_ID,
+                TRANSFER_JOB: {STATUS: 'DELETED'},
+                TRANSFER_JOB_FIELD_MASK: STATUS,
+            },
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_cancel_transfer_operation(self, get_conn):
+        cancel_method = get_conn.return_value.transferOperations.return_value.cancel
+        execute_method = cancel_method.return_value.execute
+
+        self.gct_hook.cancel_transfer_operation(operation_name=TEST_TRANSFER_OPERATION_NAME)
+        cancel_method.assert_called_once_with(name=TEST_TRANSFER_OPERATION_NAME)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_get_transfer_operation(self, get_conn):
+        get_method = get_conn.return_value.transferOperations.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_OPERATION
+        res = self.gct_hook.get_transfer_operation(operation_name=TEST_TRANSFER_OPERATION_NAME)
+        self.assertEqual(res, TEST_TRANSFER_OPERATION)
+        get_method.assert_called_once_with(name=TEST_TRANSFER_OPERATION_NAME)
+        execute_method.assert_called_once_with(num_retries=5)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_operation(self, get_conn):
+        list_method = get_conn.return_value.transferOperations.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {OPERATIONS: [TEST_TRANSFER_OPERATION]}
+
+        list_next = get_conn.return_value.transferOperations.return_value.list_next
+        list_next.return_value = None
+
+        res = self.gct_hook.list_transfer_operations(
+            filter=_without_key(TEST_TRANSFER_OPERATION_FILTER, FILTER_PROJECT_ID)
+        )
+        self.assertIsNotNone(res)
+        self.assertEqual(res, [TEST_TRANSFER_OPERATION])
+        list_method.assert_called_once_with(filter=mock.ANY, name='transferOperations')
+        args, kwargs = list_method.call_args_list[0]
+        self.assertEqual(
+            json.loads(kwargs['filter']),
+            {FILTER_PROJECT_ID: 'example-project', FILTER_JOB_NAMES: [TEST_TRANSFER_JOB_NAME]},
+        )
+        list_execute_method.assert_called_once_with(num_retries=5)
+
+    @staticmethod
+    def _without_project_id(body):
+        body = deepcopy(body)
+        del body[PROJECT_ID]
+        return body
+
+    @staticmethod
+    def _with_project_id(body, project_id):
+        body = deepcopy(body)
+        del body[PROJECT_ID]
+        body[PROJECT_ID] = project_id
+        return body
+
+
+class TestGCPTransferServiceHookWithoutProjectId(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(
+            'airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.__init__',
+            new=mock_base_gcp_hook_no_default_project_id,
+        ):
+            self.gct_hook = GCPTransferServiceHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_create_transfer_job(self, get_conn):
+        create_method = get_conn.return_value.transferJobs.return_value.create
+        execute_method = create_method.return_value.execute
+        execute_method.return_value = deepcopy(TEST_TRANSFER_JOB)
+        with self.assertRaises(AirflowException) as e:
+            self.gct_hook.create_transfer_job(body=_without_key(TEST_BODY, PROJECT_ID))
+
+        self.assertEqual(
+            'The project id must be passed either as `projectId` key in `body` parameter or as project_id '
+            'extra in GCP connection definition. Both are not set!',
+            str(e.exception),
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_get_transfer_job(self, get_conn):
+        get_method = get_conn.return_value.transferJobs.return_value.get
+        execute_method = get_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        with self.assertRaises(AirflowException) as e:
+            self.gct_hook.get_transfer_job(job_name=TEST_TRANSFER_JOB_NAME)
+        self.assertEqual(
+            'The project id must be passed either as keyword project_id '
+            'parameter or as project_id extra in GCP connection definition. '
+            'Both are not set!',
+            str(e.exception),
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_job(self, get_conn):
+        list_method = get_conn.return_value.transferJobs.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {"transferJobs": [TEST_TRANSFER_JOB]}
+
+        list_next = get_conn.return_value.transferJobs.return_value.list_next
+        list_next.return_value = None
+
+        with self.assertRaises(AirflowException) as e:
+            self.gct_hook.list_transfer_job(filter=_without_key(TEST_TRANSFER_JOB_FILTER, FILTER_PROJECT_ID))
+
+        self.assertEqual(
+            'The project id must be passed either as `project_id` key in `filter` parameter or as '
+            'project_id extra in GCP connection definition. Both are not set!',
+            str(e.exception),
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_operation_multiple_page(self, get_conn):
+        pages_requests = [
+            mock.Mock(**{'execute.return_value': {"operations": [TEST_TRANSFER_OPERATION]}}) for _ in range(4)
+        ]
+        transfer_operation_mock = mock.Mock(
+            **{'list.return_value': pages_requests[1], 'list_next.side_effect': pages_requests[1:] + [None]}
+        )
+
+        get_conn.return_value.transferOperations.return_value = transfer_operation_mock
+
+        res = self.gct_hook.list_transfer_operations(filter=TEST_TRANSFER_OPERATION_FILTER)
+        self.assertEqual(res, [TEST_TRANSFER_OPERATION] * 4)
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_update_transfer_job(self, get_conn):
+        update_method = get_conn.return_value.transferJobs.return_value.patch
+        execute_method = update_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_JOB
+        with self.assertRaises(AirflowException) as e:
+            self.gct_hook.update_transfer_job(
+                job_name=TEST_TRANSFER_JOB_NAME, body=_without_key(TEST_UPDATE_TRANSFER_JOB_BODY, PROJECT_ID)
+            )
+
+        self.assertEqual(
+            'The project id must be passed either as `projectId` key in `body` parameter or as project_id '
+            'extra in GCP connection definition. Both are not set!',
+            str(e.exception),
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_delete_transfer_job(self, get_conn):
+        with self.assertRaises(AirflowException) as e:
+            self.gct_hook.delete_transfer_job(job_name=TEST_TRANSFER_JOB_NAME)
+
+        self.assertEqual(
+            'The project id must be passed either as keyword project_id parameter or as project_id extra in '
+            'GCP connection definition. Both are not set!',
+            str(e.exception),
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook.get_conn')
+    def test_list_transfer_operation(self, get_conn):
+        list_method = get_conn.return_value.transferOperations.return_value.list
+        list_execute_method = list_method.return_value.execute
+        list_execute_method.return_value = {"operations": [TEST_TRANSFER_OPERATION]}
+
+        list_next = get_conn.return_value.transferOperations.return_value.list_next
+        list_next.return_value = None
+
+        with self.assertRaises(AirflowException) as e:
+            self.gct_hook.list_transfer_operations(
+                filter=_without_key(TEST_TRANSFER_OPERATION_FILTER, FILTER_PROJECT_ID)
+            )
+
+        self.assertEqual(
+            'The project id must be passed either as `project_id` key in `filter` parameter or as project_id '
+            'extra in GCP connection definition. Both are not set!',
+            str(e.exception),
+        )
diff --git a/tests/contrib/hooks/test_gcp_translate_hook.py b/tests/contrib/hooks/test_gcp_translate_hook.py
new file mode 100644
index 0000000000000..003e65e4de15b
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_translate_hook.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow.contrib.hooks.gcp_translate_hook import CloudTranslateHook
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id
+from tests.compat import mock
+
+PROJECT_ID_TEST = 'project-id'
+
+
+class TestCloudTranslateHook(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(
+            'airflow.contrib.hooks.gcp_translate_hook.CloudTranslateHook.__init__',
+            new=mock_base_gcp_hook_default_project_id,
+        ):
+            self.hook = CloudTranslateHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_translate_hook.CloudTranslateHook.get_conn')
+    def test_translate_called(self, get_conn):
+        # Given
+        translate_method = get_conn.return_value.translate
+        translate_method.return_value = {
+            'translatedText': 'Yellowing self Gęśle',
+            'detectedSourceLanguage': 'pl',
+            'model': 'base',
+            'input': 'zażółć gęślą jaźń',
+        }
+        # When
+        result = self.hook.translate(
+            values=['zażółć gęślą jaźń'],
+            target_language='en',
+            format_='text',
+            source_language=None,
+            model='base',
+        )
+        # Then
+        self.assertEqual(
+            result,
+            {
+                'translatedText': 'Yellowing self Gęśle',
+                'detectedSourceLanguage': 'pl',
+                'model': 'base',
+                'input': 'zażółć gęślą jaźń',
+            },
+        )
+        translate_method.assert_called_once_with(
+            values=['zażółć gęślą jaźń'],
+            target_language='en',
+            format_='text',
+            source_language=None,
+            model='base',
+        )
diff --git a/tests/contrib/hooks/test_gcp_vision_hook.py b/tests/contrib/hooks/test_gcp_vision_hook.py
new file mode 100644
index 0000000000000..2fcef744bdbef
--- /dev/null
+++ b/tests/contrib/hooks/test_gcp_vision_hook.py
@@ -0,0 +1,649 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from google.cloud.vision import enums
+from google.cloud.vision_v1 import ProductSearchClient
+from google.cloud.vision_v1.proto.product_search_service_pb2 import ProductSet, Product, ReferenceImage
+from google.protobuf.json_format import MessageToDict
+from parameterized import parameterized
+
+from airflow.contrib.hooks.gcp_vision_hook import CloudVisionHook
+from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_default_project_id
+from tests.compat import mock
+
+from airflow import AirflowException
+
+PROJECT_ID_TEST = 'project-id'
+PROJECT_ID_TEST_2 = 'project-id-2'
+LOC_ID_TEST = 'loc-id'
+LOC_ID_TEST_2 = 'loc-id-2'
+PRODUCTSET_ID_TEST = 'ps-id'
+PRODUCTSET_ID_TEST_2 = 'ps-id-2'
+PRODUCTSET_NAME_TEST = 'projects/{}/locations/{}/productSets/{}'.format(
+    PROJECT_ID_TEST, LOC_ID_TEST, PRODUCTSET_ID_TEST
+)
+PRODUCT_ID_TEST = 'p-id'
+PRODUCT_ID_TEST_2 = 'p-id-2'
+PRODUCT_NAME_TEST = "projects/{}/locations/{}/products/{}".format(
+    PROJECT_ID_TEST, LOC_ID_TEST, PRODUCT_ID_TEST
+)
+PRODUCT_NAME = "projects/{}/locations/{}/products/{}".format(PROJECT_ID_TEST, LOC_ID_TEST, PRODUCT_ID_TEST)
+REFERENCE_IMAGE_ID_TEST = 'ri-id'
+REFERENCE_IMAGE_GEN_ID_TEST = 'ri-id'
+ANNOTATE_IMAGE_REQUEST = {
+    'image': {'source': {'image_uri': "gs://bucket-name/object-name"}},
+    'features': [{'type': enums.Feature.Type.LOGO_DETECTION}],
+}
+REFERENCE_IMAGE_NAME_TEST = "projects/{}/locations/{}/products/{}/referenceImages/{}".format(
+    PROJECT_ID_TEST, LOC_ID_TEST, PRODUCTSET_ID_TEST, REFERENCE_IMAGE_ID_TEST
+)
+REFERENCE_IMAGE_TEST = ReferenceImage(name=REFERENCE_IMAGE_GEN_ID_TEST)
+REFERENCE_IMAGE_WITHOUT_ID_NAME = ReferenceImage()
+
+
+class TestGcpVisionHook(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(
+            'airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.__init__',
+            new=mock_base_gcp_hook_default_project_id,
+        ):
+            self.hook = CloudVisionHook(gcp_conn_id='test')
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_productset_explicit_id(self, get_conn):
+        # Given
+        create_product_set_method = get_conn.return_value.create_product_set
+        create_product_set_method.return_value = None
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product_set = ProductSet()
+        # When
+        result = self.hook.create_product_set(
+            location=LOC_ID_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_set=product_set,
+            project_id=PROJECT_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+        # Then
+        # ProductSet ID was provided explicitly in the method call above, should be returned from the method
+        self.assertEqual(result, PRODUCTSET_ID_TEST)
+        create_product_set_method.assert_called_once_with(
+            parent=parent,
+            product_set=product_set,
+            product_set_id=PRODUCTSET_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_productset_autogenerated_id(self, get_conn):
+        # Given
+        autogenerated_id = 'autogen-id'
+        response_product_set = ProductSet(
+            name=ProductSearchClient.product_set_path(PROJECT_ID_TEST, LOC_ID_TEST, autogenerated_id)
+        )
+        create_product_set_method = get_conn.return_value.create_product_set
+        create_product_set_method.return_value = response_product_set
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product_set = ProductSet()
+        # When
+        result = self.hook.create_product_set(
+            location=LOC_ID_TEST, product_set_id=None, product_set=product_set, project_id=PROJECT_ID_TEST
+        )
+        # Then
+        # ProductSet ID was not provided in the method call above. Should be extracted from the API response
+        # and returned.
+        self.assertEqual(result, autogenerated_id)
+        create_product_set_method.assert_called_once_with(
+            parent=parent,
+            product_set=product_set,
+            product_set_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_productset_autogenerated_id_wrong_api_response(self, get_conn):
+        # Given
+        response_product_set = None
+        create_product_set_method = get_conn.return_value.create_product_set
+        create_product_set_method.return_value = response_product_set
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product_set = ProductSet()
+        # When
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.create_product_set(
+                location=LOC_ID_TEST,
+                product_set_id=None,
+                product_set=product_set,
+                project_id=PROJECT_ID_TEST,
+                retry=None,
+                timeout=None,
+                metadata=None,
+            )
+        # Then
+        # API response was wrong (None) and thus ProductSet ID extraction should fail.
+        err = cm.exception
+        self.assertIn('Unable to get name from response...', str(err))
+        create_product_set_method.assert_called_once_with(
+            parent=parent,
+            product_set=product_set,
+            product_set_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_get_productset(self, get_conn):
+        # Given
+        name = ProductSearchClient.product_set_path(PROJECT_ID_TEST, LOC_ID_TEST, PRODUCTSET_ID_TEST)
+        response_product_set = ProductSet(name=name)
+        get_product_set_method = get_conn.return_value.get_product_set
+        get_product_set_method.return_value = response_product_set
+        # When
+        response = self.hook.get_product_set(
+            location=LOC_ID_TEST, product_set_id=PRODUCTSET_ID_TEST, project_id=PROJECT_ID_TEST
+        )
+        # Then
+        self.assertTrue(response)
+        self.assertEqual(response, MessageToDict(response_product_set))
+        get_product_set_method.assert_called_once_with(name=name, retry=None, timeout=None, metadata=None)
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_productset_no_explicit_name(self, get_conn):
+        # Given
+        product_set = ProductSet()
+        update_product_set_method = get_conn.return_value.update_product_set
+        update_product_set_method.return_value = product_set
+        productset_name = ProductSearchClient.product_set_path(
+            PROJECT_ID_TEST, LOC_ID_TEST, PRODUCTSET_ID_TEST
+        )
+        # When
+        result = self.hook.update_product_set(
+            location=LOC_ID_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_set=product_set,
+            update_mask=None,
+            project_id=PROJECT_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+        # Then
+        self.assertEqual(result, MessageToDict(product_set))
+        update_product_set_method.assert_called_once_with(
+            product_set=ProductSet(name=productset_name),
+            metadata=None,
+            retry=None,
+            timeout=None,
+            update_mask=None,
+        )
+
+    @parameterized.expand([(None, None), (None, PRODUCTSET_ID_TEST), (LOC_ID_TEST, None)])
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_productset_no_explicit_name_and_missing_params_for_constructed_name(
+        self, location, product_set_id, get_conn
+    ):
+        # Given
+        update_product_set_method = get_conn.return_value.update_product_set
+        update_product_set_method.return_value = None
+        product_set = ProductSet()
+        # When
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.update_product_set(
+                location=location,
+                product_set_id=product_set_id,
+                product_set=product_set,
+                update_mask=None,
+                project_id=PROJECT_ID_TEST,
+                retry=None,
+                timeout=None,
+                metadata=None,
+            )
+        err = cm.exception
+        self.assertTrue(err)
+        self.assertIn(
+            "Unable to determine the ProductSet name. Please either set the name directly in the "
+            "ProductSet object or provide the `location` and `productset_id` parameters.",
+            str(err),
+        )
+        update_product_set_method.assert_not_called()
+
+    @parameterized.expand([(None, None), (None, PRODUCTSET_ID_TEST), (LOC_ID_TEST, None)])
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_productset_explicit_name_missing_params_for_constructed_name(
+        self, location, product_set_id, get_conn
+    ):
+        # Given
+        explicit_ps_name = ProductSearchClient.product_set_path(
+            PROJECT_ID_TEST_2, LOC_ID_TEST_2, PRODUCTSET_ID_TEST_2
+        )
+        product_set = ProductSet(name=explicit_ps_name)
+        update_product_set_method = get_conn.return_value.update_product_set
+        update_product_set_method.return_value = product_set
+        # When
+        result = self.hook.update_product_set(
+            location=location,
+            product_set_id=product_set_id,
+            product_set=product_set,
+            update_mask=None,
+            project_id=PROJECT_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+        # Then
+        self.assertEqual(result, MessageToDict(product_set))
+        update_product_set_method.assert_called_once_with(
+            product_set=ProductSet(name=explicit_ps_name),
+            metadata=None,
+            retry=None,
+            timeout=None,
+            update_mask=None,
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_productset_explicit_name_different_from_constructed(self, get_conn):
+        # Given
+        update_product_set_method = get_conn.return_value.update_product_set
+        update_product_set_method.return_value = None
+        explicit_ps_name = ProductSearchClient.product_set_path(
+            PROJECT_ID_TEST_2, LOC_ID_TEST_2, PRODUCTSET_ID_TEST_2
+        )
+        product_set = ProductSet(name=explicit_ps_name)
+        template_ps_name = ProductSearchClient.product_set_path(
+            PROJECT_ID_TEST, LOC_ID_TEST, PRODUCTSET_ID_TEST
+        )
+        # When
+        # Location and product_set_id are passed in addition to a ProductSet with an explicit name,
+        # but both names differ (constructed != explicit).
+        # Should throw AirflowException in this case.
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.update_product_set(
+                location=LOC_ID_TEST,
+                product_set_id=PRODUCTSET_ID_TEST,
+                product_set=product_set,
+                update_mask=None,
+                project_id=PROJECT_ID_TEST,
+                retry=None,
+                timeout=None,
+                metadata=None,
+            )
+        err = cm.exception
+        # self.assertIn("The required parameter 'project_id' is missing", str(err))
+        self.assertTrue(err)
+        self.assertIn(
+            "The ProductSet name provided in the object ({}) is different than the name "
+            "created from the input parameters ({}). Please either: 1) Remove the ProductSet "
+            "name, 2) Remove the location and productset_id parameters, 3) Unify the "
+            "ProductSet name and input parameters.".format(explicit_ps_name, template_ps_name),
+            str(err),
+        )
+        update_product_set_method.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_delete_productset(self, get_conn):
+        # Given
+        delete_product_set_method = get_conn.return_value.delete_product_set
+        delete_product_set_method.return_value = None
+        name = ProductSearchClient.product_set_path(PROJECT_ID_TEST, LOC_ID_TEST, PRODUCTSET_ID_TEST)
+        # When
+        response = self.hook.delete_product_set(
+            location=LOC_ID_TEST, product_set_id=PRODUCTSET_ID_TEST, project_id=PROJECT_ID_TEST
+        )
+        # Then
+        self.assertIsNone(response)
+        delete_product_set_method.assert_called_once_with(name=name, retry=None, timeout=None, metadata=None)
+
+    @mock.patch(
+        'airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn',
+        **{'return_value.create_reference_image.return_value': REFERENCE_IMAGE_TEST}
+    )
+    def test_create_reference_image_explicit_id(self, get_conn):
+        # Given
+        create_reference_image_method = get_conn.return_value.create_reference_image
+
+        # When
+        result = self.hook.create_reference_image(
+            project_id=PROJECT_ID_TEST,
+            location=LOC_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            reference_image=REFERENCE_IMAGE_WITHOUT_ID_NAME,
+            reference_image_id=REFERENCE_IMAGE_ID_TEST,
+        )
+        # Then
+        # Product ID was provided explicitly in the method call above, should be returned from the method
+        self.assertEqual(result, REFERENCE_IMAGE_ID_TEST)
+        create_reference_image_method.assert_called_once_with(
+            parent=PRODUCT_NAME,
+            reference_image=REFERENCE_IMAGE_WITHOUT_ID_NAME,
+            reference_image_id=REFERENCE_IMAGE_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch(
+        'airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn',
+        **{'return_value.create_reference_image.return_value': REFERENCE_IMAGE_TEST}
+    )
+    def test_create_reference_image_autogenerated_id(self, get_conn):
+        # Given
+        create_reference_image_method = get_conn.return_value.create_reference_image
+
+        # When
+        result = self.hook.create_reference_image(
+            project_id=PROJECT_ID_TEST,
+            location=LOC_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            reference_image=REFERENCE_IMAGE_TEST,
+            reference_image_id=REFERENCE_IMAGE_ID_TEST,
+        )
+        # Then
+        # Product ID was provided explicitly in the method call above, should be returned from the method
+        self.assertEqual(result, REFERENCE_IMAGE_GEN_ID_TEST)
+        create_reference_image_method.assert_called_once_with(
+            parent=PRODUCT_NAME,
+            reference_image=REFERENCE_IMAGE_TEST,
+            reference_image_id=REFERENCE_IMAGE_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_add_product_to_product_set(self, get_conn):
+        # Given
+        add_product_to_product_set_method = get_conn.return_value.add_product_to_product_set
+
+        # When
+        self.hook.add_product_to_product_set(
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            location=LOC_ID_TEST,
+            project_id=PROJECT_ID_TEST,
+        )
+        # Then
+        # Product ID was provided explicitly in the method call above, should be returned from the method
+        add_product_to_product_set_method.assert_called_once_with(
+            name=PRODUCTSET_NAME_TEST, product=PRODUCT_NAME_TEST, retry=None, timeout=None, metadata=None
+        )
+
+    # remove_product_from_product_set
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_remove_product_from_product_set(self, get_conn):
+        # Given
+        remove_product_from_product_set_method = get_conn.return_value.remove_product_from_product_set
+
+        # When
+        self.hook.remove_product_from_product_set(
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            location=LOC_ID_TEST,
+            project_id=PROJECT_ID_TEST,
+        )
+        # Then
+        # Product ID was provided explicitly in the method call above, should be returned from the method
+        remove_product_from_product_set_method.assert_called_once_with(
+            name=PRODUCTSET_NAME_TEST, product=PRODUCT_NAME_TEST, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.annotator_client')
+    def test_annotate_image(self, annotator_client_mock):
+        # Given
+        annotate_image_method = annotator_client_mock.annotate_image
+
+        # When
+        self.hook.annotate_image(request=ANNOTATE_IMAGE_REQUEST)
+        # Then
+        # Product ID was provided explicitly in the method call above, should be returned from the method
+        annotate_image_method.assert_called_once_with(
+            request=ANNOTATE_IMAGE_REQUEST, retry=None, timeout=None
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_product_explicit_id(self, get_conn):
+        # Given
+        create_product_method = get_conn.return_value.create_product
+        create_product_method.return_value = None
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product = Product()
+        # When
+        result = self.hook.create_product(
+            location=LOC_ID_TEST, product_id=PRODUCT_ID_TEST, product=product, project_id=PROJECT_ID_TEST
+        )
+        # Then
+        # Product ID was provided explicitly in the method call above, should be returned from the method
+        self.assertEqual(result, PRODUCT_ID_TEST)
+        create_product_method.assert_called_once_with(
+            parent=parent,
+            product=product,
+            product_id=PRODUCT_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_product_autogenerated_id(self, get_conn):
+        # Given
+        autogenerated_id = 'autogen-p-id'
+        response_product = Product(
+            name=ProductSearchClient.product_path(PROJECT_ID_TEST, LOC_ID_TEST, autogenerated_id)
+        )
+        create_product_method = get_conn.return_value.create_product
+        create_product_method.return_value = response_product
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product = Product()
+        # When
+        result = self.hook.create_product(
+            location=LOC_ID_TEST, product_id=None, product=product, project_id=PROJECT_ID_TEST
+        )
+        # Then
+        # Product ID was not provided in the method call above. Should be extracted from the API response
+        # and returned.
+        self.assertEqual(result, autogenerated_id)
+        create_product_method.assert_called_once_with(
+            parent=parent, product=product, product_id=None, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_product_autogenerated_id_wrong_name_in_response(self, get_conn):
+        # Given
+        wrong_name = 'wrong_name_not_a_correct_path'
+        response_product = Product(name=wrong_name)
+        create_product_method = get_conn.return_value.create_product
+        create_product_method.return_value = response_product
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product = Product()
+        # When
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.create_product(
+                location=LOC_ID_TEST, product_id=None, product=product, project_id=PROJECT_ID_TEST
+            )
+        # Then
+        # API response was wrong (wrong name format) and thus ProductSet ID extraction should fail.
+        err = cm.exception
+        self.assertIn('Unable to get id from name', str(err))
+        create_product_method.assert_called_once_with(
+            parent=parent, product=product, product_id=None, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_create_product_autogenerated_id_wrong_api_response(self, get_conn):
+        # Given
+        response_product = None
+        create_product_method = get_conn.return_value.create_product
+        create_product_method.return_value = response_product
+        parent = ProductSearchClient.location_path(PROJECT_ID_TEST, LOC_ID_TEST)
+        product = Product()
+        # When
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.create_product(
+                location=LOC_ID_TEST, product_id=None, product=product, project_id=PROJECT_ID_TEST
+            )
+        # Then
+        # API response was wrong (None) and thus ProductSet ID extraction should fail.
+        err = cm.exception
+        self.assertIn('Unable to get name from response...', str(err))
+        create_product_method.assert_called_once_with(
+            parent=parent, product=product, product_id=None, retry=None, timeout=None, metadata=None
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_product_no_explicit_name(self, get_conn):
+        # Given
+        product = Product()
+        update_product_method = get_conn.return_value.update_product
+        update_product_method.return_value = product
+        product_name = ProductSearchClient.product_path(PROJECT_ID_TEST, LOC_ID_TEST, PRODUCT_ID_TEST)
+        # When
+        result = self.hook.update_product(
+            location=LOC_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            product=product,
+            update_mask=None,
+            project_id=PROJECT_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+        # Then
+        self.assertEqual(result, MessageToDict(product))
+        update_product_method.assert_called_once_with(
+            product=Product(name=product_name), metadata=None, retry=None, timeout=None, update_mask=None
+        )
+
+    @parameterized.expand([(None, None), (None, PRODUCT_ID_TEST), (LOC_ID_TEST, None)])
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_product_no_explicit_name_and_missing_params_for_constructed_name(
+        self, location, product_id, get_conn
+    ):
+        # Given
+        update_product_method = get_conn.return_value.update_product
+        update_product_method.return_value = None
+        product = Product()
+        # When
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.update_product(
+                location=location,
+                product_id=product_id,
+                product=product,
+                update_mask=None,
+                project_id=PROJECT_ID_TEST,
+                retry=None,
+                timeout=None,
+                metadata=None,
+            )
+        err = cm.exception
+        self.assertTrue(err)
+        self.assertIn(
+            "Unable to determine the Product name. Please either set the name directly in the "
+            "Product object or provide the `location` and `product_id` parameters.",
+            str(err),
+        )
+        update_product_method.assert_not_called()
+
+    @parameterized.expand([(None, None), (None, PRODUCT_ID_TEST), (LOC_ID_TEST, None)])
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_product_explicit_name_missing_params_for_constructed_name(
+        self, location, product_id, get_conn
+    ):
+        # Given
+        explicit_p_name = ProductSearchClient.product_path(
+            PROJECT_ID_TEST_2, LOC_ID_TEST_2, PRODUCT_ID_TEST_2
+        )
+        product = Product(name=explicit_p_name)
+        update_product_method = get_conn.return_value.update_product
+        update_product_method.return_value = product
+        # When
+        result = self.hook.update_product(
+            location=location,
+            product_id=product_id,
+            product=product,
+            update_mask=None,
+            project_id=PROJECT_ID_TEST,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+        # Then
+        self.assertEqual(result, MessageToDict(product))
+        update_product_method.assert_called_once_with(
+            product=Product(name=explicit_p_name), metadata=None, retry=None, timeout=None, update_mask=None
+        )
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_update_product_explicit_name_different_from_constructed(self, get_conn):
+        # Given
+        update_product_method = get_conn.return_value.update_product
+        update_product_method.return_value = None
+        explicit_p_name = ProductSearchClient.product_path(
+            PROJECT_ID_TEST_2, LOC_ID_TEST_2, PRODUCT_ID_TEST_2
+        )
+        product = Product(name=explicit_p_name)
+        template_p_name = ProductSearchClient.product_path(PROJECT_ID_TEST, LOC_ID_TEST, PRODUCT_ID_TEST)
+        # When
+        # Location and product_id are passed in addition to a Product with an explicit name,
+        # but both names differ (constructed != explicit).
+        # Should throw AirflowException in this case.
+        with self.assertRaises(AirflowException) as cm:
+            self.hook.update_product(
+                location=LOC_ID_TEST,
+                product_id=PRODUCT_ID_TEST,
+                product=product,
+                update_mask=None,
+                project_id=PROJECT_ID_TEST,
+                retry=None,
+                timeout=None,
+                metadata=None,
+            )
+        err = cm.exception
+        self.assertTrue(err)
+        self.assertIn(
+            "The Product name provided in the object ({}) is different than the name created from the input "
+            "parameters ({}). Please either: 1) Remove the Product name, 2) Remove the location and product_"
+            "id parameters, 3) Unify the Product name and input parameters.".format(
+                explicit_p_name, template_p_name
+            ),
+            str(err),
+        )
+        update_product_method.assert_not_called()
+
+    @mock.patch('airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook.get_conn')
+    def test_delete_product(self, get_conn):
+        # Given
+        delete_product_method = get_conn.return_value.delete_product
+        delete_product_method.return_value = None
+        name = ProductSearchClient.product_path(PROJECT_ID_TEST, LOC_ID_TEST, PRODUCT_ID_TEST)
+        # When
+        response = self.hook.delete_product(
+            location=LOC_ID_TEST, product_id=PRODUCT_ID_TEST, project_id=PROJECT_ID_TEST
+        )
+        # Then
+        self.assertIsNone(response)
+        delete_product_method.assert_called_once_with(name=name, retry=None, timeout=None, metadata=None)
diff --git a/tests/contrib/hooks/test_gcs_hook.py b/tests/contrib/hooks/test_gcs_hook.py
index fb65938fd9afe..faed4db6c0f09 100644
--- a/tests/contrib/hooks/test_gcs_hook.py
+++ b/tests/contrib/hooks/test_gcs_hook.py
@@ -18,19 +18,14 @@
 # under the License.
 
 import unittest
+import tempfile
+import os
 
 import airflow.contrib.hooks.gcs_hook as gcs_hook
 
 from airflow.exceptions import AirflowException
-from apiclient.errors import HttpError
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from googleapiclient.errors import HttpError
+from tests.compat import mock
 
 BASE_STRING = 'airflow.contrib.hooks.gcp_api_base_hook.{}'
 GCS_STRING = 'airflow.contrib.hooks.gcs_hook.{}'
@@ -66,14 +61,14 @@ class TestGCSBucket(unittest.TestCase):
     def test_bucket_name_value(self):
 
         bad_start_bucket_name = '/testing123'
-        with self.assertRaises(AssertionError):
+        with self.assertRaises(ValueError):
 
             gcs_hook.GoogleCloudStorageHook().create_bucket(
                 bucket_name=bad_start_bucket_name
             )
 
         bad_end_bucket_name = 'testing123/'
-        with self.assertRaises(AssertionError):
+        with self.assertRaises(ValueError):
             gcs_hook.GoogleCloudStorageHook().create_bucket(
                 bucket_name=bad_end_bucket_name
             )
@@ -206,7 +201,7 @@ def test_copy_fail_same_source_and_destination(self):
                                destination_bucket=destination_bucket,
                                destination_object=destination_object)
 
-        self.assertEquals(
+        self.assertEqual(
             str(e.exception),
             'Either source/destination bucket or source/destination object '
             'must be different, not both the same: bucket=%s, object=%s' %
@@ -226,7 +221,7 @@ def test_copy_empty_source_bucket(self):
                                destination_bucket=destination_bucket,
                                destination_object=destination_object)
 
-        self.assertEquals(
+        self.assertEqual(
             str(e.exception),
             'source_bucket and source_object cannot be empty.'
         )
@@ -244,7 +239,7 @@ def test_copy_empty_source_object(self):
                                destination_bucket=destination_bucket,
                                destination_object=destination_object)
 
-        self.assertEquals(
+        self.assertEqual(
             str(e.exception),
             'source_bucket and source_object cannot be empty.'
         )
@@ -340,3 +335,291 @@ def test_delete_nonexisting_object(self, mock_service):
         response = self.gcs_hook.delete(bucket=test_bucket, object=test_object)
 
         self.assertFalse(response)
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_create_bucket(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_project = 'test-project'
+        test_location = 'EU',
+        test_labels = {'env': 'prod'}
+        test_storage_class = 'MULTI_REGIONAL'
+        test_response_id = "{}/0123456789012345".format(test_bucket)
+
+        (mock_service.return_value.buckets.return_value
+         .insert.return_value.execute.return_value) = {"id": test_response_id}
+
+        response = self.gcs_hook.create_bucket(
+            bucket_name=test_bucket,
+            storage_class=test_storage_class,
+            location=test_location,
+            labels=test_labels,
+            project_id=test_project
+        )
+
+        self.assertEqual(response, test_response_id)
+        mock_service.return_value.buckets.return_value.insert.assert_called_with(
+            project=test_project,
+            body={
+                'name': test_bucket,
+                'location': test_location,
+                'storageClass': test_storage_class,
+                'labels': test_labels
+            }
+        )
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_create_bucket_with_resource(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_project = 'test-project'
+        test_location = 'EU',
+        test_labels = {'env': 'prod'}
+        test_storage_class = 'MULTI_REGIONAL'
+        test_response_id = "{}/0123456789012345".format(test_bucket)
+        test_lifecycle = {"rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}]}
+
+        (mock_service.return_value.buckets.return_value
+         .insert.return_value.execute.return_value) = {"id": test_response_id}
+
+        # Assert for resource other than None.
+        response = self.gcs_hook.create_bucket(
+            bucket_name=test_bucket,
+            resource={"lifecycle": {"rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}]}},
+            storage_class=test_storage_class,
+            location=test_location,
+            labels=test_labels,
+            project_id=test_project
+        )
+
+        self.assertEqual(response, test_response_id)
+        mock_service.return_value.buckets.return_value.insert.assert_called_with(
+            project=test_project,
+            body={
+                "lifecycle": test_lifecycle,
+                'name': test_bucket,
+                'location': test_location,
+                'storageClass': test_storage_class,
+                'labels': test_labels
+            }
+        )
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_compose(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_source_objects = ['test_object_1', 'test_object_2', 'test_object_3']
+        test_destination_object = 'test_object_composed'
+
+        method = (mock_service.return_value.objects.return_value.compose)
+
+        self.gcs_hook.compose(
+            bucket=test_bucket,
+            source_objects=test_source_objects,
+            destination_object=test_destination_object
+        )
+
+        body = {
+            'sourceObjects': [
+                {'name': 'test_object_1'},
+                {'name': 'test_object_2'},
+                {'name': 'test_object_3'}
+            ]
+        }
+
+        method.assert_called_once_with(
+            destinationBucket=test_bucket,
+            destinationObject=test_destination_object,
+            body=body
+        )
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_compose_with_empty_source_objects(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_source_objects = []
+        test_destination_object = 'test_object_composed'
+
+        with self.assertRaises(ValueError) as e:
+            self.gcs_hook.compose(
+                bucket=test_bucket,
+                source_objects=test_source_objects,
+                destination_object=test_destination_object
+            )
+
+        self.assertEqual(
+            str(e.exception),
+            'source_objects cannot be empty.'
+        )
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_compose_without_bucket(self, mock_service):
+        test_bucket = None
+        test_source_objects = ['test_object_1', 'test_object_2', 'test_object_3']
+        test_destination_object = 'test_object_composed'
+
+        with self.assertRaises(ValueError) as e:
+            self.gcs_hook.compose(
+                bucket=test_bucket,
+                source_objects=test_source_objects,
+                destination_object=test_destination_object
+            )
+
+        self.assertEqual(
+            str(e.exception),
+            'bucket and destination_object cannot be empty.'
+        )
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_compose_without_destination_object(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_source_objects = ['test_object_1', 'test_object_2', 'test_object_3']
+        test_destination_object = None
+
+        with self.assertRaises(ValueError) as e:
+            self.gcs_hook.compose(
+                bucket=test_bucket,
+                source_objects=test_source_objects,
+                destination_object=test_destination_object
+            )
+
+        self.assertEqual(
+            str(e.exception),
+            'bucket and destination_object cannot be empty.'
+        )
+
+
+class TestGoogleCloudStorageHookUpload(unittest.TestCase):
+    def setUp(self):
+        with mock.patch(BASE_STRING.format('GoogleCloudBaseHook.__init__')):
+            self.gcs_hook = gcs_hook.GoogleCloudStorageHook(
+                google_cloud_storage_conn_id='test'
+            )
+
+        # generate a 384KiB test file (larger than the minimum 256KiB multipart chunk size)
+        self.testfile = tempfile.NamedTemporaryFile(delete=False)
+        self.testfile.write(b"x" * 393216)
+        self.testfile.flush()
+
+    def tearDown(self):
+        os.unlink(self.testfile.name)
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_upload(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_object = 'test_object'
+
+        (mock_service.return_value.objects.return_value
+         .insert.return_value.execute.return_value) = {
+            "kind": "storage#object",
+            "id": "{}/{}/0123456789012345".format(test_bucket, test_object),
+            "name": test_object,
+            "bucket": test_bucket,
+            "generation": "0123456789012345",
+            "contentType": "application/octet-stream",
+            "timeCreated": "2018-03-15T16:51:02.502Z",
+            "updated": "2018-03-15T16:51:02.502Z",
+            "storageClass": "MULTI_REGIONAL",
+            "timeStorageClassUpdated": "2018-03-15T16:51:02.502Z",
+            "size": "393216",
+            "md5Hash": "leYUJBUWrRtks1UeUFONJQ==",
+            "crc32c": "xgdNfQ==",
+            "etag": "CLf4hODk7tkCEAE="
+        }
+
+        response = self.gcs_hook.upload(test_bucket,
+                                        test_object,
+                                        self.testfile.name)
+
+        self.assertTrue(response)
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_upload_gzip(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_object = 'test_object'
+
+        (mock_service.return_value.objects.return_value
+         .insert.return_value.execute.return_value) = {
+            "kind": "storage#object",
+            "id": "{}/{}/0123456789012345".format(test_bucket, test_object),
+            "name": test_object,
+            "bucket": test_bucket,
+            "generation": "0123456789012345",
+            "contentType": "application/octet-stream",
+            "timeCreated": "2018-03-15T16:51:02.502Z",
+            "updated": "2018-03-15T16:51:02.502Z",
+            "storageClass": "MULTI_REGIONAL",
+            "timeStorageClassUpdated": "2018-03-15T16:51:02.502Z",
+            "size": "393216",
+            "md5Hash": "leYUJBUWrRtks1UeUFONJQ==",
+            "crc32c": "xgdNfQ==",
+            "etag": "CLf4hODk7tkCEAE="
+        }
+
+        response = self.gcs_hook.upload(test_bucket,
+                                        test_object,
+                                        self.testfile.name,
+                                        gzip=True)
+        self.assertFalse(os.path.exists(self.testfile.name + '.gz'))
+        self.assertTrue(response)
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_upload_gzip_error(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_object = 'test_object'
+
+        (mock_service.return_value.objects.return_value
+         .insert.return_value.execute.side_effect) = HttpError(
+            resp={'status': '404'}, content=EMPTY_CONTENT)
+
+        response = self.gcs_hook.upload(test_bucket,
+                                        test_object,
+                                        self.testfile.name,
+                                        gzip=True)
+        self.assertFalse(os.path.exists(self.testfile.name + '.gz'))
+        self.assertFalse(response)
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_upload_multipart(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_object = 'test_object'
+
+        class MockProgress:
+            def __init__(self, value):
+                self.value = value
+
+            def progress(self):
+                return self.value
+
+        (mock_service.return_value.objects.return_value
+         .insert.return_value.next_chunk.side_effect) = [
+            (MockProgress(0.66), None),
+            (MockProgress(1.0), {
+                "kind": "storage#object",
+                "id": "{}/{}/0123456789012345".format(test_bucket, test_object),
+                "name": test_object,
+                "bucket": test_bucket,
+                "generation": "0123456789012345",
+                "contentType": "application/octet-stream",
+                "timeCreated": "2018-03-15T16:51:02.502Z",
+                "updated": "2018-03-15T16:51:02.502Z",
+                "storageClass": "MULTI_REGIONAL",
+                "timeStorageClassUpdated": "2018-03-15T16:51:02.502Z",
+                "size": "393216",
+                "md5Hash": "leYUJBUWrRtks1UeUFONJQ==",
+                "crc32c": "xgdNfQ==",
+                "etag": "CLf4hODk7tkCEAE="
+            })
+        ]
+
+        response = self.gcs_hook.upload(test_bucket,
+                                        test_object,
+                                        self.testfile.name,
+                                        multipart=True)
+
+        self.assertTrue(response)
+
+    @mock.patch(GCS_STRING.format('GoogleCloudStorageHook.get_conn'))
+    def test_upload_multipart_wrong_chunksize(self, mock_service):
+        test_bucket = 'test_bucket'
+        test_object = 'test_object'
+
+        with self.assertRaises(ValueError):
+            self.gcs_hook.upload(test_bucket, test_object,
+                                 self.testfile.name, multipart=123)
diff --git a/tests/contrib/hooks/test_imap_hook.py b/tests/contrib/hooks/test_imap_hook.py
new file mode 100644
index 0000000000000..460bd6e3ea356
--- /dev/null
+++ b/tests/contrib/hooks/test_imap_hook.py
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import imaplib
+import unittest
+
+from mock import Mock, patch, mock_open
+
+from airflow import configuration, AirflowException
+from airflow.contrib.hooks.imap_hook import ImapHook
+from airflow.models.connection import Connection
+from airflow.utils import db
+
+imaplib_string = 'airflow.contrib.hooks.imap_hook.imaplib'
+open_string = 'airflow.contrib.hooks.imap_hook.open'
+
+
+def _create_fake_imap(mock_imaplib, with_mail=False, attachment_name='test1.csv'):
+    mock_conn = Mock(spec=imaplib.IMAP4_SSL)
+    mock_imaplib.IMAP4_SSL.return_value = mock_conn
+
+    mock_conn.login.return_value = ('OK', [])
+
+    if with_mail:
+        mock_conn.select.return_value = ('OK', [])
+        mock_conn.search.return_value = ('OK', [b'1'])
+        mail_string = \
+            'Content-Type: multipart/mixed; boundary=123\r\n--123\r\n' \
+            'Content-Disposition: attachment; filename="{}";' \
+            'Content-Transfer-Encoding: base64\r\nSWQsTmFtZQoxLEZlbGl4\r\n--123--'.format(attachment_name)
+        mock_conn.fetch.return_value = ('OK', [(b'', mail_string.encode('utf-8'))])
+        mock_conn.close.return_value = ('OK', [])
+
+    mock_conn.logout.return_value = ('OK', [])
+
+    return mock_conn
+
+
+class TestImapHook(unittest.TestCase):
+    def setUp(self):
+        configuration.load_test_config()
+
+        db.merge_conn(
+            Connection(
+                conn_id='imap_default',
+                host='imap_server_address',
+                login='imap_user',
+                password='imap_password'
+            )
+        )
+
+    @patch(imaplib_string)
+    def test_connect_and_disconnect(self, mock_imaplib):
+        mock_conn = _create_fake_imap(mock_imaplib)
+
+        with ImapHook():
+            pass
+
+        mock_imaplib.IMAP4_SSL.assert_called_once_with('imap_server_address')
+        mock_conn.login.assert_called_once_with('imap_user', 'imap_password')
+        assert mock_conn.logout.call_count == 1
+
+    @patch(imaplib_string)
+    def test_has_mail_attachments_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            has_attachment_in_inbox = imap_hook.has_mail_attachment('test1.csv')
+
+        self.assertTrue(has_attachment_in_inbox)
+
+    @patch(imaplib_string)
+    def test_has_mail_attachments_not_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            has_attachment_in_inbox = imap_hook.has_mail_attachment('test1.txt')
+
+        self.assertFalse(has_attachment_in_inbox)
+
+    @patch(imaplib_string)
+    def test_has_mail_attachments_with_regex_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            has_attachment_in_inbox = imap_hook.has_mail_attachment(
+                name=r'test(\d+).csv',
+                check_regex=True
+            )
+
+        self.assertTrue(has_attachment_in_inbox)
+
+    @patch(imaplib_string)
+    def test_has_mail_attachments_with_regex_not_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            has_attachment_in_inbox = imap_hook.has_mail_attachment(
+                name=r'test_(\d+).csv',
+                check_regex=True
+            )
+
+        self.assertFalse(has_attachment_in_inbox)
+
+    @patch(imaplib_string)
+    def test_retrieve_mail_attachments_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            attachments_in_inbox = imap_hook.retrieve_mail_attachments('test1.csv')
+
+        self.assertEqual(attachments_in_inbox, [('test1.csv', b'SWQsTmFtZQoxLEZlbGl4')])
+
+    @patch(imaplib_string)
+    def test_retrieve_mail_attachments_not_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            self.assertRaises(AirflowException, imap_hook.retrieve_mail_attachments, 'test1.txt')
+
+    @patch(imaplib_string)
+    def test_retrieve_mail_attachments_with_regex_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            attachments_in_inbox = imap_hook.retrieve_mail_attachments(
+                name=r'test(\d+).csv',
+                check_regex=True
+            )
+
+        self.assertEqual(attachments_in_inbox, [('test1.csv', b'SWQsTmFtZQoxLEZlbGl4')])
+
+    @patch(imaplib_string)
+    def test_retrieve_mail_attachments_with_regex_not_found(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            self.assertRaises(AirflowException,
+                              imap_hook.retrieve_mail_attachments,
+                              name=r'test_(\d+).csv',
+                              check_regex=True)
+
+    @patch(imaplib_string)
+    def test_retrieve_mail_attachments_latest_only(self, mock_imaplib):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            attachments_in_inbox = imap_hook.retrieve_mail_attachments(
+                name='test1.csv',
+                latest_only=True
+            )
+
+        self.assertEqual(attachments_in_inbox, [('test1.csv', b'SWQsTmFtZQoxLEZlbGl4')])
+
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_found(self, mock_imaplib, mock_open_method):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            imap_hook.download_mail_attachments('test1.csv', 'test_directory')
+
+        mock_open_method.assert_called_once_with('test_directory/test1.csv', 'wb')
+        mock_open_method.return_value.write.assert_called_once_with(b'SWQsTmFtZQoxLEZlbGl4')
+
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_not_found(self, mock_imaplib, mock_open_method):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            self.assertRaises(AirflowException,
+                              imap_hook.download_mail_attachments, 'test1.txt', 'test_directory')
+
+        mock_open_method.assert_not_called()
+        mock_open_method.return_value.write.assert_not_called()
+
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_with_regex_found(self, mock_imaplib, mock_open_method):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            imap_hook.download_mail_attachments(
+                name=r'test(\d+).csv',
+                local_output_directory='test_directory',
+                check_regex=True
+            )
+
+        mock_open_method.assert_called_once_with('test_directory/test1.csv', 'wb')
+        mock_open_method.return_value.write.assert_called_once_with(b'SWQsTmFtZQoxLEZlbGl4')
+
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_with_regex_not_found(self, mock_imaplib, mock_open_method):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            self.assertRaises(AirflowException,
+                              imap_hook.download_mail_attachments,
+                              name=r'test_(\d+).csv',
+                              local_output_directory='test_directory',
+                              check_regex=True)
+
+        mock_open_method.assert_not_called()
+        mock_open_method.return_value.write.assert_not_called()
+
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_with_latest_only(self, mock_imaplib, mock_open_method):
+        _create_fake_imap(mock_imaplib, with_mail=True)
+
+        with ImapHook() as imap_hook:
+            imap_hook.download_mail_attachments(
+                name='test1.csv',
+                local_output_directory='test_directory',
+                latest_only=True
+            )
+
+        mock_open_method.assert_called_once_with('test_directory/test1.csv', 'wb')
+        mock_open_method.return_value.write.assert_called_once_with(b'SWQsTmFtZQoxLEZlbGl4')
+
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_with_escaping_chars(self, mock_imaplib, mock_open_method):
+        _create_fake_imap(mock_imaplib, with_mail=True, attachment_name='../test1.csv')
+
+        with ImapHook() as imap_hook:
+            imap_hook.download_mail_attachments(
+                name='../test1.csv',
+                local_output_directory='test_directory'
+            )
+
+        mock_open_method.assert_not_called()
+        mock_open_method.return_value.write.assert_not_called()
+
+    @patch('airflow.contrib.hooks.imap_hook.os.path.islink', return_value=True)
+    @patch(open_string, new_callable=mock_open)
+    @patch(imaplib_string)
+    def test_download_mail_attachments_with_symlink(self, mock_imaplib, mock_open_method, mock_is_symlink):
+        _create_fake_imap(mock_imaplib, with_mail=True, attachment_name='symlink')
+
+        with ImapHook() as imap_hook:
+            imap_hook.download_mail_attachments(
+                name='symlink',
+                local_output_directory='test_directory'
+            )
+
+        assert mock_is_symlink.call_count == 1
+        mock_open_method.assert_not_called()
+        mock_open_method.return_value.write.assert_not_called()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_jdbc_hook.py b/tests/contrib/hooks/test_jdbc_hook.py
index c1b994907be1c..1779b781b7e50 100644
--- a/tests/contrib/hooks/test_jdbc_hook.py
+++ b/tests/contrib/hooks/test_jdbc_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,17 +19,18 @@
 #
 
 import unittest
+import json
 
 from mock import Mock
 from mock import patch
 
 from airflow import configuration
 from airflow.hooks.jdbc_hook import JdbcHook
-from airflow import models
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 jdbc_conn_mock = Mock(
-        name="jdbc_conn"
+    name="jdbc_conn"
 )
 
 
@@ -37,10 +38,11 @@ class TestJdbcHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-                models.Connection(
-                        conn_id='jdbc_default', conn_type='jdbc',
-                        host='jdbc://localhost/', port=443,
-                        extra='{"extra__jdbc__drv_path": "/path1/test.jar,/path2/t.jar2", "extra__jdbc__drv_clsname": "com.driver.main"}'))
+            Connection(
+                conn_id='jdbc_default', conn_type='jdbc',
+                host='jdbc://localhost/', port=443,
+                extra=json.dumps({"extra__jdbc__drv_path": "/path1/test.jar,/path2/t.jar2",
+                                  "extra__jdbc__drv_clsname": "com.driver.main"})))
 
     @patch("airflow.hooks.jdbc_hook.jaydebeapi.connect", autospec=True,
            return_value=jdbc_conn_mock)
diff --git a/tests/contrib/hooks/test_jira_hook.py b/tests/contrib/hooks/test_jira_hook.py
index e07ddf32fc788..26aae945a0789 100644
--- a/tests/contrib/hooks/test_jira_hook.py
+++ b/tests/contrib/hooks/test_jira_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,11 +25,11 @@
 
 from airflow import configuration
 from airflow.contrib.hooks.jira_hook import JiraHook
-from airflow import models
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 jira_client_mock = Mock(
-        name="jira_client"
+    name="jira_client"
 )
 
 
@@ -37,10 +37,10 @@ class TestJiraHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-                models.Connection(
-                        conn_id='jira_default', conn_type='jira',
-                        host='https://localhost/jira/', port=443,
-                        extra='{"verify": "False", "project": "AIRFLOW"}'))
+            Connection(
+                conn_id='jira_default', conn_type='jira',
+                host='https://localhost/jira/', port=443,
+                extra='{"verify": "False", "project": "AIRFLOW"}'))
 
     @patch("airflow.contrib.hooks.jira_hook.JIRA", autospec=True,
            return_value=jira_client_mock)
diff --git a/tests/contrib/hooks/test_mongo_hook.py b/tests/contrib/hooks/test_mongo_hook.py
index 7696edea05a30..114ba8495911c 100644
--- a/tests/contrib/hooks/test_mongo_hook.py
+++ b/tests/contrib/hooks/test_mongo_hook.py
@@ -1,16 +1,21 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 import unittest
 import pymongo
 try:
@@ -23,10 +28,10 @@
 
 
 class MongoHookTest(MongoHook):
-    '''
+    """
     Extending hook so that a mockmongo collection object can be passed in
     to get_collection()
-    '''
+    """
     def __init__(self, conn_id='mongo_default', *args, **kwargs):
         super(MongoHookTest, self).__init__(conn_id=conn_id, *args, **kwargs)
 
@@ -69,6 +74,158 @@ def test_insert_many(self):
         result_objs = [result for result in result_objs]
         self.assertEqual(len(result_objs), 2)
 
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_update_one(self):
+        collection = mongomock.MongoClient().db.collection
+        obj = {'_id': '1', 'field': 0}
+        collection.insert_one(obj)
+
+        filter_doc = obj
+        update_doc = {'$inc': {'field': 123}}
+
+        self.hook.update_one(collection, filter_doc, update_doc)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual(123, result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_update_one_with_upsert(self):
+        collection = mongomock.MongoClient().db.collection
+
+        filter_doc = {'_id': '1', 'field': 0}
+        update_doc = {'$inc': {'field': 123}}
+
+        self.hook.update_one(collection, filter_doc, update_doc, upsert=True)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual(123, result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_update_many(self):
+        collection = mongomock.MongoClient().db.collection
+        obj1 = {'_id': '1', 'field': 0}
+        obj2 = {'_id': '2', 'field': 0}
+        collection.insert_many([obj1, obj2])
+
+        filter_doc = {'field': 0}
+        update_doc = {'$inc': {'field': 123}}
+
+        self.hook.update_many(collection, filter_doc, update_doc)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual(123, result_obj['field'])
+
+        result_obj = collection.find_one(filter='2')
+        self.assertEqual(123, result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_update_many_with_upsert(self):
+        collection = mongomock.MongoClient().db.collection
+
+        filter_doc = {'_id': '1', 'field': 0}
+        update_doc = {'$inc': {'field': 123}}
+
+        self.hook.update_many(collection, filter_doc, update_doc, upsert=True)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual(123, result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_replace_one(self):
+        collection = mongomock.MongoClient().db.collection
+        obj1 = {'_id': '1', 'field': 'test_value_1'}
+        obj2 = {'_id': '2', 'field': 'test_value_2'}
+        collection.insert_many([obj1, obj2])
+
+        obj1['field'] = 'test_value_1_updated'
+        self.hook.replace_one(collection, obj1)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual('test_value_1_updated', result_obj['field'])
+
+        # Other document should stay intact
+        result_obj = collection.find_one(filter='2')
+        self.assertEqual('test_value_2', result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_replace_one_with_filter(self):
+        collection = mongomock.MongoClient().db.collection
+        obj1 = {'_id': '1', 'field': 'test_value_1'}
+        obj2 = {'_id': '2', 'field': 'test_value_2'}
+        collection.insert_many([obj1, obj2])
+
+        obj1['field'] = 'test_value_1_updated'
+        self.hook.replace_one(collection, obj1, {'field': 'test_value_1'})
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual('test_value_1_updated', result_obj['field'])
+
+        # Other document should stay intact
+        result_obj = collection.find_one(filter='2')
+        self.assertEqual('test_value_2', result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_replace_one_with_upsert(self):
+        collection = mongomock.MongoClient().db.collection
+
+        obj = {'_id': '1', 'field': 'test_value_1'}
+        self.hook.replace_one(collection, obj, upsert=True)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual('test_value_1', result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_replace_many(self):
+        collection = mongomock.MongoClient().db.collection
+        obj1 = {'_id': '1', 'field': 'test_value_1'}
+        obj2 = {'_id': '2', 'field': 'test_value_2'}
+        collection.insert_many([obj1, obj2])
+
+        obj1['field'] = 'test_value_1_updated'
+        obj2['field'] = 'test_value_2_updated'
+        self.hook.replace_many(collection, [obj1, obj2])
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual('test_value_1_updated', result_obj['field'])
+
+        result_obj = collection.find_one(filter='2')
+        self.assertEqual('test_value_2_updated', result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_replace_many_with_upsert(self):
+        collection = mongomock.MongoClient().db.collection
+        obj1 = {'_id': '1', 'field': 'test_value_1'}
+        obj2 = {'_id': '2', 'field': 'test_value_2'}
+
+        self.hook.replace_many(collection, [obj1, obj2], upsert=True)
+
+        result_obj = collection.find_one(filter='1')
+        self.assertEqual('test_value_1', result_obj['field'])
+
+        result_obj = collection.find_one(filter='2')
+        self.assertEqual('test_value_2', result_obj['field'])
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_delete_one(self):
+        collection = mongomock.MongoClient().db.collection
+        obj = {'_id': '1'}
+        collection.insert_one(obj)
+
+        self.hook.delete_one(collection, {'_id': '1'})
+
+        self.assertEqual(0, collection.count())
+
+    @unittest.skipIf(mongomock is None, 'mongomock package not present')
+    def test_delete_many(self):
+        collection = mongomock.MongoClient().db.collection
+        obj1 = {'_id': '1', 'field': 'value'}
+        obj2 = {'_id': '2', 'field': 'value'}
+        collection.insert_many([obj1, obj2])
+
+        self.hook.delete_many(collection, {'field': 'value'})
+
+        self.assertEqual(0, collection.count())
+
     @unittest.skipIf(mongomock is None, 'mongomock package not present')
     def test_find_one(self):
         collection = mongomock.MongoClient().db.collection
@@ -118,6 +275,15 @@ def test_aggregate(self):
         results = [result for result in results]
         self.assertEqual(len(results), 2)
 
+    def test_context_manager(self):
+        with MongoHook(conn_id='mongo_default', mongo_db='default') as ctxHook:
+            ctxHook.get_conn()
+
+            self.assertIsInstance(ctxHook, MongoHook)
+            self.assertIsNotNone(ctxHook.client)
+
+        self.assertIsNone(ctxHook.client)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_openfaas_hook.py b/tests/contrib/hooks/test_openfaas_hook.py
new file mode 100644
index 0000000000000..162bf38c59ded
--- /dev/null
+++ b/tests/contrib/hooks/test_openfaas_hook.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+import requests_mock
+from airflow.models.connection import Connection
+from airflow.contrib.hooks.openfaas_hook import OpenFaasHook
+from airflow.hooks.base_hook import BaseHook
+from airflow import configuration, AirflowException
+from tests.compat import mock
+
+FUNCTION_NAME = "function_name"
+
+
+class TestOpenFaasHook(unittest.TestCase):
+    GET_FUNCTION = "/system/function/"
+    INVOKE_ASYNC_FUNCTION = "/async-function/"
+    DEPLOY_FUNCTION = "/system/functions"
+    UPDATE_FUNCTION = "/system/functions"
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.hook = OpenFaasHook(function_name=FUNCTION_NAME)
+        self.mock_response = {'ans': 'a'}
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_is_function_exist_false(self, mock_get_connection, m):
+        m.get("http://open-faas.io" + self.GET_FUNCTION + FUNCTION_NAME,
+              json=self.mock_response, status_code=404)
+        mock_connection = Connection(host="http://open-faas.io")
+
+        mock_get_connection.return_value = mock_connection
+        does_function_exist = self.hook.does_function_exist()
+        self.assertFalse(does_function_exist)
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_is_function_exist_true(self, mock_get_connection, m):
+        m.get("http://open-faas.io" + self.GET_FUNCTION + FUNCTION_NAME,
+              json=self.mock_response, status_code=202)
+        mock_connection = Connection(host="http://open-faas.io")
+
+        mock_get_connection.return_value = mock_connection
+        does_function_exist = self.hook.does_function_exist()
+        self.assertTrue(does_function_exist)
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_update_function_true(self, mock_get_connection, m):
+        m.put("http://open-faas.io" + self.UPDATE_FUNCTION, json=self.mock_response, status_code=202)
+        mock_connection = Connection(host="http://open-faas.io")
+
+        mock_get_connection.return_value = mock_connection
+        update_function_ans = self.hook.update_function({})
+        self.assertEqual(update_function_ans, None)
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_update_function_false(self, mock_get_connection, m):
+        m.put("http://open-faas.io" + self.UPDATE_FUNCTION, json=self.mock_response, status_code=400)
+        mock_connection = Connection(host="http://open-faas.io")
+        mock_get_connection.return_value = mock_connection
+
+        with self.assertRaises(AirflowException) as context:
+            self.hook.update_function({})
+        self.assertIn('failed to update ' + FUNCTION_NAME, str(context.exception))
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_invoke_async_function_false(self, mock_get_connection, m):
+        m.post("http://open-faas.io" + self.INVOKE_ASYNC_FUNCTION + FUNCTION_NAME, json=self.mock_response,
+               status_code=400)
+        mock_connection = Connection(host="http://open-faas.io")
+        mock_get_connection.return_value = mock_connection
+
+        with self.assertRaises(AirflowException) as context:
+            self.hook.invoke_async_function({})
+        self.assertIn('failed to invoke function', str(context.exception))
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_invoke_async_function_true(self, mock_get_connection, m):
+        m.post("http://open-faas.io" + self.INVOKE_ASYNC_FUNCTION + FUNCTION_NAME, json=self.mock_response,
+               status_code=202)
+        mock_connection = Connection(host="http://open-faas.io")
+        mock_get_connection.return_value = mock_connection
+        self.assertEqual(self.hook.invoke_async_function({}), None)
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_deploy_function_function_already_exist(self, mock_get_connection, m):
+        m.put("http://open-faas.io/" + self.UPDATE_FUNCTION, json=self.mock_response, status_code=202)
+        mock_connection = Connection(host="http://open-faas.io/")
+        mock_get_connection.return_value = mock_connection
+        self.assertEqual(self.hook.deploy_function(True, {}), None)
+
+    @mock.patch.object(BaseHook, 'get_connection')
+    @requests_mock.mock()
+    def test_deploy_function_function_not_exist(self, mock_get_connection, m):
+        m.post("http://open-faas.io" + self.DEPLOY_FUNCTION, json={}, status_code=202)
+        mock_connection = Connection(host="http://open-faas.io")
+        mock_get_connection.return_value = mock_connection
+        self.assertEqual(self.hook.deploy_function(False, {}), None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_opsgenie_alert_hook.py b/tests/contrib/hooks/test_opsgenie_alert_hook.py
new file mode 100644
index 0000000000000..be6bdf6fb9d8c
--- /dev/null
+++ b/tests/contrib/hooks/test_opsgenie_alert_hook.py
@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import unittest
+import requests_mock
+import json
+
+from airflow import configuration
+from airflow.models.connection import Connection
+from airflow.utils import db
+from airflow.contrib.hooks.opsgenie_alert_hook import OpsgenieAlertHook
+from airflow import AirflowException
+
+
+class TestOpsgenieAlertHook(unittest.TestCase):
+    conn_id = 'opsgenie_conn_id_test'
+    opsgenie_alert_endpoint = 'https://api.opsgenie.com/v2/alerts'
+    _payload = {
+        'message': 'An example alert message',
+        'alias': 'Life is too short for no alias',
+        'description': 'Every alert needs a description',
+        'responders': [
+            {'id': '4513b7ea-3b91-438f-b7e4-e3e54af9147c', 'type': 'team'},
+            {'name': 'NOC', 'type': 'team'},
+            {'id': 'bb4d9938-c3c2-455d-aaab-727aa701c0d8', 'type': 'user'},
+            {'username': 'trinity@opsgenie.com', 'type': 'user'},
+            {'id': 'aee8a0de-c80f-4515-a232-501c0bc9d715', 'type': 'escalation'},
+            {'name': 'Nightwatch Escalation', 'type': 'escalation'},
+            {'id': '80564037-1984-4f38-b98e-8a1f662df552', 'type': 'schedule'},
+            {'name': 'First Responders Schedule', 'type': 'schedule'}
+        ],
+        'visibleTo': [
+            {'id': '4513b7ea-3b91-438f-b7e4-e3e54af9147c', 'type': 'team'},
+            {'name': 'rocket_team', 'type': 'team'},
+            {'id': 'bb4d9938-c3c2-455d-aaab-727aa701c0d8', 'type': 'user'},
+            {'username': 'trinity@opsgenie.com', 'type': 'user'}
+        ],
+        'actions': ['Restart', 'AnExampleAction'],
+        'tags': ['OverwriteQuietHours', 'Critical'],
+        'details': {'key1': 'value1', 'key2': 'value2'},
+        'entity': 'An example entity',
+        'source': 'Airflow',
+        'priority': 'P1',
+        'user': 'Jesse',
+        'note': 'Write this down'
+    }
+    _mock_success_response_body = {
+        "result": "Request will be processed",
+        "took": 0.302,
+        "requestId": "43a29c5c-3dbf-4fa4-9c26-f4f71023e120"
+    }
+
+    def setUp(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id=self.conn_id,
+                host='https://api.opsgenie.com/',
+                password='eb243592-faa2-4ba2-a551q-1afdf565c889'
+            )
+        )
+
+    def test_get_api_key(self):
+        hook = OpsgenieAlertHook(opsgenie_conn_id=self.conn_id)
+        api_key = hook._get_api_key()
+        self.assertEqual('eb243592-faa2-4ba2-a551q-1afdf565c889', api_key)
+
+    def test_get_conn_defaults_host(self):
+        hook = OpsgenieAlertHook()
+        hook.get_conn()
+        self.assertEqual('https://api.opsgenie.com', hook.base_url)
+
+    @requests_mock.mock()
+    def test_call_with_success(self, m):
+        hook = OpsgenieAlertHook(opsgenie_conn_id=self.conn_id)
+        m.post(
+            self.opsgenie_alert_endpoint,
+            status_code=202,
+            json=self._mock_success_response_body
+        )
+        resp = hook.execute(payload=self._payload)
+        self.assertEqual(resp.status_code, 202)
+        self.assertEqual(resp.json(), self._mock_success_response_body)
+
+    @requests_mock.mock()
+    def test_api_key_set(self, m):
+        hook = OpsgenieAlertHook(opsgenie_conn_id=self.conn_id)
+        m.post(
+            self.opsgenie_alert_endpoint,
+            status_code=202,
+            json=self._mock_success_response_body
+        )
+        resp = hook.execute(payload=self._payload)
+        self.assertEqual(resp.request.headers.get('Authorization'),
+                         'GenieKey eb243592-faa2-4ba2-a551q-1afdf565c889')
+
+    @requests_mock.mock()
+    def test_api_key_not_set(self, m):
+        hook = OpsgenieAlertHook()
+        m.post(
+            self.opsgenie_alert_endpoint,
+            status_code=202,
+            json=self._mock_success_response_body
+        )
+        with self.assertRaises(AirflowException):
+            hook.execute(payload=self._payload)
+
+    @requests_mock.mock()
+    def test_payload_set(self, m):
+        hook = OpsgenieAlertHook(opsgenie_conn_id=self.conn_id)
+        m.post(
+            self.opsgenie_alert_endpoint,
+            status_code=202,
+            json=self._mock_success_response_body
+        )
+        resp = hook.execute(payload=self._payload)
+        self.assertEqual(json.loads(resp.request.body), self._payload)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_pinot_hook.py b/tests/contrib/hooks/test_pinot_hook.py
index d63ee1cc7de7c..74be9a3624e09 100644
--- a/tests/contrib/hooks/test_pinot_hook.py
+++ b/tests/contrib/hooks/test_pinot_hook.py
@@ -51,7 +51,7 @@ def test_get_uri(self):
         Test on getting a pinot connection uri
         """
         db_hook = self.db_hook()
-        self.assertEquals(db_hook.get_uri(), 'http://host:1000/pql')
+        self.assertEqual(db_hook.get_uri(), 'http://host:1000/pql')
 
     def test_get_conn(self):
         """
diff --git a/tests/contrib/hooks/test_redis_hook.py b/tests/contrib/hooks/test_redis_hook.py
index ca8bdea3b3448..978b304d41f03 100644
--- a/tests/contrib/hooks/test_redis_hook.py
+++ b/tests/contrib/hooks/test_redis_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,32 +19,43 @@
 
 
 import unittest
-from mock import patch
-
 from airflow import configuration
 from airflow.contrib.hooks.redis_hook import RedisHook
 
 
 class TestRedisHook(unittest.TestCase):
+
     def setUp(self):
         configuration.load_test_config()
 
     def test_get_conn(self):
         hook = RedisHook(redis_conn_id='redis_default')
-        self.assertEqual(hook.client, None)
-        self.assertEqual(
-            repr(hook.get_conn()),
-            (
-                'StrictRedis<ConnectionPool'
-                '<Connection<host=localhost,port=6379,db=0>>>'
-            )
-        )
-
-    @patch("airflow.contrib.hooks.redis_hook.RedisHook.get_conn")
-    def test_first_conn_instantiation(self, get_conn):
+        self.assertEqual(hook.redis, None)
+
+        self.assertEqual(hook.host, None, 'host initialised as None.')
+        self.assertEqual(hook.port, None, 'port initialised as None.')
+        self.assertEqual(hook.password, None, 'password initialised as None.')
+        self.assertEqual(hook.db, None, 'db initialised as None.')
+        self.assertIs(hook.get_conn(), hook.get_conn(), 'Connection initialized only if None.')
+
+    def test_get_conn_password_stays_none(self):
+        hook = RedisHook(redis_conn_id='redis_default')
+        hook.get_conn()
+        self.assertEqual(hook.password, None)
+
+    def test_real_ping(self):
+        hook = RedisHook(redis_conn_id='redis_default')
+        redis = hook.get_conn()
+
+        self.assertTrue(redis.ping(), 'Connection to Redis with PING works.')
+
+    def test_real_get_and_set(self):
         hook = RedisHook(redis_conn_id='redis_default')
-        hook.key_exists('test_key')
-        self.assertTrue(get_conn.called_once())
+        redis = hook.get_conn()
+
+        self.assertTrue(redis.set('test_key', 'test_value'), 'Connection to Redis with SET works.')
+        self.assertEqual(redis.get('test_key'), b'test_value', 'Connection to Redis with GET works.')
+        self.assertEqual(redis.delete('test_key'), 1, 'Connection to Redis with DELETE works.')
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/hooks/test_redshift_hook.py b/tests/contrib/hooks/test_redshift_hook.py
index d5e791b4b7c80..103411207071c 100644
--- a/tests/contrib/hooks/test_redshift_hook.py
+++ b/tests/contrib/hooks/test_redshift_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -31,10 +31,12 @@
     mock_redshift = None
 
 
-@mock_redshift
 class TestRedshiftHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
+
+    @staticmethod
+    def _create_clusters():
         client = boto3.client('redshift', region_name='us-east-1')
         client.create_cluster(
             ClusterIdentifier='test_cluster',
@@ -52,7 +54,9 @@ def setUp(self):
             raise ValueError('AWS not properly mocked')
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_get_client_type_returns_a_boto3_client_of_the_requested_type(self):
+        self._create_clusters()
         hook = AwsHook(aws_conn_id='aws_default')
         client_from_hook = hook.get_client_type('redshift')
 
@@ -60,7 +64,9 @@ def test_get_client_type_returns_a_boto3_client_of_the_requested_type(self):
         self.assertEqual(len(clusters), 2)
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_restore_from_cluster_snapshot_returns_dict_with_cluster_data(self):
+        self._create_clusters()
         hook = RedshiftHook(aws_conn_id='aws_default')
         hook.create_cluster_snapshot('test_snapshot', 'test_cluster')
         self.assertEqual(
@@ -70,27 +76,35 @@ def test_restore_from_cluster_snapshot_returns_dict_with_cluster_data(self):
             'test_cluster_3')
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_delete_cluster_returns_a_dict_with_cluster_data(self):
+        self._create_clusters()
         hook = RedshiftHook(aws_conn_id='aws_default')
 
         cluster = hook.delete_cluster('test_cluster_2')
         self.assertNotEqual(cluster, None)
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_create_cluster_snapshot_returns_snapshot_data(self):
+        self._create_clusters()
         hook = RedshiftHook(aws_conn_id='aws_default')
 
         snapshot = hook.create_cluster_snapshot('test_snapshot_2', 'test_cluster')
         self.assertNotEqual(snapshot, None)
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_cluster_status_returns_cluster_not_found(self):
+        self._create_clusters()
         hook = RedshiftHook(aws_conn_id='aws_default')
         status = hook.cluster_status('test_cluster_not_here')
         self.assertEqual(status, 'cluster_not_found')
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_cluster_status_returns_available_cluster(self):
+        self._create_clusters()
         hook = RedshiftHook(aws_conn_id='aws_default')
         status = hook.cluster_status('test_cluster')
         self.assertEqual(status, 'available')
diff --git a/tests/contrib/hooks/test_sagemaker_hook.py b/tests/contrib/hooks/test_sagemaker_hook.py
new file mode 100644
index 0000000000000..4837e7af0340e
--- /dev/null
+++ b/tests/contrib/hooks/test_sagemaker_hook.py
@@ -0,0 +1,673 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+import time
+from datetime import datetime
+from tzlocal import get_localzone
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import (SageMakerHook, secondary_training_status_changed,
+                                                  secondary_training_status_message, LogState)
+from airflow.hooks.S3_hook import S3Hook
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+
+role = 'arn:aws:iam:role/test-role'
+
+path = 'local/data'
+bucket = 'test-bucket'
+key = 'test/data'
+data_url = 's3://{}/{}'.format(bucket, key)
+
+job_name = 'test-job'
+model_name = 'test-model'
+config_name = 'test-endpoint-config'
+endpoint_name = 'test-endpoint'
+
+image = 'test-image'
+test_arn_return = {'Arn': 'testarn'}
+output_url = 's3://{}/test/output'.format(bucket)
+
+create_training_params = {
+    'AlgorithmSpecification': {
+        'TrainingImage': image,
+        'TrainingInputMode': 'File'
+    },
+    'RoleArn': role,
+    'OutputDataConfig': {
+        'S3OutputPath': output_url
+    },
+    'ResourceConfig': {
+        'InstanceCount': 2,
+        'InstanceType': 'ml.c4.8xlarge',
+        'VolumeSizeInGB': 50
+    },
+    'TrainingJobName': job_name,
+    'HyperParameters': {
+        'k': '10',
+        'feature_dim': '784',
+        'mini_batch_size': '500',
+        'force_dense': 'True'
+    },
+    'StoppingCondition': {
+        'MaxRuntimeInSeconds': 60 * 60
+    },
+    'InputDataConfig': [
+        {
+            'ChannelName': 'train',
+            'DataSource': {
+                'S3DataSource': {
+                    'S3DataType': 'S3Prefix',
+                    'S3Uri': data_url,
+                    'S3DataDistributionType': 'FullyReplicated'
+                }
+            },
+            'CompressionType': 'None',
+            'RecordWrapperType': 'None'
+        }
+    ]
+}
+
+create_tuning_params = {
+    'HyperParameterTuningJobName': job_name,
+    'HyperParameterTuningJobConfig': {
+        'Strategy': 'Bayesian',
+        'HyperParameterTuningJobObjective': {
+            'Type': 'Maximize',
+            'MetricName': 'test_metric'
+        },
+        'ResourceLimits': {
+            'MaxNumberOfTrainingJobs': 123,
+            'MaxParallelTrainingJobs': 123
+        },
+        'ParameterRanges': {
+            'IntegerParameterRanges': [
+                {
+                    'Name': 'k',
+                    'MinValue': '2',
+                    'MaxValue': '10'
+                },
+
+            ]
+        }
+    },
+    'TrainingJobDefinition': {
+        'StaticHyperParameters': create_training_params['HyperParameters'],
+        'AlgorithmSpecification': create_training_params['AlgorithmSpecification'],
+        'RoleArn': 'string',
+        'InputDataConfig': create_training_params['InputDataConfig'],
+        'OutputDataConfig': create_training_params['OutputDataConfig'],
+        'ResourceConfig': create_training_params['ResourceConfig'],
+        'StoppingCondition': dict(MaxRuntimeInSeconds=60 * 60)
+    }
+}
+
+create_transform_params = {
+    'TransformJobName': job_name,
+    'ModelName': model_name,
+    'BatchStrategy': 'MultiRecord',
+    'TransformInput': {
+        'DataSource': {
+            'S3DataSource': {
+                'S3DataType': 'S3Prefix',
+                'S3Uri': data_url
+            }
+        }
+    },
+    'TransformOutput': {
+        'S3OutputPath': output_url,
+    },
+    'TransformResources': {
+        'InstanceType': 'ml.m4.xlarge',
+        'InstanceCount': 123
+    }
+}
+
+create_model_params = {
+    'ModelName': model_name,
+    'PrimaryContainer': {
+        'Image': image,
+        'ModelDataUrl': output_url,
+    },
+    'ExecutionRoleArn': role
+}
+
+create_endpoint_config_params = {
+    'EndpointConfigName': config_name,
+    'ProductionVariants': [
+        {
+            'VariantName': 'AllTraffic',
+            'ModelName': model_name,
+            'InitialInstanceCount': 1,
+            'InstanceType': 'ml.c4.xlarge'
+        }
+    ]
+}
+
+create_endpoint_params = {
+    'EndpointName': endpoint_name,
+    'EndpointConfigName': config_name
+}
+
+update_endpoint_params = create_endpoint_params
+
+DESCRIBE_TRAINING_COMPELETED_RETURN = {
+    'TrainingJobStatus': 'Completed',
+    'ResourceConfig': {
+        'InstanceCount': 1,
+        'InstanceType': 'ml.c4.xlarge',
+        'VolumeSizeInGB': 10
+    },
+    'TrainingStartTime': datetime(2018, 2, 17, 7, 15, 0, 103000),
+    'TrainingEndTime': datetime(2018, 2, 17, 7, 19, 34, 953000),
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+DESCRIBE_TRAINING_INPROGRESS_RETURN = dict(DESCRIBE_TRAINING_COMPELETED_RETURN)
+DESCRIBE_TRAINING_INPROGRESS_RETURN.update({'TrainingJobStatus': 'InProgress'})
+
+DESCRIBE_TRAINING_FAILED_RETURN = dict(DESCRIBE_TRAINING_COMPELETED_RETURN)
+DESCRIBE_TRAINING_FAILED_RETURN.update({'TrainingJobStatus': 'Failed',
+                                        'FailureReason': 'Unknown'})
+
+DESCRIBE_TRAINING_STOPPING_RETURN = dict(DESCRIBE_TRAINING_COMPELETED_RETURN)
+DESCRIBE_TRAINING_STOPPING_RETURN.update({'TrainingJobStatus': 'Stopping'})
+
+message = 'message'
+status = 'status'
+SECONDARY_STATUS_DESCRIPTION_1 = {
+    'SecondaryStatusTransitions': [{'StatusMessage': message, 'Status': status}]
+}
+SECONDARY_STATUS_DESCRIPTION_2 = {
+    'SecondaryStatusTransitions': [{'StatusMessage': 'different message', 'Status': status}]
+}
+
+DEFAULT_LOG_STREAMS = {'logStreams': [{'logStreamName': job_name + '/xxxxxxxxx'}]}
+LIFECYCLE_LOG_STREAMS = [DEFAULT_LOG_STREAMS,
+                         DEFAULT_LOG_STREAMS,
+                         DEFAULT_LOG_STREAMS,
+                         DEFAULT_LOG_STREAMS,
+                         DEFAULT_LOG_STREAMS,
+                         DEFAULT_LOG_STREAMS]
+
+DEFAULT_LOG_EVENTS = [{'nextForwardToken': None, 'events': [{'timestamp': 1, 'message': 'hi there #1'}]},
+                      {'nextForwardToken': None, 'events': []}]
+STREAM_LOG_EVENTS = [{'nextForwardToken': None, 'events': [{'timestamp': 1, 'message': 'hi there #1'}]},
+                     {'nextForwardToken': None, 'events': []},
+                     {'nextForwardToken': None, 'events': [{'timestamp': 1, 'message': 'hi there #1'},
+                                                           {'timestamp': 2, 'message': 'hi there #2'}]},
+                     {'nextForwardToken': None, 'events': []},
+                     {'nextForwardToken': None, 'events': [{'timestamp': 2, 'message': 'hi there #2'},
+                                                           {'timestamp': 2, 'message': 'hi there #2a'},
+                                                           {'timestamp': 3, 'message': 'hi there #3'}]},
+                     {'nextForwardToken': None, 'events': []}]
+
+test_evaluation_config = {
+    'Image': image,
+    'Role': role,
+    'S3Operations': {
+        'S3CreateBucket': [
+            {
+                'Bucket': bucket
+            }
+        ],
+        'S3Upload': [
+            {
+                'Path': path,
+                'Bucket': bucket,
+                'Key': key,
+                'Tar': False
+            }
+        ]
+    }
+}
+
+
+class TestSageMakerHook(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+
+    @mock.patch.object(SageMakerHook, 'log_stream')
+    def test_multi_stream_iter(self, mock_log_stream):
+        event = {'timestamp': 1}
+        mock_log_stream.side_effect = [iter([event]), iter([]), None]
+        hook = SageMakerHook()
+        event_iter = hook.multi_stream_iter('log', [None, None, None])
+        self.assertEqual(next(event_iter), (0, event))
+
+    @mock.patch.object(S3Hook, 'create_bucket')
+    @mock.patch.object(S3Hook, 'load_file')
+    def test_configure_s3_resources(self, mock_load_file, mock_create_bucket):
+        hook = SageMakerHook()
+        evaluation_result = {
+            'Image': image,
+            'Role': role
+        }
+        hook.configure_s3_resources(test_evaluation_config)
+        self.assertEqual(test_evaluation_config, evaluation_result)
+        mock_create_bucket.assert_called_once_with(bucket_name=bucket)
+        mock_load_file.assert_called_once_with(path, key, bucket)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(S3Hook, 'check_for_key')
+    @mock.patch.object(S3Hook, 'check_for_bucket')
+    @mock.patch.object(S3Hook, 'check_for_prefix')
+    def test_check_s3_url(self,
+                          mock_check_prefix,
+                          mock_check_bucket,
+                          mock_check_key,
+                          mock_client):
+        mock_client.return_value = None
+        hook = SageMakerHook()
+        mock_check_bucket.side_effect = [False, True, True, True]
+        mock_check_key.side_effect = [False, True, False]
+        mock_check_prefix.side_effect = [False, True, True]
+        self.assertRaises(AirflowException,
+                          hook.check_s3_url, data_url)
+        self.assertRaises(AirflowException,
+                          hook.check_s3_url, data_url)
+        self.assertEqual(hook.check_s3_url(data_url), True)
+        self.assertEqual(hook.check_s3_url(data_url), True)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'check_s3_url')
+    def test_check_valid_training(self, mock_check_url, mock_client):
+        mock_client.return_value = None
+        hook = SageMakerHook()
+        hook.check_training_config(create_training_params)
+        mock_check_url.assert_called_once_with(data_url)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'check_s3_url')
+    def test_check_valid_tuning(self, mock_check_url, mock_client):
+        mock_client.return_value = None
+        hook = SageMakerHook()
+        hook.check_tuning_config(create_tuning_params)
+        mock_check_url.assert_called_once_with(data_url)
+
+    @mock.patch.object(SageMakerHook, 'get_client_type')
+    def test_conn(self, mock_get_client):
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        self.assertEqual(hook.aws_conn_id, 'sagemaker_test_conn_id')
+
+    @mock.patch.object(SageMakerHook, 'check_training_config')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_create_training_job(self, mock_client, mock_check_training):
+        mock_check_training.return_value = True
+        mock_session = mock.Mock()
+        attrs = {'create_training_job.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.create_training_job(create_training_params,
+                                            wait_for_completion=False,
+                                            print_log=False)
+        mock_session.create_training_job.assert_called_once_with(**create_training_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'check_training_config')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_training_ends_with_wait(self, mock_client, mock_check_training):
+        mock_check_training.return_value = True
+        mock_session = mock.Mock()
+        attrs = {'create_training_job.return_value':
+                 test_arn_return,
+                 'describe_training_job.side_effect':
+                 [DESCRIBE_TRAINING_INPROGRESS_RETURN,
+                  DESCRIBE_TRAINING_STOPPING_RETURN,
+                  DESCRIBE_TRAINING_COMPELETED_RETURN,
+                  DESCRIBE_TRAINING_COMPELETED_RETURN]
+                 }
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id_1')
+        hook.create_training_job(create_training_params, wait_for_completion=True,
+                                 print_log=False, check_interval=1)
+        self.assertEqual(mock_session.describe_training_job.call_count, 4)
+
+    @mock.patch.object(SageMakerHook, 'check_training_config')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_training_throws_error_when_failed_with_wait(
+            self, mock_client, mock_check_training):
+        mock_check_training.return_value = True
+        mock_session = mock.Mock()
+        attrs = {'create_training_job.return_value':
+                 test_arn_return,
+                 'describe_training_job.side_effect':
+                 [DESCRIBE_TRAINING_INPROGRESS_RETURN,
+                  DESCRIBE_TRAINING_STOPPING_RETURN,
+                  DESCRIBE_TRAINING_FAILED_RETURN,
+                  DESCRIBE_TRAINING_COMPELETED_RETURN]
+                 }
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id_1')
+        self.assertRaises(AirflowException, hook.create_training_job,
+                          create_training_params, wait_for_completion=True,
+                          print_log=False, check_interval=1)
+        self.assertEqual(mock_session.describe_training_job.call_count, 3)
+
+    @mock.patch.object(SageMakerHook, 'check_tuning_config')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_create_tuning_job(self, mock_client, mock_check_tuning):
+        mock_session = mock.Mock()
+        attrs = {'create_hyper_parameter_tuning_job.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.create_tuning_job(create_tuning_params,
+                                          wait_for_completion=False)
+        mock_session.create_hyper_parameter_tuning_job.\
+            assert_called_once_with(**create_tuning_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'check_s3_url')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_create_transform_job(self, mock_client, mock_check_url):
+        mock_check_url.return_value = True
+        mock_session = mock.Mock()
+        attrs = {'create_transform_job.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.create_transform_job(create_transform_params,
+                                             wait_for_completion=False)
+        mock_session.create_transform_job.assert_called_once_with(
+            **create_transform_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_create_model(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'create_model.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.create_model(create_model_params)
+        mock_session.create_model.assert_called_once_with(**create_model_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_create_endpoint_config(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'create_endpoint_config.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.create_endpoint_config(create_endpoint_config_params)
+        mock_session.create_endpoint_config\
+            .assert_called_once_with(**create_endpoint_config_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_create_endpoint(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'create_endpoint.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.create_endpoint(create_endpoint_params,
+                                        wait_for_completion=False)
+        mock_session.create_endpoint\
+            .assert_called_once_with(**create_endpoint_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_update_endpoint(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'update_endpoint.return_value':
+                 test_arn_return}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.update_endpoint(update_endpoint_params,
+                                        wait_for_completion=False)
+        mock_session.update_endpoint\
+            .assert_called_once_with(**update_endpoint_params)
+        self.assertEqual(response, test_arn_return)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_training_job(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'describe_training_job.return_value': 'InProgress'}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_training_job(job_name)
+        mock_session.describe_training_job.\
+            assert_called_once_with(TrainingJobName=job_name)
+        self.assertEqual(response, 'InProgress')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_tuning_job(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'describe_hyper_parameter_tuning_job.return_value':
+                 'InProgress'}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_tuning_job(job_name)
+        mock_session.describe_hyper_parameter_tuning_job.\
+            assert_called_once_with(HyperParameterTuningJobName=job_name)
+        self.assertEqual(response, 'InProgress')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_transform_job(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'describe_transform_job.return_value':
+                 'InProgress'}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_transform_job(job_name)
+        mock_session.describe_transform_job.\
+            assert_called_once_with(TransformJobName=job_name)
+        self.assertEqual(response, 'InProgress')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_model(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'describe_model.return_value':
+                 model_name}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_model(model_name)
+        mock_session.describe_model.\
+            assert_called_once_with(ModelName=model_name)
+        self.assertEqual(response, model_name)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_endpoint_config(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'describe_endpoint_config.return_value':
+                 config_name}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_endpoint_config(config_name)
+        mock_session.describe_endpoint_config.\
+            assert_called_once_with(EndpointConfigName=config_name)
+        self.assertEqual(response, config_name)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_endpoint(self, mock_client):
+        mock_session = mock.Mock()
+        attrs = {'describe_endpoint.return_value':
+                 'InProgress'}
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_endpoint(endpoint_name)
+        mock_session.describe_endpoint.\
+            assert_called_once_with(EndpointName=endpoint_name)
+        self.assertEqual(response, 'InProgress')
+
+    def test_secondary_training_status_changed_true(self):
+        changed = secondary_training_status_changed(SECONDARY_STATUS_DESCRIPTION_1,
+                                                    SECONDARY_STATUS_DESCRIPTION_2)
+        self.assertTrue(changed)
+
+    def test_secondary_training_status_changed_false(self):
+        changed = secondary_training_status_changed(SECONDARY_STATUS_DESCRIPTION_1,
+                                                    SECONDARY_STATUS_DESCRIPTION_1)
+        self.assertFalse(changed)
+
+    def test_secondary_training_status_message_status_changed(self):
+        now = datetime.now(get_localzone())
+        SECONDARY_STATUS_DESCRIPTION_1['LastModifiedTime'] = now
+        expected = '{} {} - {}'.format(
+            datetime.utcfromtimestamp(time.mktime(now.timetuple())).strftime('%Y-%m-%d %H:%M:%S'),
+            status,
+            message
+        )
+        self.assertEqual(
+            secondary_training_status_message(SECONDARY_STATUS_DESCRIPTION_1, SECONDARY_STATUS_DESCRIPTION_2),
+            expected)
+
+    @mock.patch.object(SageMakerHook, 'get_log_conn')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(time, 'time')
+    def test_describe_training_job_with_logs_in_progress(self, mock_time, mock_client, mock_log_client):
+        mock_session = mock.Mock()
+        mock_log_session = mock.Mock()
+        attrs = {'describe_training_job.return_value':
+                 DESCRIBE_TRAINING_COMPELETED_RETURN
+                 }
+        log_attrs = {'describe_log_streams.side_effect':
+                     LIFECYCLE_LOG_STREAMS,
+                     'get_log_events.side_effect':
+                     STREAM_LOG_EVENTS
+                     }
+        mock_time.return_value = 50
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        mock_log_session.configure_mock(**log_attrs)
+        mock_log_client.return_value = mock_log_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_training_job_with_log(job_name=job_name,
+                                                       positions={},
+                                                       stream_names=[],
+                                                       instance_count=1,
+                                                       state=LogState.WAIT_IN_PROGRESS,
+                                                       last_description={},
+                                                       last_describe_job_call=0)
+        self.assertEqual(response, (LogState.JOB_COMPLETE, {}, 50))
+
+    @mock.patch.object(SageMakerHook, 'get_log_conn')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_training_job_with_logs_job_complete(self, mock_client, mock_log_client):
+        mock_session = mock.Mock()
+        mock_log_session = mock.Mock()
+        attrs = {'describe_training_job.return_value':
+                 DESCRIBE_TRAINING_COMPELETED_RETURN
+                 }
+        log_attrs = {'describe_log_streams.side_effect':
+                     LIFECYCLE_LOG_STREAMS,
+                     'get_log_events.side_effect':
+                     STREAM_LOG_EVENTS
+                     }
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        mock_log_session.configure_mock(**log_attrs)
+        mock_log_client.return_value = mock_log_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_training_job_with_log(job_name=job_name,
+                                                       positions={},
+                                                       stream_names=[],
+                                                       instance_count=1,
+                                                       state=LogState.JOB_COMPLETE,
+                                                       last_description={},
+                                                       last_describe_job_call=0)
+        self.assertEqual(response, (LogState.COMPLETE, {}, 0))
+
+    @mock.patch.object(SageMakerHook, 'get_log_conn')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    def test_describe_training_job_with_logs_complete(self, mock_client, mock_log_client):
+        mock_session = mock.Mock()
+        mock_log_session = mock.Mock()
+        attrs = {'describe_training_job.return_value':
+                 DESCRIBE_TRAINING_COMPELETED_RETURN
+                 }
+        log_attrs = {'describe_log_streams.side_effect':
+                     LIFECYCLE_LOG_STREAMS,
+                     'get_log_events.side_effect':
+                     STREAM_LOG_EVENTS
+                     }
+        mock_session.configure_mock(**attrs)
+        mock_client.return_value = mock_session
+        mock_log_session.configure_mock(**log_attrs)
+        mock_log_client.return_value = mock_log_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id')
+        response = hook.describe_training_job_with_log(job_name=job_name,
+                                                       positions={},
+                                                       stream_names=[],
+                                                       instance_count=1,
+                                                       state=LogState.COMPLETE,
+                                                       last_description={},
+                                                       last_describe_job_call=0)
+        self.assertEqual(response, (LogState.COMPLETE, {}, 0))
+
+    @mock.patch.object(SageMakerHook, 'check_training_config')
+    @mock.patch.object(SageMakerHook, 'get_log_conn')
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'describe_training_job_with_log')
+    def test_training_with_logs(self, mock_describe, mock_client, mock_log_client, mock_check_training):
+        mock_check_training.return_value = True
+        mock_describe.side_effect = \
+            [(LogState.WAIT_IN_PROGRESS, DESCRIBE_TRAINING_INPROGRESS_RETURN, 0),
+             (LogState.JOB_COMPLETE, DESCRIBE_TRAINING_STOPPING_RETURN, 0),
+             (LogState.COMPLETE, DESCRIBE_TRAINING_COMPELETED_RETURN, 0)]
+        mock_session = mock.Mock()
+        mock_log_session = mock.Mock()
+        attrs = {'create_training_job.return_value':
+                 test_arn_return,
+                 'describe_training_job.return_value':
+                     DESCRIBE_TRAINING_COMPELETED_RETURN
+                 }
+        log_attrs = {'describe_log_streams.side_effect':
+                     LIFECYCLE_LOG_STREAMS,
+                     'get_log_events.side_effect':
+                     STREAM_LOG_EVENTS
+                     }
+        mock_session.configure_mock(**attrs)
+        mock_log_session.configure_mock(**log_attrs)
+        mock_client.return_value = mock_session
+        mock_log_client.return_value = mock_log_session
+        hook = SageMakerHook(aws_conn_id='sagemaker_test_conn_id_1')
+        hook.create_training_job(create_training_params, wait_for_completion=True,
+                                 print_log=True, check_interval=1)
+        self.assertEqual(mock_describe.call_count, 3)
+        self.assertEqual(mock_session.describe_training_job.call_count, 1)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/hooks/test_sftp_hook.py b/tests/contrib/hooks/test_sftp_hook.py
index 105b6b81d8478..bcdcb80dc3840 100644
--- a/tests/contrib/hooks/test_sftp_hook.py
+++ b/tests/contrib/hooks/test_sftp_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,6 +19,7 @@
 
 from __future__ import print_function
 
+import mock
 import unittest
 import shutil
 import os
@@ -26,6 +27,7 @@
 
 from airflow import configuration
 from airflow.contrib.hooks.sftp_hook import SFTPHook
+from airflow.models.connection import Connection
 
 TMP_PATH = '/tmp'
 TMP_DIR_FOR_TESTS = 'tests_sftp_hook_dir'
@@ -105,6 +107,63 @@ def test_get_mod_time(self):
             TMP_PATH, TMP_DIR_FOR_TESTS, TMP_FILE_FOR_TESTS))
         self.assertEqual(len(output), 14)
 
+    @mock.patch('airflow.contrib.hooks.sftp_hook.SFTPHook.get_connection')
+    def test_no_host_key_check_default(self, get_connection):
+        connection = Connection(login='login', host='host')
+        get_connection.return_value = connection
+        hook = SFTPHook()
+        self.assertEqual(hook.no_host_key_check, False)
+
+    @mock.patch('airflow.contrib.hooks.sftp_hook.SFTPHook.get_connection')
+    def test_no_host_key_check_enabled(self, get_connection):
+        connection = Connection(
+            login='login', host='host',
+            extra='{"no_host_key_check": true}')
+
+        get_connection.return_value = connection
+        hook = SFTPHook()
+        self.assertEqual(hook.no_host_key_check, True)
+
+    @mock.patch('airflow.contrib.hooks.sftp_hook.SFTPHook.get_connection')
+    def test_no_host_key_check_disabled(self, get_connection):
+        connection = Connection(
+            login='login', host='host',
+            extra='{"no_host_key_check": false}')
+
+        get_connection.return_value = connection
+        hook = SFTPHook()
+        self.assertEqual(hook.no_host_key_check, False)
+
+    @mock.patch('airflow.contrib.hooks.sftp_hook.SFTPHook.get_connection')
+    def test_no_host_key_check_disabled_for_all_but_true(self, get_connection):
+        connection = Connection(
+            login='login', host='host',
+            extra='{"no_host_key_check": "foo"}')
+
+        get_connection.return_value = connection
+        hook = SFTPHook()
+        self.assertEqual(hook.no_host_key_check, False)
+
+    @mock.patch('airflow.contrib.hooks.sftp_hook.SFTPHook.get_connection')
+    def test_no_host_key_check_ignore(self, get_connection):
+        connection = Connection(
+            login='login', host='host',
+            extra='{"ignore_hostkey_verification": true}')
+
+        get_connection.return_value = connection
+        hook = SFTPHook()
+        self.assertEqual(hook.no_host_key_check, True)
+
+    @mock.patch('airflow.contrib.hooks.sftp_hook.SFTPHook.get_connection')
+    def test_no_host_key_check_no_ignore(self, get_connection):
+        connection = Connection(
+            login='login', host='host',
+            extra='{"ignore_hostkey_verification": false}')
+
+        get_connection.return_value = connection
+        hook = SFTPHook()
+        self.assertEqual(hook.no_host_key_check, False)
+
     def tearDown(self):
         shutil.rmtree(os.path.join(TMP_PATH, TMP_DIR_FOR_TESTS))
         os.remove(os.path.join(TMP_PATH, TMP_FILE_FOR_TESTS))
diff --git a/tests/contrib/hooks/test_slack_webhook_hook.py b/tests/contrib/hooks/test_slack_webhook_hook.py
index 3f657aa741e31..5d92f86e6f5c1 100644
--- a/tests/contrib/hooks/test_slack_webhook_hook.py
+++ b/tests/contrib/hooks/test_slack_webhook_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,7 +20,8 @@
 import json
 import unittest
 
-from airflow import configuration, models
+from airflow import configuration
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 from airflow.contrib.hooks.slack_webhook_hook import SlackWebhookHook
@@ -32,24 +33,27 @@ class TestSlackWebhookHook(unittest.TestCase):
         'http_conn_id': 'slack-webhook-default',
         'webhook_token': 'manual_token',
         'message': 'Awesome message to put on Slack',
+        'attachments': [{'fallback': 'Required plain-text summary'}],
         'channel': '#general',
         'username': 'SlackMcSlackFace',
         'icon_emoji': ':hankey:',
         'link_names': True,
         'proxy': 'https://my-horrible-proxy.proxyist.com:8080'
     }
-    expected_message_dict = {'channel': _config['channel'],
-                             'username': _config['username'],
-                             'icon_emoji': _config['icon_emoji'],
-                             'link_names': 1,
-                             'text': _config['message']
-                             }
+    expected_message_dict = {
+        'channel': _config['channel'],
+        'username': _config['username'],
+        'icon_emoji': _config['icon_emoji'],
+        'link_names': 1,
+        'attachments': _config['attachments'],
+        'text': _config['message']
+    }
     expected_message = json.dumps(expected_message_dict)
 
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='slack-webhook-default',
                 extra='{"webhook_token": "your_token_here"}')
         )
diff --git a/tests/contrib/hooks/test_snowflake_hook.py b/tests/contrib/hooks/test_snowflake_hook.py
index aaaf57a00b8d8..a6bc375be60e6 100644
--- a/tests/contrib/hooks/test_snowflake_hook.py
+++ b/tests/contrib/hooks/test_snowflake_hook.py
@@ -8,9 +8,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/hooks/test_spark_jdbc_hook.py b/tests/contrib/hooks/test_spark_jdbc_hook.py
index 2451b4a157c60..41bc1f3e30cfb 100644
--- a/tests/contrib/hooks/test_spark_jdbc_hook.py
+++ b/tests/contrib/hooks/test_spark_jdbc_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,8 @@
 #
 import unittest
 
-from airflow import configuration, models
+from airflow import configuration
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 from airflow.contrib.hooks.spark_jdbc_hook import SparkJDBCHook
@@ -67,13 +68,13 @@ class TestSparkJDBCHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark-default', conn_type='spark',
                 host='yarn://yarn-master',
                 extra='{"queue": "root.etl", "deploy-mode": "cluster"}')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='jdbc-default', conn_type='postgres',
                 host='localhost', schema='default', port=5432,
                 login='user', password='supersecret',
@@ -125,7 +126,7 @@ def test_build_jdbc_arguments(self):
             '-createTableColumnTypes', 'columnMcColumnFace INTEGER(100), name CHAR(64),'
                                        'comments VARCHAR(1024)'
         ]
-        self.assertEquals(expected_jdbc_arguments, cmd)
+        self.assertEqual(expected_jdbc_arguments, cmd)
 
     def test_build_jdbc_arguments_invalid(self):
         # Given
diff --git a/tests/contrib/hooks/test_spark_sql_hook.py b/tests/contrib/hooks/test_spark_sql_hook.py
index c95ee3fb130d0..04d5439b9bcc1 100644
--- a/tests/contrib/hooks/test_spark_sql_hook.py
+++ b/tests/contrib/hooks/test_spark_sql_hook.py
@@ -19,14 +19,13 @@
 #
 
 import six
-import sys
 import unittest
-from io import StringIO
 from itertools import dropwhile
 
 from mock import patch, call
 
-from airflow import configuration, models
+from airflow import configuration
+from airflow.models.connection import Connection
 from airflow.utils import db
 from airflow.contrib.hooks.spark_sql_hook import SparkSqlHook
 
@@ -37,8 +36,8 @@ def get_after(sentinel, iterable):
     next(truncated)
     return next(truncated)
 
-class TestSparkSqlHook(unittest.TestCase):
 
+class TestSparkSqlHook(unittest.TestCase):
     _config = {
         'conn_id': 'spark_default',
         'executor_cores': 4,
@@ -55,7 +54,7 @@ def setUp(self):
 
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_default', conn_type='spark',
                 host='yarn://yarn-master')
         )
@@ -100,7 +99,8 @@ def test_spark_process_runcmd(self, mock_popen):
                 hook.run_query()
                 mock_debug.assert_called_with(
                     'Spark-Sql cmd: %s',
-                    ['spark-sql', '-e', 'SELECT 1', '--master', 'yarn', '--name', 'default-name', '--verbose', '--queue', 'default']
+                    ['spark-sql', '-e', 'SELECT 1', '--master', 'yarn', '--name', 'default-name', '--verbose',
+                     '--queue', 'default']
                 )
                 mock_info.assert_called_with(
                     'Spark-sql communicates using stdout'
@@ -109,7 +109,8 @@ def test_spark_process_runcmd(self, mock_popen):
         # Then
         self.assertEqual(
             mock_popen.mock_calls[0],
-            call(['spark-sql', '-e', 'SELECT 1', '--master', 'yarn', '--name', 'default-name', '--verbose', '--queue', 'default'], stderr=-2, stdout=-1)
+            call(['spark-sql', '-e', 'SELECT 1', '--master', 'yarn', '--name', 'default-name', '--verbose',
+                  '--queue', 'default'], stderr=-2, stdout=-1)
         )
 
 
diff --git a/tests/contrib/hooks/test_spark_submit_hook.py b/tests/contrib/hooks/test_spark_submit_hook.py
index 1bdcda56ec6f6..45b10d42bce5b 100644
--- a/tests/contrib/hooks/test_spark_submit_hook.py
+++ b/tests/contrib/hooks/test_spark_submit_hook.py
@@ -20,7 +20,8 @@
 import six
 import unittest
 
-from airflow import configuration, models, AirflowException
+from airflow import configuration, AirflowException
+from airflow.models.connection import Connection
 from airflow.utils import db
 from mock import patch, call
 
@@ -37,6 +38,7 @@ class TestSparkSubmitHook(unittest.TestCase):
         'conn_id': 'default_spark',
         'files': 'hive-site.xml',
         'py_files': 'sample_library.py',
+        'archives': 'sample_archive.zip#SAMPLE',
         'jars': 'parquet.jar',
         'packages': 'com.databricks:spark-avro_2.11:3.2.0',
         'exclude_packages': 'org.bad.dependency:1.0.0',
@@ -72,13 +74,13 @@ def setUp(self):
 
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_yarn_cluster', conn_type='spark',
                 host='yarn://yarn-master',
                 extra='{"queue": "root.etl", "deploy-mode": "cluster"}')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_k8s_cluster', conn_type='spark',
                 host='k8s://https://k8s-master',
                 extra='{"spark-home": "/opt/spark", ' +
@@ -86,43 +88,43 @@ def setUp(self):
                       '"namespace": "mynamespace"}')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_default_mesos', conn_type='spark',
                 host='mesos://host', port=5050)
         )
 
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_home_set', conn_type='spark',
                 host='yarn://yarn-master',
                 extra='{"spark-home": "/opt/myspark"}')
         )
 
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_home_not_set', conn_type='spark',
                 host='yarn://yarn-master')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_binary_set', conn_type='spark',
                 host='yarn', extra='{"spark-binary": "custom-spark-submit"}')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_binary_and_home_set', conn_type='spark',
                 host='yarn',
                 extra='{"spark-home": "/path/to/spark_home", ' +
                       '"spark-binary": "custom-spark-submit"}')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_standalone_cluster', conn_type='spark',
                 host='spark://spark-standalone-master:6066',
                 extra='{"spark-home": "/path/to/spark_home", "deploy-mode": "cluster"}')
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='spark_standalone_cluster_client_mode', conn_type='spark',
                 host='spark://spark-standalone-master:6066',
                 extra='{"spark-home": "/path/to/spark_home", "deploy-mode": "client"}')
@@ -142,6 +144,7 @@ def test_build_spark_submit_command(self):
             '--conf', 'parquet.compression=SNAPPY',
             '--files', 'hive-site.xml',
             '--py-files', 'sample_library.py',
+            '--archives', 'sample_archive.zip#SAMPLE',
             '--jars', 'parquet.jar',
             '--packages', 'com.databricks:spark-avro_2.11:3.2.0',
             '--exclude-packages', 'org.bad.dependency:1.0.0',
@@ -162,7 +165,7 @@ def test_build_spark_submit_command(self):
             '--with-spaces', 'args should keep embdedded spaces',
             'baz'
         ]
-        self.assertEquals(expected_build_cmd, cmd)
+        self.assertEqual(expected_build_cmd, cmd)
 
     @patch('airflow.contrib.hooks.spark_submit_hook.subprocess.Popen')
     def test_spark_process_runcmd(self, mock_popen):
diff --git a/tests/contrib/hooks/test_sqoop_hook.py b/tests/contrib/hooks/test_sqoop_hook.py
index 47c9a7be8e283..3d1589e3ee1c8 100644
--- a/tests/contrib/hooks/test_sqoop_hook.py
+++ b/tests/contrib/hooks/test_sqoop_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,9 +22,10 @@
 import json
 import unittest
 
-from airflow import configuration, models
+from airflow import configuration
 from airflow.contrib.hooks.sqoop_hook import SqoopHook
 from airflow.exceptions import AirflowException
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 from mock import patch, call
@@ -59,7 +60,8 @@ class TestSqoopHook(unittest.TestCase):
         'relaxed_isolation': True,
         'extra_export_options': collections.OrderedDict([
             ('update-key', 'id'),
-            ('update-mode', 'allowinsert')
+            ('update-mode', 'allowinsert'),
+            ('fetch-size', 1)
         ])
     }
     _config_import = {
@@ -71,7 +73,8 @@ class TestSqoopHook(unittest.TestCase):
         'driver': 'com.microsoft.jdbc.sqlserver.SQLServerDriver',
         'extra_import_options': {
             'hcatalog-storage-stanza': "\"stored as orcfile\"",
-            'show': ''
+            'show': '',
+            'fetch-size': 1
         }
     }
 
@@ -86,7 +89,7 @@ class TestSqoopHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='sqoop_test', conn_type='sqoop', schema='schema',
                 host='rmdbs', port=5050, extra=json.dumps(self._config_json)
             )
@@ -98,7 +101,8 @@ def test_popen(self, mock_popen):
         mock_popen.return_value.stdout = StringIO(u'stdout')
         mock_popen.return_value.stderr = StringIO(u'stderr')
         mock_popen.return_value.returncode = 0
-        mock_popen.return_value.communicate.return_value = [StringIO(u'stdout\nstdout'), StringIO(u'stderr\nstderr')]
+        mock_popen.return_value.communicate.return_value = \
+            [StringIO(u'stdout\nstdout'), StringIO(u'stderr\nstderr')]
 
         # When
         hook = SqoopHook(conn_id='sqoop_test')
@@ -128,6 +132,7 @@ def test_popen(self, mock_popen):
              '--export-dir', self._config_export['export_dir'],
              '--update-key', 'id',
              '--update-mode', 'allowinsert',
+             '--fetch-size', str(self._config_export['extra_export_options'].get('fetch-size')),
              '--table', self._config_export['table']], stderr=-2, stdout=-1))
 
     def test_submit_none_mappers(self):
@@ -163,7 +168,7 @@ def test_submit(self):
             self.assertIn("-files {}".format(self._config_json['files']), cmd)
 
         if self._config_json['archives']:
-            self.assertIn( "-archives {}".format(self._config_json['archives']), cmd)
+            self.assertIn("-archives {}".format(self._config_json['archives']), cmd)
 
         self.assertIn("--hcatalog-database {}".format(self._config['hcatalog_database']), cmd)
         self.assertIn("--hcatalog-table {}".format(self._config['hcatalog_table']), cmd)
@@ -173,7 +178,7 @@ def test_submit(self):
             self.assertIn("--verbose", cmd)
 
         if self._config['num_mappers']:
-            self.assertIn( "--num-mappers {}".format(self._config['num_mappers']), cmd)
+            self.assertIn("--num-mappers {}".format(self._config['num_mappers']), cmd)
 
         for key, value in self._config['properties'].items():
             self.assertIn("-D {}={}".format(key, value), cmd)
@@ -248,6 +253,11 @@ def test_export_cmd(self):
         if self._config_export['relaxed_isolation']:
             self.assertIn("--relaxed-isolation", cmd)
 
+        if self._config_export['extra_export_options']:
+            self.assertIn("--update-key", cmd)
+            self.assertIn("--update-mode", cmd)
+            self.assertIn("--fetch-size", cmd)
+
     def test_import_cmd(self):
         """
         Tests to verify the hook import command is building correct Sqoop import command.
@@ -298,10 +308,12 @@ def test_import_cmd(self):
         # these checks are from the extra import options
         self.assertIn('--show', cmd)
         self.assertIn('hcatalog-storage-stanza \"stored as orcfile\"', cmd)
+        self.assertIn('--fetch-size', cmd)
 
     def test_get_export_format_argument(self):
         """
-        Tests to verify the hook get format function is building correct Sqoop command with correct format type.
+        Tests to verify the hook get format function is building
+        correct Sqoop command with correct format type.
         """
         hook = SqoopHook()
         self.assertIn("--as-avrodatafile",
diff --git a/tests/contrib/hooks/test_ssh_hook.py b/tests/contrib/hooks/test_ssh_hook.py
index b185a565c2f69..6cc1e0df88070 100644
--- a/tests/contrib/hooks/test_ssh_hook.py
+++ b/tests/contrib/hooks/test_ssh_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,8 +19,12 @@
 
 import unittest
 from airflow import configuration
+from airflow.models.connection import Connection
 from airflow.utils import db
-from airflow import models
+from tests.compat import mock
+
+from airflow.contrib.hooks.ssh_hook import SSHHook
+
 
 HELLO_SERVER_CMD = """
 import socket, sys
@@ -36,54 +40,134 @@
 
 
 class SSHHookTest(unittest.TestCase):
+
     def setUp(self):
         configuration.load_test_config()
-        from airflow.contrib.hooks.ssh_hook import SSHHook
-        self.hook = SSHHook(ssh_conn_id='ssh_default', keepalive_interval=10)
-        self.hook.no_host_key_check = True
+
+    @mock.patch('airflow.contrib.hooks.ssh_hook.paramiko.SSHClient')
+    def test_ssh_connection_with_password(self, ssh_mock):
+        hook = SSHHook(remote_host='remote_host',
+                       port='port',
+                       username='username',
+                       password='password',
+                       timeout=10,
+                       key_file='fake.file')
+
+        with hook.get_conn():
+            ssh_mock.return_value.connect.assert_called_once_with(
+                hostname='remote_host',
+                username='username',
+                password='password',
+                key_filename='fake.file',
+                timeout=10,
+                compress=True,
+                port='port',
+                sock=None
+            )
+
+    @mock.patch('airflow.contrib.hooks.ssh_hook.paramiko.SSHClient')
+    def test_ssh_connection_without_password(self, ssh_mock):
+        hook = SSHHook(remote_host='remote_host',
+                       port='port',
+                       username='username',
+                       timeout=10,
+                       key_file='fake.file')
+
+        with hook.get_conn():
+            ssh_mock.return_value.connect.assert_called_once_with(
+                hostname='remote_host',
+                username='username',
+                key_filename='fake.file',
+                timeout=10,
+                compress=True,
+                port='port',
+                sock=None
+            )
+
+    @mock.patch('airflow.contrib.hooks.ssh_hook.SSHTunnelForwarder')
+    def test_tunnel_with_password(self, ssh_mock):
+        hook = SSHHook(remote_host='remote_host',
+                       port='port',
+                       username='username',
+                       password='password',
+                       timeout=10,
+                       key_file='fake.file')
+
+        with hook.get_tunnel(1234):
+            ssh_mock.assert_called_once_with('remote_host',
+                                             ssh_port='port',
+                                             ssh_username='username',
+                                             ssh_password='password',
+                                             ssh_pkey='fake.file',
+                                             ssh_proxy=None,
+                                             local_bind_address=('localhost', ),
+                                             remote_bind_address=('localhost', 1234),
+                                             logger=hook.log)
+
+    @mock.patch('airflow.contrib.hooks.ssh_hook.SSHTunnelForwarder')
+    def test_tunnel_without_password(self, ssh_mock):
+        hook = SSHHook(remote_host='remote_host',
+                       port='port',
+                       username='username',
+                       timeout=10,
+                       key_file='fake.file')
+
+        with hook.get_tunnel(1234):
+            ssh_mock.assert_called_once_with('remote_host',
+                                             ssh_port='port',
+                                             ssh_username='username',
+                                             ssh_pkey='fake.file',
+                                             ssh_proxy=None,
+                                             local_bind_address=('localhost', ),
+                                             remote_bind_address=('localhost', 1234),
+                                             host_pkey_directories=[],
+                                             logger=hook.log)
+
+    def test_conn_with_extra_parameters(self):
+        db.merge_conn(
+            Connection(
+                conn_id='ssh_with_extra',
+                host='localhost',
+                conn_type='ssh',
+                extra='{"compress" : true, "no_host_key_check" : "true", '
+                      '"allow_host_key_change": false}'
+            )
+        )
+        ssh_hook = SSHHook(ssh_conn_id='ssh_with_extra')
+        self.assertEqual(ssh_hook.compress, True)
+        self.assertEqual(ssh_hook.no_host_key_check, True)
+        self.assertEqual(ssh_hook.allow_host_key_change, False)
 
     def test_ssh_connection(self):
-        ssh_hook = self.hook.get_conn()
-        self.assertIsNotNone(ssh_hook)
+        hook = SSHHook(ssh_conn_id='ssh_default')
+        with hook.get_conn() as client:
+            (_, stdout, _) = client.exec_command('ls')
+            self.assertIsNotNone(stdout.read())
+
+    def test_ssh_connection_old_cm(self):
+        with SSHHook(ssh_conn_id='ssh_default') as hook:
+            client = hook.get_conn()
+            (_, stdout, _) = client.exec_command('ls')
+            self.assertIsNotNone(stdout.read())
 
     def test_tunnel(self):
-        print("Setting up remote listener")
+        hook = SSHHook(ssh_conn_id='ssh_default')
+
         import subprocess
         import socket
 
-        self.server_handle = subprocess.Popen(["python", "-c", HELLO_SERVER_CMD],
-                                              stdout=subprocess.PIPE)
-        print("Setting up tunnel")
-        with self.hook.create_tunnel(2135, 2134):
-            print("Tunnel up")
-            server_output = self.server_handle.stdout.read(5)
+        server_handle = subprocess.Popen(["python", "-c", HELLO_SERVER_CMD],
+                                         stdout=subprocess.PIPE)
+        with hook.create_tunnel(2135, 2134):
+            server_output = server_handle.stdout.read(5)
             self.assertEqual(server_output, b"ready")
-            print("Connecting to server via tunnel")
             s = socket.socket()
             s.connect(("localhost", 2135))
-            print("Receiving...",)
             response = s.recv(5)
             self.assertEqual(response, b"hello")
-            print("Closing connection")
             s.close()
-            print("Waiting for listener...")
-            output, _ = self.server_handle.communicate()
-            self.assertEqual(self.server_handle.returncode, 0)
-            print("Closing tunnel")
-
-    def test_conn_with_extra_parameters(self):
-        from airflow.contrib.hooks.ssh_hook import SSHHook
-        db.merge_conn(
-            models.Connection(conn_id='ssh_with_extra',
-                              host='localhost',
-                              conn_type='ssh',
-                              extra='{"compress" : true, "no_host_key_check" : "true"}'
-                              )
-        )
-        ssh_hook = SSHHook(ssh_conn_id='ssh_with_extra', keepalive_interval=10)
-        ssh_hook.get_conn()
-        self.assertEqual(ssh_hook.compress, True)
-        self.assertEqual(ssh_hook.no_host_key_check, True)
+            output, _ = server_handle.communicate()
+            self.assertEqual(server_handle.returncode, 0)
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/hooks/test_wasb_hook.py b/tests/contrib/hooks/test_wasb_hook.py
index 7a81512835a76..c9458287e9097 100644
--- a/tests/contrib/hooks/test_wasb_hook.py
+++ b/tests/contrib/hooks/test_wasb_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,19 +21,13 @@
 
 import json
 import unittest
+from collections import namedtuple
 
-from airflow import configuration
-from airflow import models
+from airflow import configuration, AirflowException
 from airflow.contrib.hooks.wasb_hook import WasbHook
+from airflow.models.connection import Connection
 from airflow.utils import db
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class TestWasbHook(unittest.TestCase):
@@ -41,13 +35,13 @@ class TestWasbHook(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='wasb_test_key', conn_type='wasb',
                 login='login', password='key'
             )
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='wasb_test_sas_token', conn_type='wasb',
                 login='login', extra=json.dumps({'sas_token': 'token'})
             )
@@ -143,6 +137,59 @@ def test_read_file(self, mock_service):
             'container', 'blob', max_connections=1
         )
 
+    @mock.patch('airflow.contrib.hooks.wasb_hook.BlockBlobService',
+                autospec=True)
+    def test_delete_single_blob(self, mock_service):
+        mock_instance = mock_service.return_value
+        hook = WasbHook(wasb_conn_id='wasb_test_sas_token')
+        hook.delete_file('container', 'blob', is_prefix=False)
+        mock_instance.delete_blob.assert_called_once_with(
+            'container', 'blob', delete_snapshots='include'
+        )
+
+    @mock.patch('airflow.contrib.hooks.wasb_hook.BlockBlobService',
+                autospec=True)
+    def test_delete_multiple_blobs(self, mock_service):
+        mock_instance = mock_service.return_value
+        Blob = namedtuple('Blob', ['name'])
+        mock_instance.list_blobs.return_value = iter(
+            [Blob('blob_prefix/blob1'), Blob('blob_prefix/blob2')]
+        )
+        hook = WasbHook(wasb_conn_id='wasb_test_sas_token')
+        hook.delete_file('container', 'blob_prefix', is_prefix=True)
+        mock_instance.delete_blob.assert_any_call(
+            'container', 'blob_prefix/blob1', delete_snapshots='include'
+        )
+        mock_instance.delete_blob.assert_any_call(
+            'container', 'blob_prefix/blob2', delete_snapshots='include'
+        )
+
+    @mock.patch('airflow.contrib.hooks.wasb_hook.BlockBlobService',
+                autospec=True)
+    def test_delete_nonexisting_blob_fails(self, mock_service):
+        mock_instance = mock_service.return_value
+        mock_instance.exists.return_value = False
+        hook = WasbHook(wasb_conn_id='wasb_test_sas_token')
+        with self.assertRaises(Exception) as context:
+            hook.delete_file(
+                'container', 'nonexisting_blob',
+                is_prefix=False, ignore_if_missing=False
+            )
+        self.assertIsInstance(context.exception, AirflowException)
+
+    @mock.patch('airflow.contrib.hooks.wasb_hook.BlockBlobService',
+                autospec=True)
+    def test_delete_multiple_nonexisting_blobs_fails(self, mock_service):
+        mock_instance = mock_service.return_value
+        mock_instance.list_blobs.return_value = iter([])
+        hook = WasbHook(wasb_conn_id='wasb_test_sas_token')
+        with self.assertRaises(Exception) as context:
+            hook.delete_file(
+                'container', 'nonexisting_blob_prefix',
+                is_prefix=True, ignore_if_missing=False
+            )
+        self.assertIsInstance(context.exception, AirflowException)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/hooks/test_zendesk_hook.py b/tests/contrib/hooks/test_zendesk_hook.py
index 799919b3a0d3b..51be042086969 100644
--- a/tests/contrib/hooks/test_zendesk_hook.py
+++ b/tests/contrib/hooks/test_zendesk_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -92,8 +92,8 @@ def test_zdesk_is_inited_correctly(self, mock_zendesk):
         zendesk_hook = ZendeskHook("conn_id")
         zendesk_hook.get_connection = mock.Mock(return_value=conn_mock)
         zendesk_hook.get_conn()
-        mock_zendesk.assert_called_with('https://conn_host', 'conn_login',
-                                        'conn_pass', True)
+        mock_zendesk.assert_called_with(zdesk_url='https://conn_host', zdesk_email='conn_login',
+                                        zdesk_password='conn_pass', zdesk_token=True)
 
     @mock.patch("airflow.hooks.zendesk_hook.Zendesk")
     def test_zdesk_sideloading_works_correctly(self, mock_zendesk):
diff --git a/tests/contrib/kubernetes/__init__.py b/tests/contrib/kubernetes/__init__.py
new file mode 100644
index 0000000000000..114d189da14ab
--- /dev/null
+++ b/tests/contrib/kubernetes/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/contrib/kubernetes/kubernetes_request_factory/__init__.py b/tests/contrib/kubernetes/kubernetes_request_factory/__init__.py
new file mode 100644
index 0000000000000..114d189da14ab
--- /dev/null
+++ b/tests/contrib/kubernetes/kubernetes_request_factory/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/contrib/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py b/tests/contrib/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py
new file mode 100644
index 0000000000000..e7f444abb14a1
--- /dev/null
+++ b/tests/contrib/kubernetes/kubernetes_request_factory/test_kubernetes_request_factory.py
@@ -0,0 +1,306 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.kubernetes.kubernetes_request_factory.\
+    kubernetes_request_factory import KubernetesRequestFactory
+from airflow.contrib.kubernetes.pod import Pod, Resources
+from airflow.contrib.kubernetes.secret import Secret
+from parameterized import parameterized
+import unittest
+import copy
+
+
+class TestKubernetesRequestFactory(unittest.TestCase):
+
+    def setUp(self):
+
+        self.expected = {
+            'apiVersion': 'v1',
+            'kind': 'Pod',
+            'metadata': {
+                'name': 'name'
+            },
+            'spec': {
+                'containers': [{
+                    'name': 'base',
+                    'image': 'airflow-worker:latest',
+                    'command': [
+                        "/usr/local/airflow/entrypoint.sh",
+                        "/bin/bash sleep 25"
+                    ],
+                }],
+                'restartPolicy': 'Never'
+            }
+        }
+        self.input_req = copy.deepcopy(self.expected)
+
+    def test_extract_image(self):
+        image = 'v3.14'
+        pod = Pod(image, {}, [])
+        KubernetesRequestFactory.extract_image(pod, self.input_req)
+        self.expected['spec']['containers'][0]['image'] = image
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_image_pull_policy(self):
+        # Test when pull policy is not none
+        pull_policy = 'IfNotPresent'
+        pod = Pod('v3.14', {}, [], image_pull_policy=pull_policy)
+
+        KubernetesRequestFactory.extract_image_pull_policy(pod, self.input_req)
+        self.expected['spec']['containers'][0]['imagePullPolicy'] = pull_policy
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_add_secret_to_env(self):
+        secret = Secret('env', 'target', 'my-secret', 'KEY')
+        secret_list = []
+        self.expected = [{
+            'name': 'TARGET',
+            'valueFrom': {
+                'secretKeyRef': {
+                    'name': 'my-secret',
+                    'key': 'KEY'
+                }
+            }
+        }]
+        KubernetesRequestFactory.add_secret_to_env(secret_list, secret)
+        self.assertListEqual(secret_list, self.expected)
+
+    def test_extract_labels(self):
+        # Test when labels are not empty
+        labels = {'label_a': 'val_a', 'label_b': 'val_b'}
+        pod = Pod('v3.14', {}, [], labels=labels)
+        self.expected['metadata']['labels'] = labels
+        KubernetesRequestFactory.extract_labels(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_annotations(self):
+        # Test when annotations are not empty
+        annotations = {'annot_a': 'val_a', 'annot_b': 'val_b'}
+        pod = Pod('v3.14', {}, [], annotations=annotations)
+        self.expected['metadata']['annotations'] = annotations
+        KubernetesRequestFactory.extract_annotations(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_affinity(self):
+        # Test when affinity is not empty
+        affinity = {'podAffinity': 'requiredDuringSchedulingIgnoredDuringExecution'}
+        pod = Pod('v3.14', {}, [], affinity=affinity)
+        self.expected['spec']['affinity'] = affinity
+        KubernetesRequestFactory.extract_affinity(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_node_selector(self):
+        # Test when affinity is not empty
+        node_selectors = {'disktype': 'ssd', 'accelerator': 'nvidia-tesla-p100'}
+        pod = Pod('v3.14', {}, [], node_selectors=node_selectors)
+        self.expected['spec']['nodeSelector'] = node_selectors
+        KubernetesRequestFactory.extract_node_selector(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_cmds(self):
+        cmds = ['test-cmd.sh']
+        pod = Pod('v3.14', {}, cmds)
+        KubernetesRequestFactory.extract_cmds(pod, self.input_req)
+        self.expected['spec']['containers'][0]['command'] = cmds
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_args(self):
+        args = ['test_arg.sh']
+        pod = Pod('v3.14', {}, [], args=args)
+        KubernetesRequestFactory.extract_args(pod, self.input_req)
+        self.expected['spec']['containers'][0]['args'] = args
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_attach_volumes(self):
+        # Test when volumes is not empty list
+        volumes = ['vol_a', 'vol_b']
+        pod = Pod('v3.14', {}, [], volumes=volumes)
+        self.expected['spec']['volumes'] = volumes
+        KubernetesRequestFactory.attach_volumes(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_attach_volume_mounts(self):
+        # Test when volumes is not empty list
+        volume_mounts = ['vol_a', 'vol_b']
+        pod = Pod('v3.14', {}, [], volume_mounts=volume_mounts)
+        self.expected['spec']['containers'][0]['volumeMounts'] = volume_mounts
+        KubernetesRequestFactory.attach_volume_mounts(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_name(self):
+        name = 'pod-name'
+        pod = Pod('v3.14', {}, [], name=name)
+        self.expected['metadata']['name'] = name
+        KubernetesRequestFactory.extract_name(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_volume_secrets(self):
+        # Test when secrets is not empty
+        secrets = [
+            Secret('volume', 'KEY1', 's1', 'key-1'),
+            Secret('env', 'KEY2', 's2'),
+            Secret('volume', 'KEY3', 's3', 'key-2')
+        ]
+        pod = Pod('v3.14', {}, [], secrets=secrets)
+        self.expected['spec']['containers'][0]['volumeMounts'] = [{
+            'mountPath': 'KEY1',
+            'name': 'secretvol0',
+            'readOnly': True
+        }, {
+            'mountPath': 'KEY3',
+            'name': 'secretvol1',
+            'readOnly': True
+        }]
+        self.expected['spec']['volumes'] = [{
+            'name': 'secretvol0',
+            'secret': {
+                'secretName': 's1'
+            }
+        }, {
+            'name': 'secretvol1',
+            'secret': {
+                'secretName': 's3'
+            }
+        }]
+        KubernetesRequestFactory.extract_volume_secrets(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_env_and_secrets(self):
+        # Test when secrets and envs are not empty
+        secrets = [
+            Secret('env', None, 's1'),
+            Secret('volume', 'KEY2', 's2', 'key-2'),
+            Secret('env', None, 's3')
+        ]
+        envs = {
+            'ENV1': 'val1',
+            'ENV2': 'val2'
+        }
+        configmaps = ['configmap_a', 'configmap_b']
+        pod = Pod('v3.14', envs, [], secrets=secrets, configmaps=configmaps)
+        self.expected['spec']['containers'][0]['env'] = [
+            {'name': 'ENV1', 'value': 'val1'},
+            {'name': 'ENV2', 'value': 'val2'},
+        ]
+        self.expected['spec']['containers'][0]['envFrom'] = [{
+            'secretRef': {
+                'name': 's1'
+            }
+        }, {
+            'secretRef': {
+                'name': 's3'
+            }
+        }, {
+            'configMapRef': {
+                'name': 'configmap_a'
+            }
+        }, {
+            'configMapRef': {
+                'name': 'configmap_b'
+            }
+        }]
+
+        KubernetesRequestFactory.extract_env_and_secrets(pod, self.input_req)
+        self.input_req['spec']['containers'][0]['env'].sort(key=lambda x: x['name'])
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_resources(self):
+        # Test when resources is not empty
+        resources = Resources('1Gi', 1, '2Gi', 2)
+        pod = Pod('v3.14', {}, [], resources=resources)
+        self.expected['spec']['containers'][0]['resources'] = {
+            'requests': {
+                'memory': '1Gi',
+                'cpu': 1
+            },
+            'limits': {
+                'memory': '2Gi',
+                'cpu': 2
+            },
+        }
+        KubernetesRequestFactory.extract_resources(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_init_containers(self):
+        init_container = 'init_container'
+        pod = Pod('v3.14', {}, [], init_containers=init_container)
+        self.expected['spec']['initContainers'] = init_container
+        KubernetesRequestFactory.extract_init_containers(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_service_account_name(self):
+        service_account_name = 'service_account_name'
+        pod = Pod('v3.14', {}, [], service_account_name=service_account_name)
+        self.expected['spec']['serviceAccountName'] = service_account_name
+        KubernetesRequestFactory.extract_service_account_name(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_hostnetwork(self):
+        hostnetwork = True
+        pod = Pod('v3.14', {}, [], hostnetwork=hostnetwork)
+        self.expected['spec']['hostNetwork'] = hostnetwork
+        KubernetesRequestFactory.extract_hostnetwork(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_image_pull_secrets(self):
+        image_pull_secrets = 'secret_a,secret_b,secret_c'
+        pod = Pod('v3.14', {}, [], image_pull_secrets=image_pull_secrets)
+        self.expected['spec']['imagePullSecrets'] = [
+            {'name': 'secret_a'},
+            {'name': 'secret_b'},
+            {'name': 'secret_c'},
+        ]
+        KubernetesRequestFactory.extract_image_pull_secrets(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_extract_tolerations(self):
+        tolerations = [{
+            'key': 'key',
+            'operator': 'Equal',
+            'value': 'value',
+            'effect': 'NoSchedule'
+        }]
+        pod = Pod('v3.14', {}, [], tolerations=tolerations)
+        self.expected['spec']['tolerations'] = tolerations
+        KubernetesRequestFactory.extract_tolerations(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    def test_security_context(self):
+        security_context = {
+            'runAsUser': 1000,
+            'fsGroup': 2000
+        }
+        pod = Pod('v3.14', {}, [], security_context=security_context)
+        self.expected['spec']['securityContext'] = security_context
+        KubernetesRequestFactory.extract_security_context(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
+
+    @parameterized.expand([
+        'extract_resources',
+        'extract_init_containers',
+        'extract_service_account_name',
+        'extract_hostnetwork',
+        'extract_image_pull_secrets',
+        'extract_tolerations',
+        'extract_security_context',
+        'extract_volume_secrets'
+    ])
+    def test_identity(self, name):
+        kube_request_factory_func = getattr(KubernetesRequestFactory, name)
+        pod = Pod('v3.14', {}, [])
+        kube_request_factory_func(pod, self.input_req)
+        self.assertEqual(self.input_req, self.expected)
diff --git a/tests/contrib/kubernetes/kubernetes_request_factory/test_pod_request_factory.py b/tests/contrib/kubernetes/kubernetes_request_factory/test_pod_request_factory.py
new file mode 100644
index 0000000000000..9492ec0e25fe8
--- /dev/null
+++ b/tests/contrib/kubernetes/kubernetes_request_factory/test_pod_request_factory.py
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.contrib.kubernetes.kubernetes_request_factory.\
+    pod_request_factory import SimplePodRequestFactory, \
+    ExtractXcomPodRequestFactory
+from airflow.contrib.kubernetes.pod import Pod
+from airflow.contrib.kubernetes.secret import Secret
+from airflow.exceptions import AirflowConfigException
+import unittest
+
+XCOM_CMD = """import time
+while True:
+    try:
+        time.sleep(3600)
+    except KeyboardInterrupt:
+        exit(0)
+"""
+
+
+class TestPodRequestFactory(unittest.TestCase):
+
+    def setUp(self):
+        self.simple_pod_request_factory = SimplePodRequestFactory()
+        self.xcom_pod_request_factory = ExtractXcomPodRequestFactory()
+        self.pod = Pod(
+            image='busybox',
+            envs={
+                'ENVIRONMENT': 'prod',
+                'LOG_LEVEL': 'warning'
+            },
+            name='myapp-pod',
+            cmds=['sh', '-c', 'echo Hello Kubernetes!'],
+            labels={'app': 'myapp'},
+            image_pull_secrets='pull_secret_a,pull_secret_b',
+            configmaps=['configmap_a', 'configmap_b'],
+            secrets=[
+                # This should be a secretRef
+                Secret('env', None, 'secret_a'),
+                # This should be a single secret mounted in volumeMounts
+                Secret('volume', '/etc/foo', 'secret_b'),
+                # This should produce a single secret mounted in env
+                Secret('env', 'TARGET', 'secret_b', 'source_b'),
+            ],
+            security_context={
+                'runAsUser': 1000,
+                'fsGroup': 2000,
+            }
+        )
+        self.maxDiff = None
+        self.expected = {
+            'apiVersion': 'v1',
+            'kind': 'Pod',
+            'metadata': {
+                'name': 'myapp-pod',
+                'labels': {'app': 'myapp'},
+                'annotations': {}},
+            'spec': {
+                'containers': [{
+                    'name': 'base',
+                    'image': 'busybox',
+                    'command': [
+                        'sh', '-c', 'echo Hello Kubernetes!'
+                    ],
+                    'imagePullPolicy': 'IfNotPresent',
+                    'args': [],
+                    'env': [{
+                        'name': 'ENVIRONMENT',
+                        'value': 'prod'
+                    }, {
+                        'name': 'LOG_LEVEL',
+                        'value': 'warning'
+                    }, {
+                        'name': 'TARGET',
+                        'valueFrom': {
+                            'secretKeyRef': {
+                                'name': 'secret_b',
+                                'key': 'source_b'
+                            }
+                        }
+                    }],
+                    'envFrom': [{
+                        'secretRef': {
+                            'name': 'secret_a'
+                        }
+                    }, {
+                        'configMapRef': {
+                            'name': 'configmap_a'
+                        }
+                    }, {
+                        'configMapRef': {
+                            'name': 'configmap_b'
+                        }
+                    }],
+                    'volumeMounts': [{
+                        'mountPath': '/etc/foo',
+                        'name': 'secretvol0',
+                        'readOnly': True
+                    }]
+                }],
+                'restartPolicy': 'Never',
+                'nodeSelector': {},
+                'volumes': [{
+                    'name': 'secretvol0',
+                    'secret': {
+                        'secretName': 'secret_b'
+                    }
+                }],
+                'imagePullSecrets': [
+                    {'name': 'pull_secret_a'},
+                    {'name': 'pull_secret_b'}
+                ],
+                'affinity': {},
+                'securityContext': {
+                    'runAsUser': 1000,
+                    'fsGroup': 2000,
+                },
+            }
+        }
+
+    def test_secret_throws(self):
+        with self.assertRaises(AirflowConfigException):
+            Secret('volume', None, 'secret_a', 'key')
+
+    def test_simple_pod_request_factory_create(self):
+        result = self.simple_pod_request_factory.create(self.pod)
+        # sort
+        result['spec']['containers'][0]['env'].sort(key=lambda x: x['name'])
+        self.assertEqual(result, self.expected)
+
+    def test_xcom_pod_request_factory_create(self):
+        result = self.xcom_pod_request_factory.create(self.pod)
+        container_two = {
+            'name': 'airflow-xcom-sidecar',
+            'image': 'python:3.5-alpine',
+            'command': ['python', '-c', XCOM_CMD],
+            'volumeMounts': [
+                {
+                    'name': 'xcom',
+                    'mountPath': '/airflow/xcom'
+                }
+            ]
+        }
+        self.expected['spec']['containers'].append(container_two)
+        self.expected['spec']['containers'][0]['volumeMounts'].insert(0, {
+            'name': 'xcom',
+            'mountPath': '/airflow/xcom'
+        })
+        self.expected['spec']['volumes'].insert(0, {
+            'name': 'xcom', 'emptyDir': {}
+        })
+        result['spec']['containers'][0]['env'].sort(key=lambda x: x['name'])
+        self.assertEqual(result, self.expected)
diff --git a/tests/contrib/minikube/test_kubernetes_executor.py b/tests/contrib/minikube/test_kubernetes_executor.py
index fe38efb81bec6..fad358205fc93 100644
--- a/tests/contrib/minikube/test_kubernetes_executor.py
+++ b/tests/contrib/minikube/test_kubernetes_executor.py
@@ -16,6 +16,7 @@
 # under the License.
 
 
+import os
 import unittest
 from subprocess import check_call, check_output
 import requests.exceptions
@@ -25,27 +26,35 @@
 import re
 
 try:
-    check_call(["kubectl", "get", "pods"])
+    check_call(["/usr/local/bin/kubectl", "get", "pods"])
 except Exception as e:
-    raise unittest.SkipTest(
-        "Kubernetes integration tests require a minikube cluster;"
-        "Skipping tests {}".format(e)
-    )
+    if os.environ.get('KUBERNETES_VERSION'):
+        raise e
+    else:
+        raise unittest.SkipTest(
+            "Kubernetes integration tests require a minikube cluster;"
+            "Skipping tests {}".format(e)
+        )
 
 
 def get_minikube_host():
-    host_ip = check_output(['minikube', 'ip'])
-    if six.PY3:
-        host_ip = host_ip.decode('UTF-8')
+    if "MINIKUBE_IP" in os.environ:
+        host_ip = os.environ['MINIKUBE_IP']
+    else:
+        host_ip = check_output(['/usr/local/bin/minikube', 'ip'])
+        if six.PY3:
+            host_ip = host_ip.decode('UTF-8')
+
     host = '{}:30809'.format(host_ip.strip())
     return host
 
 
 class KubernetesExecutorTest(unittest.TestCase):
-    def _delete_airflow_pod(self):
+    @staticmethod
+    def _delete_airflow_pod():
         air_pod = check_output(['kubectl', 'get', 'pods']).decode()
         air_pod = air_pod.split('\n')
-        names = [re.compile('\s+').split(x)[0] for x in air_pod if 'airflow' in x]
+        names = [re.compile(r'\s+').split(x)[0] for x in air_pod if 'airflow' in x]
         if names:
             check_call(['kubectl', 'delete', 'pod', names[0]])
 
@@ -124,8 +133,13 @@ def start_dag(self, dag_id, host):
             'http://{host}/api/experimental/'
             'dags/{dag_id}/paused/false'.format(host=host, dag_id=dag_id)
         )
+        try:
+            result_json = result.json()
+        except ValueError:
+            result_json = str(result)
+
         self.assertEqual(result.status_code, 200, "Could not enable DAG: {result}"
-                         .format(result=result.json()))
+                         .format(result=result_json))
 
         # Trigger a new dagrun
         result = requests.post(
@@ -133,8 +147,13 @@ def start_dag(self, dag_id, host):
             'dags/{dag_id}/dag_runs'.format(host=host, dag_id=dag_id),
             json={}
         )
+        try:
+            result_json = result.json()
+        except ValueError:
+            result_json = str(result)
+
         self.assertEqual(result.status_code, 200, "Could not trigger a DAG-run: {result}"
-                         .format(result=result.json()))
+                         .format(result=result_json))
 
         time.sleep(1)
 
@@ -149,8 +168,9 @@ def start_dag(self, dag_id, host):
 
     def test_integration_run_dag(self):
         host = get_minikube_host()
+        dag_id = 'example_kubernetes_executor_config'
 
-        result_json = self.start_dag(dag_id='example_python_operator', host=host)
+        result_json = self.start_dag(dag_id=dag_id, host=host)
 
         self.assertGreater(len(result_json['items']), 0)
 
@@ -160,19 +180,20 @@ def test_integration_run_dag(self):
         # Wait 100 seconds for the operator to complete
         self.monitor_task(host=host,
                           execution_date=execution_date,
-                          dag_id='example_python_operator',
-                          task_id='print_the_context',
+                          dag_id=dag_id,
+                          task_id='start_task',
                           expected_final_state='success', timeout=100)
 
         self.ensure_dag_expected_state(host=host,
                                        execution_date=execution_date,
-                                       dag_id='example_python_operator',
+                                       dag_id=dag_id,
                                        expected_final_state='success', timeout=100)
 
     def test_integration_run_dag_with_scheduler_failure(self):
         host = get_minikube_host()
+        dag_id = 'example_kubernetes_executor_config'
 
-        result_json = self.start_dag(dag_id='example_python_operator', host=host)
+        result_json = self.start_dag(dag_id=dag_id, host=host)
 
         self.assertGreater(len(result_json['items']), 0)
 
@@ -186,13 +207,13 @@ def test_integration_run_dag_with_scheduler_failure(self):
         # Wait 100 seconds for the operator to complete
         self.monitor_task(host=host,
                           execution_date=execution_date,
-                          dag_id='example_python_operator',
-                          task_id='print_the_context',
+                          dag_id=dag_id,
+                          task_id='start_task',
                           expected_final_state='success', timeout=120)
 
         self.ensure_dag_expected_state(host=host,
                                        execution_date=execution_date,
-                                       dag_id='example_python_operator',
+                                       dag_id=dag_id,
                                        expected_final_state='success', timeout=100)
 
 
diff --git a/tests/contrib/minikube/test_kubernetes_pod_operator.py b/tests/contrib/minikube/test_kubernetes_pod_operator.py
index 5c799f40f98ed..9c8419286280e 100644
--- a/tests/contrib/minikube/test_kubernetes_pod_operator.py
+++ b/tests/contrib/minikube/test_kubernetes_pod_operator.py
@@ -19,26 +19,32 @@
 import os
 import shutil
 from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
+from airflow.contrib.kubernetes.secret import Secret
 from airflow import AirflowException
+from kubernetes.client.rest import ApiException
 from subprocess import check_call
-import mock
 import json
 from airflow.contrib.kubernetes.pod_launcher import PodLauncher
 from airflow.contrib.kubernetes.volume_mount import VolumeMount
 from airflow.contrib.kubernetes.volume import Volume
+from tests.compat import mock
 
 try:
-    check_call(["kubectl", "get", "pods"])
+    check_call(["/usr/local/bin/kubectl", "get", "pods"])
 except Exception as e:
-    raise unittest.SkipTest(
-        "Kubernetes integration tests require a minikube cluster;"
-        "Skipping tests {}".format(e)
-    )
+    if os.environ.get('KUBERNETES_VERSION'):
+        raise e
+    else:
+        raise unittest.SkipTest(
+            "Kubernetes integration tests require a minikube cluster;"
+            "Skipping tests {}".format(e)
+        )
 
 
 class KubernetesPodOperatorTest(unittest.TestCase):
 
-    def test_config_path_move(self):
+    @staticmethod
+    def test_config_path_move():
         new_config_path = '/tmp/kube_config'
         old_config_path = os.path.expanduser('~/.kube/config')
         shutil.copy(old_config_path, new_config_path)
@@ -79,7 +85,52 @@ def test_config_path(self, client_mock, launcher_mock):
                                        cluster_context='default',
                                        config_file=file_path)
 
-    def test_working_pod(self):
+    @mock.patch("airflow.contrib.kubernetes.pod_launcher.PodLauncher.run_pod")
+    @mock.patch("airflow.contrib.kubernetes.kube_client.get_kube_client")
+    def test_image_pull_secrets_correctly_set(self, client_mock, launcher_mock):
+        from airflow.utils.state import State
+
+        fake_pull_secrets = "fakeSecret"
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            image_pull_secrets=fake_pull_secrets,
+            in_cluster=False,
+            cluster_context='default'
+        )
+        launcher_mock.return_value = (State.SUCCESS, None)
+        k.execute(None)
+        self.assertEqual(launcher_mock.call_args[0][0].image_pull_secrets,
+                         fake_pull_secrets)
+
+    @mock.patch("airflow.contrib.kubernetes.pod_launcher.PodLauncher.run_pod")
+    @mock.patch("airflow.contrib.kubernetes.pod_launcher.PodLauncher.delete_pod")
+    @mock.patch("airflow.contrib.kubernetes.kube_client.get_kube_client")
+    def test_pod_delete_even_on_launcher_error(self, client_mock, delete_pod_mock, run_pod_mock):
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            in_cluster=False,
+            cluster_context='default',
+            is_delete_operator_pod=True
+        )
+        run_pod_mock.side_effect = AirflowException('fake failure')
+        with self.assertRaises(AirflowException):
+            k.execute(None)
+        assert delete_pod_mock.called
+
+    @staticmethod
+    def test_working_pod():
         k = KubernetesPodOperator(
             namespace='default',
             image="ubuntu:16.04",
@@ -91,7 +142,86 @@ def test_working_pod(self):
         )
         k.execute(None)
 
-    def test_logging(self):
+    @staticmethod
+    def test_delete_operator_pod():
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            is_delete_operator_pod=True
+        )
+        k.execute(None)
+
+    @staticmethod
+    def test_pod_hostnetwork():
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            hostnetwork=True
+        )
+        k.execute(None)
+
+    @staticmethod
+    def test_pod_node_selectors():
+        node_selectors = {
+            'beta.kubernetes.io/os': 'linux'
+        }
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo", "10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            node_selectors=node_selectors,
+            executor_config={'KubernetesExecutor': {'node_selectors': node_selectors}}
+        )
+        k.execute(None)
+
+    @staticmethod
+    def test_pod_affinity():
+        affinity = {
+            'nodeAffinity': {
+                'requiredDuringSchedulingIgnoredDuringExecution': {
+                    'nodeSelectorTerms': [
+                        {
+                            'matchExpressions': [
+                                {
+                                    'key': 'beta.kubernetes.io/os',
+                                    'operator': 'In',
+                                    'values': ['linux']
+                                }
+                            ]
+                        }
+                    ]
+                }
+            }
+        }
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo", "10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            affinity=affinity,
+            executor_config={'KubernetesExecutor': {'affinity': affinity}}
+        )
+        k.execute(None)
+
+    @staticmethod
+    def test_logging():
         with mock.patch.object(PodLauncher, 'log') as mock_logger:
             k = KubernetesPodOperator(
                 namespace='default',
@@ -106,7 +236,8 @@ def test_logging(self):
             k.execute(None)
             mock_logger.info.assert_any_call(b"+ echo 10\n")
 
-    def test_volume_mount(self):
+    @staticmethod
+    def test_volume_mount():
         with mock.patch.object(PodLauncher, 'log') as mock_logger:
             volume_mount = VolumeMount('test-volume',
                                        mount_path='/root/mount_file',
@@ -134,6 +265,69 @@ def test_volume_mount(self):
             k.execute(None)
             mock_logger.info.assert_any_call(b"retrieved from mount\n")
 
+    @staticmethod
+    def test_run_as_user_root():
+        security_context = {
+            'securityContext': {
+                'runAsUser': 0,
+            }
+        }
+
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo", "10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            security_context=security_context,
+            executor_config={'KubernetesExecutor': {'securityContext': security_context}}
+        )
+        k.execute(None)
+
+    @staticmethod
+    def test_run_as_user_non_root():
+        security_context = {
+            'securityContext': {
+                'runAsUser': 1000,
+            }
+        }
+
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo", "10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            security_context=security_context,
+            executor_config={'KubernetesExecutor': {'securityContext': security_context}}
+        )
+        k.execute(None)
+
+    @staticmethod
+    def test_fs_group():
+        security_context = {
+            'securityContext': {
+                'fsGroup': 1000,
+            }
+        }
+
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo", "10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            security_context=security_context,
+            executor_config={'KubernetesExecutor': {'securityContext': security_context}}
+        )
+        k.execute(None)
+
     def test_faulty_image(self):
         bad_image_name = "foobar"
         k = KubernetesPodOperator(
@@ -146,10 +340,24 @@ def test_faulty_image(self):
             task_id="task",
             startup_timeout_seconds=5
         )
-        with self.assertRaises(AirflowException) as cm:
-            k.execute(None),
+        with self.assertRaises(AirflowException):
+            k.execute(None)
 
-        print("exception: {}".format(cm))
+    def test_faulty_service_account(self):
+        bad_service_account_name = "foobar"
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            startup_timeout_seconds=5,
+            service_account_name=bad_service_account_name
+        )
+        with self.assertRaises(ApiException):
+            k.execute(None)
 
     def test_pod_failure(self):
         """
@@ -182,6 +390,50 @@ def test_xcom_push(self):
         )
         self.assertEqual(k.execute(None), json.loads(return_value))
 
+    @mock.patch("airflow.contrib.kubernetes.pod_launcher.PodLauncher.run_pod")
+    @mock.patch("airflow.contrib.kubernetes.kube_client.get_kube_client")
+    def test_envs_from_configmaps(self, client_mock, launcher_mock):
+        # GIVEN
+        from airflow.utils.state import State
+        configmaps = ['test-configmap']
+        # WHEN
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+            configmaps=configmaps
+        )
+        # THEN
+        launcher_mock.return_value = (State.SUCCESS, None)
+        k.execute(None)
+        self.assertEqual(launcher_mock.call_args[0][0].configmaps, configmaps)
+
+    @mock.patch("airflow.contrib.kubernetes.pod_launcher.PodLauncher.run_pod")
+    @mock.patch("airflow.contrib.kubernetes.kube_client.get_kube_client")
+    def test_envs_from_secrets(self, client_mock, launcher_mock):
+        # GIVEN
+        from airflow.utils.state import State
+        secrets = [Secret('env', None, "secret_name")]
+        # WHEN
+        k = KubernetesPodOperator(
+            namespace='default',
+            image="ubuntu:16.04",
+            cmds=["bash", "-cx"],
+            arguments=["echo 10"],
+            secrets=secrets,
+            labels={"foo": "bar"},
+            name="test",
+            task_id="task",
+        )
+        # THEN
+        launcher_mock.return_value = (State.SUCCESS, None)
+        k.execute(None)
+        self.assertEqual(launcher_mock.call_args[0][0].secrets, secrets)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/operators/__init__.py b/tests/contrib/operators/__init__.py
index fc532e61c6c64..b7f8352944d3f 100644
--- a/tests/contrib/operators/__init__.py
+++ b/tests/contrib/operators/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,4 +17,3 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
diff --git a/tests/contrib/operators/postgres_local_executor.cfg b/tests/contrib/operators/postgres_local_executor.cfg
new file mode 100644
index 0000000000000..1f8eb3d3c34c0
--- /dev/null
+++ b/tests/contrib/operators/postgres_local_executor.cfg
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# This configuration file is used to override
+# the executor to be LocalExecutor and switch to using Postgres as the database
+# Some system tests require LocalExecutor to be used because of parallelism required
+# One example is test_gcp_bigtable_operator_system.py in which sensor
+# Has to be run in parallell with a task to create BigTable table
+[core]
+executor = LocalExecutor
+sql_alchemy_conn = postgresql:///airflow/airflow.db
diff --git a/tests/contrib/operators/test_adls_list_operator.py b/tests/contrib/operators/test_adls_list_operator.py
new file mode 100644
index 0000000000000..f668b55a5a80f
--- /dev/null
+++ b/tests/contrib/operators/test_adls_list_operator.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow.contrib.operators.adls_list_operator import AzureDataLakeStorageListOperator
+from tests.compat import mock
+
+TASK_ID = 'test-adls-list-operator'
+TEST_PATH = 'test/*'
+MOCK_FILES = ["test/TEST1.csv", "test/TEST2.csv", "test/path/TEST3.csv",
+              "test/path/PARQUET.parquet", "test/path/PIC.png"]
+
+
+class AzureDataLakeStorageListOperatorTest(unittest.TestCase):
+
+    @mock.patch('airflow.contrib.operators.adls_list_operator.AzureDataLakeHook')
+    def test_execute(self, mock_hook):
+        mock_hook.return_value.list.return_value = MOCK_FILES
+
+        operator = AzureDataLakeStorageListOperator(task_id=TASK_ID,
+                                                    path=TEST_PATH)
+
+        files = operator.execute(None)
+        mock_hook.return_value.list.assert_called_once_with(
+            path=TEST_PATH
+        )
+        self.assertEqual(sorted(files), sorted(MOCK_FILES))
diff --git a/tests/contrib/operators/test_adls_to_gcs_operator.py b/tests/contrib/operators/test_adls_to_gcs_operator.py
new file mode 100644
index 0000000000000..d551a05971417
--- /dev/null
+++ b/tests/contrib/operators/test_adls_to_gcs_operator.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow.contrib.hooks.gcs_hook import _parse_gcs_url
+from airflow.contrib.operators.adls_to_gcs import \
+    AdlsToGoogleCloudStorageOperator
+from tests.compat import mock
+
+TASK_ID = 'test-adls-gcs-operator'
+ADLS_PATH_1 = '*'
+GCS_PATH = 'gs://test/'
+MOCK_FILES = ["test/TEST1.csv", "test/TEST2.csv", "test/path/TEST3.csv",
+              "test/path/PARQUET.parquet", "test/path/PIC.png"]
+AZURE_CONN_ID = 'azure_data_lake_default'
+GCS_CONN_ID = 'google_cloud_default'
+
+
+class AdlsToGoogleCloudStorageOperatorTest(unittest.TestCase):
+    def test_init(self):
+        """Test AdlsToGoogleCloudStorageOperator instance is properly initialized."""
+
+        operator = AdlsToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            src_adls=ADLS_PATH_1,
+            dest_gcs=GCS_PATH,
+            replace=False,
+            azure_data_lake_conn_id=AZURE_CONN_ID,
+            google_cloud_storage_conn_id=GCS_CONN_ID
+        )
+
+        self.assertEqual(operator.task_id, TASK_ID)
+        self.assertEqual(operator.src_adls, ADLS_PATH_1)
+        self.assertEqual(operator.dest_gcs, GCS_PATH)
+        self.assertEqual(operator.replace, False)
+        self.assertEqual(operator.google_cloud_storage_conn_id, GCS_CONN_ID)
+        self.assertEqual(operator.azure_data_lake_conn_id, AZURE_CONN_ID)
+
+    @mock.patch('airflow.contrib.operators.adls_to_gcs.AzureDataLakeHook')
+    @mock.patch('airflow.contrib.operators.adls_list_operator.AzureDataLakeHook')
+    @mock.patch(
+        'airflow.contrib.operators.adls_to_gcs.GoogleCloudStorageHook')
+    def test_execute(self, gcs_mock_hook, adls_one_mock_hook, adls_two_mock_hook):
+        """Test the execute function when the run is successful."""
+
+        operator = AdlsToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            src_adls=ADLS_PATH_1,
+            dest_gcs=GCS_PATH,
+            replace=False,
+            azure_data_lake_conn_id=AZURE_CONN_ID,
+            google_cloud_storage_conn_id=GCS_CONN_ID
+        )
+
+        adls_one_mock_hook.return_value.list.return_value = MOCK_FILES
+        adls_two_mock_hook.return_value.list.return_value = MOCK_FILES
+
+        def _assert_upload(bucket, object, filename):
+            gcs_bucket, gcs_object_path = _parse_gcs_url(GCS_PATH)
+
+            self.assertEqual(gcs_bucket, 'test')
+            self.assertIn(object[len(gcs_object_path):], MOCK_FILES)
+
+        gcs_mock_hook.return_value.upload.side_effect = _assert_upload
+
+        uploaded_files = operator.execute(None)
+
+        adls_one_mock_hook.assert_called_once_with(azure_data_lake_conn_id=AZURE_CONN_ID)
+        adls_two_mock_hook.assert_called_once_with(azure_data_lake_conn_id=AZURE_CONN_ID)
+        gcs_mock_hook.assert_called_once_with(
+            google_cloud_storage_conn_id=GCS_CONN_ID, delegate_to=None)
+
+        # we expect MOCK_FILES to be uploaded
+        self.assertEqual(sorted(MOCK_FILES), sorted(uploaded_files))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_aws_athena_operator.py b/tests/contrib/operators/test_aws_athena_operator.py
new file mode 100644
index 0000000000000..8c6b045d1aa3a
--- /dev/null
+++ b/tests/contrib/operators/test_aws_athena_operator.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# 'License'); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+
+from airflow.contrib.operators.aws_athena_operator import AWSAthenaOperator
+from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook
+from airflow import configuration
+from tests.compat import mock
+
+MOCK_DATA = {
+    'task_id': 'test_aws_athena_operator',
+    'query': 'SELECT * FROM TEST_TABLE',
+    'database': 'TEST_DATABASE',
+    'outputLocation': 's3://test_s3_bucket/',
+    'client_request_token': 'eac427d0-1c6d-4dfb-96aa-2835d3ac6595'
+}
+
+query_context = {
+    'Database': MOCK_DATA['database']
+}
+result_configuration = {
+    'OutputLocation': MOCK_DATA['outputLocation']
+}
+
+
+class TestAWSAthenaOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+
+        self.athena = AWSAthenaOperator(task_id='test_aws_athena_operator', query='SELECT * FROM TEST_TABLE',
+                                        database='TEST_DATABASE', output_location='s3://test_s3_bucket/',
+                                        client_request_token='eac427d0-1c6d-4dfb-96aa-2835d3ac6595',
+                                        sleep_time=1)
+
+    def test_init(self):
+        self.assertEqual(self.athena.task_id, MOCK_DATA['task_id'])
+        self.assertEqual(self.athena.query, MOCK_DATA['query'])
+        self.assertEqual(self.athena.database, MOCK_DATA['database'])
+        self.assertEqual(self.athena.aws_conn_id, 'aws_default')
+        self.assertEqual(self.athena.client_request_token, MOCK_DATA['client_request_token'])
+        self.assertEqual(self.athena.sleep_time, 1)
+
+    @mock.patch.object(AWSAthenaHook, 'check_query_status', side_effect=("SUCCESS",))
+    @mock.patch.object(AWSAthenaHook, 'run_query', return_value='1234')
+    @mock.patch.object(AWSAthenaHook, 'get_conn')
+    def test_hook_run_small_success_query(self, mock_conn, mock_run_query, mock_check_query_status):
+        self.athena.execute(None)
+        mock_run_query.assert_called_once_with(MOCK_DATA['query'], query_context, result_configuration,
+                                               MOCK_DATA['client_request_token'])
+        self.assertEqual(mock_check_query_status.call_count, 1)
+
+    @mock.patch.object(AWSAthenaHook, 'check_query_status', side_effect=("RUNNING", "RUNNING", "SUCCESS",))
+    @mock.patch.object(AWSAthenaHook, 'run_query', return_value='1234')
+    @mock.patch.object(AWSAthenaHook, 'get_conn')
+    def test_hook_run_big_success_query(self, mock_conn, mock_run_query, mock_check_query_status):
+        self.athena.execute(None)
+        mock_run_query.assert_called_once_with(MOCK_DATA['query'], query_context, result_configuration,
+                                               MOCK_DATA['client_request_token'])
+        self.assertEqual(mock_check_query_status.call_count, 3)
+
+    @mock.patch.object(AWSAthenaHook, 'check_query_status', side_effect=("RUNNING", "FAILED",))
+    @mock.patch.object(AWSAthenaHook, 'run_query', return_value='1234')
+    @mock.patch.object(AWSAthenaHook, 'get_conn')
+    def test_hook_run_failure_query(self, mock_conn, mock_run_query, mock_check_query_status):
+        self.athena.execute(None)
+        mock_run_query.assert_called_once_with(MOCK_DATA['query'], query_context, result_configuration,
+                                               MOCK_DATA['client_request_token'])
+        self.assertEqual(mock_check_query_status.call_count, 2)
+
+    @mock.patch.object(AWSAthenaHook, 'check_query_status', side_effect=("RUNNING", "RUNNING", "CANCELLED",))
+    @mock.patch.object(AWSAthenaHook, 'run_query', return_value='1234')
+    @mock.patch.object(AWSAthenaHook, 'get_conn')
+    def test_hook_run_cancelled_query(self, mock_conn, mock_run_query, mock_check_query_status):
+        self.athena.execute(None)
+        mock_run_query.assert_called_once_with(MOCK_DATA['query'], query_context, result_configuration,
+                                               MOCK_DATA['client_request_token'])
+        self.assertEqual(mock_check_query_status.call_count, 3)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_awsbatch_operator.py b/tests/contrib/operators/test_awsbatch_operator.py
index 66540bc44f267..c825545cc6a47 100644
--- a/tests/contrib/operators/test_awsbatch_operator.py
+++ b/tests/contrib/operators/test_awsbatch_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,15 +24,7 @@
 from airflow import configuration
 from airflow.exceptions import AirflowException
 from airflow.contrib.operators.awsbatch_operator import AWSBatchOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
+from tests.compat import mock
 
 RESPONSE_WITHOUT_FAILURES = {
     "jobName": "51455483-c62c-48ac-9b88-53a6a725baa3",
@@ -58,7 +50,6 @@ def setUp(self, aws_hook_mock):
             region_name='eu-west-1')
 
     def test_init(self):
-
         self.assertEqual(self.batch.job_name, '51455483-c62c-48ac-9b88-53a6a725baa3')
         self.assertEqual(self.batch.job_queue, 'queue')
         self.assertEqual(self.batch.job_definition, 'hello-world')
@@ -76,13 +67,13 @@ def test_template_fields_overrides(self):
     @mock.patch.object(AWSBatchOperator, '_wait_for_task_ended')
     @mock.patch.object(AWSBatchOperator, '_check_success_task')
     def test_execute_without_failures(self, check_mock, wait_mock):
-
         client_mock = self.aws_hook_mock.return_value.get_client_type.return_value
         client_mock.submit_job.return_value = RESPONSE_WITHOUT_FAILURES
 
         self.batch.execute(None)
 
-        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('batch', region_name='eu-west-1')
+        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('batch',
+                                                                                region_name='eu-west-1')
         client_mock.submit_job.assert_called_once_with(
             jobQueue='queue',
             jobName='51455483-c62c-48ac-9b88-53a6a725baa3',
@@ -95,14 +86,14 @@ def test_execute_without_failures(self, check_mock, wait_mock):
         self.assertEqual(self.batch.jobId, '8ba9d676-4108-4474-9dca-8bbac1da9b19')
 
     def test_execute_with_failures(self):
-
         client_mock = self.aws_hook_mock.return_value.get_client_type.return_value
         client_mock.submit_job.return_value = ""
 
         with self.assertRaises(AirflowException):
             self.batch.execute(None)
 
-        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('batch', region_name='eu-west-1')
+        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('batch',
+                                                                                region_name='eu-west-1')
         client_mock.submit_job.assert_called_once_with(
             jobQueue='queue',
             jobName='51455483-c62c-48ac-9b88-53a6a725baa3',
@@ -111,7 +102,6 @@ def test_execute_with_failures(self):
         )
 
     def test_wait_end_tasks(self):
-
         client_mock = mock.Mock()
         self.batch.jobId = '8ba9d676-4108-4474-9dca-8bbac1da9b19'
         self.batch.client = client_mock
@@ -122,7 +112,7 @@ def test_wait_end_tasks(self):
         client_mock.get_waiter.return_value.wait.assert_called_once_with(
             jobs=['8ba9d676-4108-4474-9dca-8bbac1da9b19']
         )
-        self.assertEquals(sys.maxsize, client_mock.get_waiter.return_value.config.max_attempts)
+        self.assertEqual(sys.maxsize, client_mock.get_waiter.return_value.config.max_attempts)
 
     def test_check_success_tasks_raises(self):
         client_mock = mock.Mock()
@@ -147,6 +137,7 @@ def test_check_success_tasks_raises_failed(self):
         client_mock.describe_jobs.return_value = {
             'jobs': [{
                 'status': 'FAILED',
+                'statusReason': 'This is an error reason',
                 'attempts': [{
                     'exitCode': 1
                 }]
@@ -157,7 +148,7 @@ def test_check_success_tasks_raises_failed(self):
             self.batch._check_success_task()
 
         # Ordering of str(dict) is not guaranteed.
-        self.assertIn('This containers encounter an error during execution ', str(e.exception))
+        self.assertIn('Job failed with status ', str(e.exception))
 
     def test_check_success_tasks_raises_pending(self):
         client_mock = mock.Mock()
@@ -184,6 +175,7 @@ def test_check_success_tasks_raises_mutliple(self):
         client_mock.describe_jobs.return_value = {
             'jobs': [{
                 'status': 'FAILED',
+                'statusReason': 'This is an error reason',
                 'attempts': [{
                     'exitCode': 1
                 }, {
@@ -196,7 +188,7 @@ def test_check_success_tasks_raises_mutliple(self):
             self.batch._check_success_task()
 
         # Ordering of str(dict) is not guaranteed.
-        self.assertIn('This containers encounter an error during execution ', str(e.exception))
+        self.assertIn('Job failed with status ', str(e.exception))
 
     def test_check_success_task_not_raises(self):
         client_mock = mock.Mock()
diff --git a/tests/contrib/operators/test_azure_container_instances_operator.py b/tests/contrib/operators/test_azure_container_instances_operator.py
new file mode 100644
index 0000000000000..53ed71116b805
--- /dev/null
+++ b/tests/contrib/operators/test_azure_container_instances_operator.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from airflow.exceptions import AirflowException
+from airflow.contrib.operators.azure_container_instances_operator import AzureContainerInstancesOperator
+from tests.compat import mock
+
+import unittest
+
+
+class TestACIOperator(unittest.TestCase):
+
+    @mock.patch("airflow.contrib.operators."
+                "azure_container_instances_operator.AzureContainerInstanceHook")
+    def test_execute(self, aci_mock):
+        aci_mock.return_value.get_state_exitcode_details.return_value = "Terminated", 0, "test"
+        aci_mock.return_value.exists.return_value = False
+
+        aci = AzureContainerInstancesOperator(ci_conn_id=None,
+                                              registry_conn_id=None,
+                                              resource_group='resource-group',
+                                              name='container-name',
+                                              image='container-image',
+                                              region='region',
+                                              task_id='task')
+        aci.execute(None)
+
+        self.assertEqual(aci_mock.return_value.create_or_update.call_count, 1)
+        (called_rg, called_cn, called_cg), _ = \
+            aci_mock.return_value.create_or_update.call_args
+
+        self.assertEqual(called_rg, 'resource-group')
+        self.assertEqual(called_cn, 'container-name')
+
+        self.assertEqual(called_cg.location, 'region')
+        self.assertEqual(called_cg.image_registry_credentials, None)
+        self.assertEqual(called_cg.restart_policy, 'Never')
+        self.assertEqual(called_cg.os_type, 'Linux')
+
+        called_cg_container = called_cg.containers[0]
+        self.assertEqual(called_cg_container.name, 'container-name')
+        self.assertEqual(called_cg_container.image, 'container-image')
+
+        self.assertEqual(aci_mock.return_value.delete.call_count, 1)
+
+    @mock.patch("airflow.contrib.operators."
+                "azure_container_instances_operator.AzureContainerInstanceHook")
+    def test_execute_with_failures(self, aci_mock):
+        aci_mock.return_value.get_state_exitcode_details.return_value = "Terminated", 1, "test"
+        aci_mock.return_value.exists.return_value = False
+
+        aci = AzureContainerInstancesOperator(ci_conn_id=None,
+                                              registry_conn_id=None,
+                                              resource_group='resource-group',
+                                              name='container-name',
+                                              image='container-image',
+                                              region='region',
+                                              task_id='task')
+        with self.assertRaises(AirflowException):
+            aci.execute(None)
+
+        self.assertEqual(aci_mock.return_value.delete.call_count, 1)
+
+    @mock.patch("airflow.contrib.operators."
+                "azure_container_instances_operator.AzureContainerInstanceHook")
+    def test_execute_with_messages_logs(self, aci_mock):
+        aci_mock.return_value.get_state_exitcode_details.side_effect = [("Running", 0, "test"),
+                                                                        ("Terminated", 0, "test")]
+        aci_mock.return_value.get_messages.return_value = ["test", "messages"]
+        aci_mock.return_value.get_logs.return_value = ["test", "logs"]
+        aci_mock.return_value.exists.return_value = False
+
+        aci = AzureContainerInstancesOperator(ci_conn_id=None,
+                                              registry_conn_id=None,
+                                              resource_group='resource-group',
+                                              name='container-name',
+                                              image='container-image',
+                                              region='region',
+                                              task_id='task')
+        aci.execute(None)
+
+        self.assertEqual(aci_mock.return_value.create_or_update.call_count, 1)
+        self.assertEqual(aci_mock.return_value.get_state_exitcode_details.call_count, 2)
+        self.assertEqual(aci_mock.return_value.get_messages.call_count, 2)
+        self.assertEqual(aci_mock.return_value.get_logs.call_count, 2)
+
+        self.assertEqual(aci_mock.return_value.delete.call_count, 1)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py b/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py
new file mode 100644
index 0000000000000..282114a1e3cf7
--- /dev/null
+++ b/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+import json
+import unittest
+import uuid
+
+from airflow.contrib.operators.azure_cosmos_operator import AzureCosmosInsertDocumentOperator
+
+from airflow import configuration
+from airflow.models.connection import Connection
+from airflow.utils import db
+from tests.compat import mock
+
+
+class TestAzureCosmosDbHook(unittest.TestCase):
+
+    # Set up an environment to test with
+    def setUp(self):
+        # set up some test variables
+        self.test_end_point = 'https://test_endpoint:443'
+        self.test_master_key = 'magic_test_key'
+        self.test_database_name = 'test_database_name'
+        self.test_collection_name = 'test_collection_name'
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='azure_cosmos_test_key_id',
+                conn_type='azure_cosmos',
+                login=self.test_end_point,
+                password=self.test_master_key,
+                extra=json.dumps({'database_name': self.test_database_name,
+                                  'collection_name': self.test_collection_name})
+            )
+        )
+
+    @mock.patch('azure.cosmos.cosmos_client.CosmosClient')
+    def test_insert_document(self, cosmos_mock):
+        test_id = str(uuid.uuid4())
+        cosmos_mock.return_value.CreateItem.return_value = {'id': test_id}
+        self.cosmos = AzureCosmosInsertDocumentOperator(
+            database_name=self.test_database_name,
+            collection_name=self.test_collection_name,
+            document={'id': test_id, 'data': 'sometestdata'},
+            azure_cosmos_conn_id='azure_cosmos_test_key_id',
+            task_id='azure_cosmos_sensor')
+
+        expected_calls = [mock.call().CreateItem(
+            'dbs/' + self.test_database_name + '/colls/' + self.test_collection_name,
+            {'data': 'sometestdata', 'id': test_id})]
+
+        self.cosmos.execute(None)
+        cosmos_mock.assert_any_call(self.test_end_point, {'masterKey': self.test_master_key})
+        cosmos_mock.assert_has_calls(expected_calls)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_bigquery_operator.py b/tests/contrib/operators/test_bigquery_operator.py
index 6a51d0c46d294..11dab67ec1111 100644
--- a/tests/contrib/operators/test_bigquery_operator.py
+++ b/tests/contrib/operators/test_bigquery_operator.py
@@ -19,39 +19,30 @@
 
 import unittest
 import warnings
+from datetime import datetime
+
+import six
+
+from airflow import configuration, models
+from airflow.models import TaskInstance, DAG
+from airflow.models.taskfail import TaskFail
 
 from airflow.contrib.operators.bigquery_operator import \
-    BigQueryCreateExternalTableOperator, \
-    BigQueryOperator, \
-    BigQueryCreateEmptyTableOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+    BigQueryCreateExternalTableOperator, BigQueryCreateEmptyTableOperator, \
+    BigQueryDeleteDatasetOperator, BigQueryCreateEmptyDatasetOperator, \
+    BigQueryOperator
+from airflow.settings import Session
+from tests.compat import mock
 
 TASK_ID = 'test-bq-create-table-operator'
 TEST_DATASET = 'test-dataset'
-TEST_PROJECT_ID = 'test-project'
+TEST_GCP_PROJECT_ID = 'test-project'
 TEST_TABLE_ID = 'test-table-id'
 TEST_GCS_BUCKET = 'test-bucket'
 TEST_GCS_DATA = ['dir1/*.csv']
 TEST_SOURCE_FORMAT = 'CSV'
-
-
-class BigQueryOperatorTest(unittest.TestCase):
-    def test_bql_deprecation_warning(self):
-        with warnings.catch_warnings(record=True) as w:
-            BigQueryOperator(
-                task_id='test_deprecation_warning_for_bql',
-                bql='select * from test_table'
-            )
-        self.assertIn(
-            'Deprecated parameter `bql`',
-            w[0].message.args[0])
+DEFAULT_DATE = datetime(2015, 1, 1)
+TEST_DAG_ID = 'test-bigquery-operators'
 
 
 class BigQueryCreateEmptyTableOperatorTest(unittest.TestCase):
@@ -60,7 +51,7 @@ class BigQueryCreateEmptyTableOperatorTest(unittest.TestCase):
     def test_execute(self, mock_hook):
         operator = BigQueryCreateEmptyTableOperator(task_id=TASK_ID,
                                                     dataset_id=TEST_DATASET,
-                                                    project_id=TEST_PROJECT_ID,
+                                                    project_id=TEST_GCP_PROJECT_ID,
                                                     table_id=TEST_TABLE_ID)
 
         operator.execute(None)
@@ -70,10 +61,11 @@ def test_execute(self, mock_hook):
             .create_empty_table \
             .assert_called_once_with(
                 dataset_id=TEST_DATASET,
-                project_id=TEST_PROJECT_ID,
+                project_id=TEST_GCP_PROJECT_ID,
                 table_id=TEST_TABLE_ID,
                 schema_fields=None,
-                time_partitioning={}
+                time_partitioning={},
+                labels=None
             )
 
 
@@ -112,5 +104,161 @@ def test_execute(self, mock_hook):
                 quote_character=None,
                 allow_quoted_newlines=False,
                 allow_jagged_rows=False,
-                src_fmt_configs={}
+                src_fmt_configs={},
+                labels=None
             )
+
+
+class BigQueryDeleteDatasetOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.bigquery_operator.BigQueryHook')
+    def test_execute(self, mock_hook):
+        operator = BigQueryDeleteDatasetOperator(
+            task_id=TASK_ID,
+            dataset_id=TEST_DATASET,
+            project_id=TEST_GCP_PROJECT_ID
+        )
+
+        operator.execute(None)
+        mock_hook.return_value \
+            .get_conn() \
+            .cursor() \
+            .delete_dataset \
+            .assert_called_once_with(
+                dataset_id=TEST_DATASET,
+                project_id=TEST_GCP_PROJECT_ID
+            )
+
+
+class BigQueryCreateEmptyDatasetOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.bigquery_operator.BigQueryHook')
+    def test_execute(self, mock_hook):
+        operator = BigQueryCreateEmptyDatasetOperator(
+            task_id=TASK_ID,
+            dataset_id=TEST_DATASET,
+            project_id=TEST_GCP_PROJECT_ID
+        )
+
+        operator.execute(None)
+        mock_hook.return_value \
+            .get_conn() \
+            .cursor() \
+            .create_empty_dataset \
+            .assert_called_once_with(
+                dataset_id=TEST_DATASET,
+                project_id=TEST_GCP_PROJECT_ID,
+                dataset_reference={}
+            )
+
+
+class BigQueryOperatorTest(unittest.TestCase):
+    def test_bql_deprecation_warning(self):
+        with warnings.catch_warnings(record=True) as w:
+            BigQueryOperator(
+                task_id='test_deprecation_warning_for_bql',
+                bql='select * from test_table'
+            )
+        self.assertIn(
+            'Deprecated parameter `bql`',
+            w[0].message.args[0])
+
+    def setUp(self):
+        configuration.conf.load_test_config()
+        self.dagbag = models.DagBag(
+            dag_folder='/dev/null', include_examples=True)
+        self.args = {'owner': 'airflow', 'start_date': DEFAULT_DATE}
+        self.dag = DAG(TEST_DAG_ID, default_args=self.args)
+
+    def tearDown(self):
+        session = Session()
+        session.query(models.TaskInstance).filter_by(
+            dag_id=TEST_DAG_ID).delete()
+        session.query(TaskFail).filter_by(
+            dag_id=TEST_DAG_ID).delete()
+        session.commit()
+        session.close()
+
+    @mock.patch('airflow.contrib.operators.bigquery_operator.BigQueryHook')
+    def test_execute(self, mock_hook):
+        operator = BigQueryOperator(
+            task_id=TASK_ID,
+            sql='Select * from test_table',
+            destination_dataset_table=None,
+            write_disposition='WRITE_EMPTY',
+            allow_large_results=False,
+            flatten_results=None,
+            bigquery_conn_id='bigquery_default',
+            udf_config=None,
+            use_legacy_sql=True,
+            maximum_billing_tier=None,
+            maximum_bytes_billed=None,
+            create_disposition='CREATE_IF_NEEDED',
+            schema_update_options=(),
+            query_params=None,
+            labels=None,
+            priority='INTERACTIVE',
+            time_partitioning=None,
+            api_resource_configs=None,
+            cluster_fields=None,
+        )
+
+        operator.execute(None)
+        mock_hook.return_value \
+            .get_conn() \
+            .cursor() \
+            .run_query \
+            .assert_called_once_with(
+                sql='Select * from test_table',
+                destination_dataset_table=None,
+                write_disposition='WRITE_EMPTY',
+                allow_large_results=False,
+                flatten_results=None,
+                udf_config=None,
+                maximum_billing_tier=None,
+                maximum_bytes_billed=None,
+                create_disposition='CREATE_IF_NEEDED',
+                schema_update_options=(),
+                query_params=None,
+                labels=None,
+                priority='INTERACTIVE',
+                time_partitioning=None,
+                api_resource_configs=None,
+                cluster_fields=None,
+            )
+
+    @mock.patch('airflow.contrib.operators.bigquery_operator.BigQueryHook')
+    def test_bigquery_operator_defaults(self, mock_hook):
+
+        operator = BigQueryOperator(
+            task_id=TASK_ID,
+            sql='Select * from test_table',
+            dag=self.dag, default_args=self.args
+        )
+
+        operator.execute(None)
+        mock_hook.return_value \
+            .get_conn() \
+            .cursor() \
+            .run_query \
+            .assert_called_once_with(
+                sql='Select * from test_table',
+                destination_dataset_table=None,
+                write_disposition='WRITE_EMPTY',
+                allow_large_results=False,
+                flatten_results=None,
+                udf_config=None,
+                maximum_billing_tier=None,
+                maximum_bytes_billed=None,
+                create_disposition='CREATE_IF_NEEDED',
+                schema_update_options=(),
+                query_params=None,
+                labels=None,
+                priority='INTERACTIVE',
+                time_partitioning=None,
+                api_resource_configs=None,
+                cluster_fields=None,
+            )
+
+        self.assertTrue(isinstance(operator.sql, six.string_types))
+        ti = TaskInstance(task=operator, execution_date=DEFAULT_DATE)
+        ti.render_templates()
+        self.assertTrue(isinstance(ti.task.sql, six.string_types))
diff --git a/tests/contrib/operators/test_cassandra_to_gcs_operator.py b/tests/contrib/operators/test_cassandra_to_gcs_operator.py
index 42f8195f51f0c..7807ca816472a 100644
--- a/tests/contrib/operators/test_cassandra_to_gcs_operator.py
+++ b/tests/contrib/operators/test_cassandra_to_gcs_operator.py
@@ -44,49 +44,49 @@ def test_execute(self, upload, get_conn):
 
     def test_convert_value(self):
         op = CassandraToGoogleCloudStorageOperator
-        self.assertEquals(op.convert_value('None', None), None)
-        self.assertEquals(op.convert_value('int', 1), 1)
-        self.assertEquals(op.convert_value('float', 1.0), 1.0)
-        self.assertEquals(op.convert_value('str', "text"), "text")
-        self.assertEquals(op.convert_value('bool', True), True)
-        self.assertEquals(op.convert_value('dict', {"a": "b"}), {"a": "b"})
+        self.assertEqual(op.convert_value('None', None), None)
+        self.assertEqual(op.convert_value('int', 1), 1)
+        self.assertEqual(op.convert_value('float', 1.0), 1.0)
+        self.assertEqual(op.convert_value('str', "text"), "text")
+        self.assertEqual(op.convert_value('bool', True), True)
+        self.assertEqual(op.convert_value('dict', {"a": "b"}), {"a": "b"})
 
         from datetime import datetime
         now = datetime.now()
-        self.assertEquals(op.convert_value('datetime', now), str(now))
+        self.assertEqual(op.convert_value('datetime', now), str(now))
 
         from cassandra.util import Date
         date_str = '2018-01-01'
         date = Date(date_str)
-        self.assertEquals(op.convert_value('date', date), str(date_str))
+        self.assertEqual(op.convert_value('date', date), str(date_str))
 
         import uuid
         from base64 import b64encode
         test_uuid = uuid.uuid4()
         encoded_uuid = b64encode(test_uuid.bytes).decode('ascii')
-        self.assertEquals(op.convert_value('uuid', test_uuid), encoded_uuid)
+        self.assertEqual(op.convert_value('uuid', test_uuid), encoded_uuid)
 
         b = b'abc'
         encoded_b = b64encode(b).decode('ascii')
-        self.assertEquals(op.convert_value('binary', b), encoded_b)
+        self.assertEqual(op.convert_value('binary', b), encoded_b)
 
         from decimal import Decimal
         d = Decimal(1.0)
-        self.assertEquals(op.convert_value('decimal', d), float(d))
+        self.assertEqual(op.convert_value('decimal', d), float(d))
 
         from cassandra.util import Time
         time = Time(0)
-        self.assertEquals(op.convert_value('time', time), '00:00:00')
+        self.assertEqual(op.convert_value('time', time), '00:00:00')
 
         date_str_lst = ['2018-01-01', '2018-01-02', '2018-01-03']
         date_lst = [Date(d) for d in date_str_lst]
-        self.assertEquals(op.convert_value('list', date_lst), date_str_lst)
+        self.assertEqual(op.convert_value('list', date_lst), date_str_lst)
 
         date_tpl = tuple(date_lst)
-        self.assertEquals(op.convert_value('tuple', date_tpl),
-                          {'field_0': '2018-01-01',
-                           'field_1': '2018-01-02',
-                           'field_2': '2018-01-03', })
+        self.assertEqual(op.convert_value('tuple', date_tpl),
+                         {'field_0': '2018-01-01',
+                          'field_1': '2018-01-02',
+                          'field_2': '2018-01-03', })
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/operators/test_databricks_operator.py b/tests/contrib/operators/test_databricks_operator.py
index f77da2ec18eda..57b0829e2e215 100644
--- a/tests/contrib/operators/test_databricks_operator.py
+++ b/tests/contrib/operators/test_databricks_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,22 +18,16 @@
 # under the License.
 #
 
-import jinja2
 import unittest
 from datetime import datetime
 
 from airflow.contrib.hooks.databricks_hook import RunState
+import airflow.contrib.operators.databricks_operator as databricks_operator
 from airflow.contrib.operators.databricks_operator import DatabricksSubmitRunOperator
+from airflow.contrib.operators.databricks_operator import DatabricksRunNowOperator
 from airflow.exceptions import AirflowException
 from airflow.models import DAG
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 DATE = '2017-04-20'
 TASK_ID = 'databricks-operator'
@@ -58,6 +52,40 @@
 EXISTING_CLUSTER_ID = 'existing-cluster-id'
 RUN_NAME = 'run-name'
 RUN_ID = 1
+JOB_ID = 42
+NOTEBOOK_PARAMS = {
+    "dry-run": "true",
+    "oldest-time-to-consider": "1457570074236"
+}
+JAR_PARAMS = ["param1", "param2"]
+RENDERED_TEMPLATED_JAR_PARAMS = [
+    '/test-{0}'.format(DATE)
+]
+TEMPLATED_JAR_PARAMS = [
+    '/test-{{ ds }}'
+]
+PYTHON_PARAMS = ["john doe", "35"]
+SPARK_SUBMIT_PARAMS = ["--class", "org.apache.spark.examples.SparkPi"]
+
+
+class DatabricksOperatorSharedFunctions(unittest.TestCase):
+    def test_deep_string_coerce(self):
+        test_json = {
+            'test_int': 1,
+            'test_float': 1.0,
+            'test_dict': {'key': 'value'},
+            'test_list': [1, 1.0, 'a', 'b'],
+            'test_tuple': (1, 1.0, 'a', 'b')
+        }
+
+        expected = {
+            'test_int': '1',
+            'test_float': '1.0',
+            'test_dict': {'key': 'value'},
+            'test_list': ['1', '1.0', 'a', 'b'],
+            'test_tuple': ['1', '1.0', 'a', 'b']
+        }
+        self.assertDictEqual(databricks_operator._deep_string_coerce(test_json), expected)
 
 
 class DatabricksSubmitRunOperatorTest(unittest.TestCase):
@@ -65,12 +93,15 @@ def test_init_with_named_parameters(self):
         """
         Test the initializer with the named parameters.
         """
-        op = DatabricksSubmitRunOperator(task_id=TASK_ID, new_cluster=NEW_CLUSTER, notebook_task=NOTEBOOK_TASK)
-        expected = op._deep_string_coerce({
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': TASK_ID
+        op = DatabricksSubmitRunOperator(task_id=TASK_ID,
+                                         new_cluster=NEW_CLUSTER,
+                                         notebook_task=NOTEBOOK_TASK)
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': TASK_ID
         })
+
         self.assertDictEqual(expected, op.json)
 
     def test_init_with_json(self):
@@ -78,14 +109,14 @@ def test_init_with_json(self):
         Test the initializer with json data.
         """
         json = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK
         }
         op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=json)
-        expected = op._deep_string_coerce({
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': TASK_ID
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': TASK_ID
         })
         self.assertDictEqual(expected, op.json)
 
@@ -94,15 +125,15 @@ def test_init_with_specified_run_name(self):
         Test the initializer with a specified run_name.
         """
         json = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': RUN_NAME
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': RUN_NAME
         }
         op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=json)
-        expected = op._deep_string_coerce({
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': RUN_NAME
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': RUN_NAME
         })
         self.assertDictEqual(expected, op.json)
 
@@ -114,29 +145,31 @@ def test_init_with_merging(self):
         """
         override_new_cluster = {'workers': 999}
         json = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-        }
-        op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=json, new_cluster=override_new_cluster)
-        expected = op._deep_string_coerce({
-          'new_cluster': override_new_cluster,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': TASK_ID,
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+        }
+        op = DatabricksSubmitRunOperator(task_id=TASK_ID,
+                                         json=json,
+                                         new_cluster=override_new_cluster)
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': override_new_cluster,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': TASK_ID,
         })
         self.assertDictEqual(expected, op.json)
 
     def test_init_with_templating(self):
         json = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': TEMPLATED_NOTEBOOK_TASK,
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': TEMPLATED_NOTEBOOK_TASK,
         }
         dag = DAG('test', start_date=datetime.now())
         op = DatabricksSubmitRunOperator(dag=dag, task_id=TASK_ID, json=json)
         op.json = op.render_template('json', op.json, {'ds': DATE})
-        expected = op._deep_string_coerce({
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': RENDERED_TEMPLATED_NOTEBOOK_TASK,
-          'run_name': TASK_ID,
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': RENDERED_TEMPLATED_NOTEBOOK_TASK,
+            'run_name': TASK_ID,
         })
         self.assertDictEqual(expected, op.json)
 
@@ -145,28 +178,10 @@ def test_init_with_bad_type(self):
             'test': datetime.now()
         }
         # Looks a bit weird since we have to escape regex reserved symbols.
-        exception_message = 'Type \<(type|class) \'datetime.datetime\'\> used ' + \
-                        'for parameter json\[test\] is not a number or a string'
+        exception_message = r'Type \<(type|class) \'datetime.datetime\'\> used ' + \
+                            r'for parameter json\[test\] is not a number or a string'
         with self.assertRaisesRegexp(AirflowException, exception_message):
-            op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=json)
-
-    def test_deep_string_coerce(self):
-        op = DatabricksSubmitRunOperator(task_id='test')
-        test_json = {
-            'test_int': 1,
-            'test_float': 1.0,
-            'test_dict': {'key': 'value'},
-            'test_list': [1, 1.0, 'a', 'b'],
-            'test_tuple': (1, 1.0, 'a', 'b')
-        }
-        expected = {
-            'test_int': '1',
-            'test_float': '1.0',
-            'test_dict': {'key': 'value'},
-            'test_list': ['1', '1.0', 'a', 'b'],
-            'test_tuple': ['1', '1.0', 'a', 'b']
-        }
-        self.assertDictEqual(op._deep_string_coerce(test_json), expected)
+            DatabricksSubmitRunOperator(task_id=TASK_ID, json=json)
 
     @mock.patch('airflow.contrib.operators.databricks_operator.DatabricksHook')
     def test_exec_success(self, db_mock_class):
@@ -174,8 +189,8 @@ def test_exec_success(self, db_mock_class):
         Test the execute function in case where the run is successful.
         """
         run = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
         }
         op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=run)
         db_mock = db_mock_class.return_value
@@ -184,18 +199,20 @@ def test_exec_success(self, db_mock_class):
 
         op.execute(None)
 
-        expected = op._deep_string_coerce({
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': TASK_ID
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': TASK_ID
         })
         db_mock_class.assert_called_once_with(
-                DEFAULT_CONN_ID,
-                retry_limit=op.databricks_retry_limit)
+            DEFAULT_CONN_ID,
+            retry_limit=op.databricks_retry_limit,
+            retry_delay=op.databricks_retry_delay)
+
         db_mock.submit_run.assert_called_once_with(expected)
         db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
         db_mock.get_run_state.assert_called_once_with(RUN_ID)
-        self.assertEquals(RUN_ID, op.run_id)
+        self.assertEqual(RUN_ID, op.run_id)
 
     @mock.patch('airflow.contrib.operators.databricks_operator.DatabricksHook')
     def test_exec_failure(self, db_mock_class):
@@ -203,8 +220,8 @@ def test_exec_failure(self, db_mock_class):
         Test the execute function in case where the run failed.
         """
         run = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
         }
         op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=run)
         db_mock = db_mock_class.return_value
@@ -214,24 +231,25 @@ def test_exec_failure(self, db_mock_class):
         with self.assertRaises(AirflowException):
             op.execute(None)
 
-        expected = op._deep_string_coerce({
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
-          'run_name': TASK_ID,
+        expected = databricks_operator._deep_string_coerce({
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
+            'run_name': TASK_ID,
         })
         db_mock_class.assert_called_once_with(
-                DEFAULT_CONN_ID,
-                retry_limit=op.databricks_retry_limit)
+            DEFAULT_CONN_ID,
+            retry_limit=op.databricks_retry_limit,
+            retry_delay=op.databricks_retry_delay)
         db_mock.submit_run.assert_called_once_with(expected)
         db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
         db_mock.get_run_state.assert_called_once_with(RUN_ID)
-        self.assertEquals(RUN_ID, op.run_id)
+        self.assertEqual(RUN_ID, op.run_id)
 
     @mock.patch('airflow.contrib.operators.databricks_operator.DatabricksHook')
     def test_on_kill(self, db_mock_class):
         run = {
-          'new_cluster': NEW_CLUSTER,
-          'notebook_task': NOTEBOOK_TASK,
+            'new_cluster': NEW_CLUSTER,
+            'notebook_task': NOTEBOOK_TASK,
         }
         op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=run)
         db_mock = db_mock_class.return_value
@@ -241,3 +259,173 @@ def test_on_kill(self, db_mock_class):
 
         db_mock.cancel_run.assert_called_once_with(RUN_ID)
 
+
+class DatabricksRunNowOperatorTest(unittest.TestCase):
+
+    def test_init_with_named_parameters(self):
+        """
+        Test the initializer with the named parameters.
+        """
+        op = DatabricksRunNowOperator(job_id=JOB_ID, task_id=TASK_ID)
+        expected = databricks_operator._deep_string_coerce({
+            'job_id': 42
+        })
+
+        self.assertDictEqual(expected, op.json)
+
+    def test_init_with_json(self):
+        """
+        Test the initializer with json data.
+        """
+        json = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'jar_params': JAR_PARAMS,
+            'python_params': PYTHON_PARAMS,
+            'spark_submit_params': SPARK_SUBMIT_PARAMS
+        }
+        op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID, json=json)
+
+        expected = databricks_operator._deep_string_coerce({
+            'notebook_params': NOTEBOOK_PARAMS,
+            'jar_params': JAR_PARAMS,
+            'python_params': PYTHON_PARAMS,
+            'spark_submit_params': SPARK_SUBMIT_PARAMS,
+            'job_id': JOB_ID
+        })
+
+        self.assertDictEqual(expected, op.json)
+
+    def test_init_with_merging(self):
+        """
+        Test the initializer when json and other named parameters are both
+        provided. The named parameters should override top level keys in the
+        json dict.
+        """
+        override_notebook_params = {'workers': 999}
+        json = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'jar_params': JAR_PARAMS
+        }
+
+        op = DatabricksRunNowOperator(task_id=TASK_ID,
+                                      json=json,
+                                      job_id=JOB_ID,
+                                      notebook_params=override_notebook_params,
+                                      python_params=PYTHON_PARAMS,
+                                      spark_submit_params=SPARK_SUBMIT_PARAMS)
+
+        expected = databricks_operator._deep_string_coerce({
+            'notebook_params': override_notebook_params,
+            'jar_params': JAR_PARAMS,
+            'python_params': PYTHON_PARAMS,
+            'spark_submit_params': SPARK_SUBMIT_PARAMS,
+            'job_id': JOB_ID
+        })
+
+        self.assertDictEqual(expected, op.json)
+
+    def test_init_with_templating(self):
+        json = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'jar_params': TEMPLATED_JAR_PARAMS
+        }
+
+        dag = DAG('test', start_date=datetime.now())
+        op = DatabricksRunNowOperator(dag=dag, task_id=TASK_ID, job_id=JOB_ID, json=json)
+        op.json = op.render_template('json', op.json, {'ds': DATE})
+        expected = databricks_operator._deep_string_coerce({
+            'notebook_params': NOTEBOOK_PARAMS,
+            'jar_params': RENDERED_TEMPLATED_JAR_PARAMS,
+            'job_id': JOB_ID
+        })
+        self.assertDictEqual(expected, op.json)
+
+    def test_init_with_bad_type(self):
+        json = {
+            'test': datetime.now()
+        }
+        # Looks a bit weird since we have to escape regex reserved symbols.
+        exception_message = r'Type \<(type|class) \'datetime.datetime\'\> used ' + \
+                            r'for parameter json\[test\] is not a number or a string'
+        with self.assertRaisesRegexp(AirflowException, exception_message):
+            DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID, json=json)
+
+    @mock.patch('airflow.contrib.operators.databricks_operator.DatabricksHook')
+    def test_exec_success(self, db_mock_class):
+        """
+        Test the execute function in case where the run is successful.
+        """
+        run = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'notebook_task': NOTEBOOK_TASK,
+            'jar_params': JAR_PARAMS
+        }
+        op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID, json=run)
+        db_mock = db_mock_class.return_value
+        db_mock.run_now.return_value = 1
+        db_mock.get_run_state.return_value = RunState('TERMINATED', 'SUCCESS', '')
+
+        op.execute(None)
+
+        expected = databricks_operator._deep_string_coerce({
+            'notebook_params': NOTEBOOK_PARAMS,
+            'notebook_task': NOTEBOOK_TASK,
+            'jar_params': JAR_PARAMS,
+            'job_id': JOB_ID
+        })
+
+        db_mock_class.assert_called_once_with(
+            DEFAULT_CONN_ID,
+            retry_limit=op.databricks_retry_limit,
+            retry_delay=op.databricks_retry_delay)
+        db_mock.run_now.assert_called_once_with(expected)
+        db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
+        db_mock.get_run_state.assert_called_once_with(RUN_ID)
+        self.assertEqual(RUN_ID, op.run_id)
+
+    @mock.patch('airflow.contrib.operators.databricks_operator.DatabricksHook')
+    def test_exec_failure(self, db_mock_class):
+        """
+        Test the execute function in case where the run failed.
+        """
+        run = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'notebook_task': NOTEBOOK_TASK,
+            'jar_params': JAR_PARAMS
+        }
+        op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID, json=run)
+        db_mock = db_mock_class.return_value
+        db_mock.run_now.return_value = 1
+        db_mock.get_run_state.return_value = RunState('TERMINATED', 'FAILED', '')
+
+        with self.assertRaises(AirflowException):
+            op.execute(None)
+
+        expected = databricks_operator._deep_string_coerce({
+            'notebook_params': NOTEBOOK_PARAMS,
+            'notebook_task': NOTEBOOK_TASK,
+            'jar_params': JAR_PARAMS,
+            'job_id': JOB_ID
+        })
+        db_mock_class.assert_called_once_with(
+            DEFAULT_CONN_ID,
+            retry_limit=op.databricks_retry_limit,
+            retry_delay=op.databricks_retry_delay)
+        db_mock.run_now.assert_called_once_with(expected)
+        db_mock.get_run_page_url.assert_called_once_with(RUN_ID)
+        db_mock.get_run_state.assert_called_once_with(RUN_ID)
+        self.assertEqual(RUN_ID, op.run_id)
+
+    @mock.patch('airflow.contrib.operators.databricks_operator.DatabricksHook')
+    def test_on_kill(self, db_mock_class):
+        run = {
+            'notebook_params': NOTEBOOK_PARAMS,
+            'notebook_task': NOTEBOOK_TASK,
+            'jar_params': JAR_PARAMS
+        }
+        op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID, json=run)
+        db_mock = db_mock_class.return_value
+        op.run_id = RUN_ID
+
+        op.on_kill()
+        db_mock.cancel_run.assert_called_once_with(RUN_ID)
diff --git a/tests/contrib/operators/test_dataflow_operator.py b/tests/contrib/operators/test_dataflow_operator.py
index 4ea5f65698f03..b56db3de51bff 100644
--- a/tests/contrib/operators/test_dataflow_operator.py
+++ b/tests/contrib/operators/test_dataflow_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,21 +20,16 @@
 
 import unittest
 
-from airflow.contrib.operators.dataflow_operator import DataFlowPythonOperator, \
-    DataFlowJavaOperator, DataflowTemplateOperator
-from airflow.contrib.operators.dataflow_operator import DataFlowPythonOperator
-from airflow.version import version
+from airflow.contrib.operators.dataflow_operator import \
+    DataFlowPythonOperator, DataFlowJavaOperator, \
+    DataflowTemplateOperator, GoogleCloudBucketHelper
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from airflow.version import version
+from tests.compat import mock
 
 
 TASK_ID = 'test-dataflow-operator'
+JOB_NAME = 'test-dataflow-pipeline'
 TEMPLATE = 'gs://dataflow-templates/wordcount/template_file'
 PARAMETERS = {
     'inputFile': 'gs://dataflow-samples/shakespeare/kinglear.txt',
@@ -73,6 +68,7 @@ def setUp(self):
         self.dataflow = DataFlowPythonOperator(
             task_id=TASK_ID,
             py_file=PY_FILE,
+            job_name=JOB_NAME,
             py_options=PY_OPTIONS,
             dataflow_default_options=DEFAULT_OPTIONS_PYTHON,
             options=ADDITIONAL_OPTIONS,
@@ -81,6 +77,7 @@ def setUp(self):
     def test_init(self):
         """Test DataFlowPythonOperator instance is properly initialized."""
         self.assertEqual(self.dataflow.task_id, TASK_ID)
+        self.assertEqual(self.dataflow.job_name, JOB_NAME)
         self.assertEqual(self.dataflow.py_file, PY_FILE)
         self.assertEqual(self.dataflow.py_options, PY_OPTIONS)
         self.assertEqual(self.dataflow.poll_sleep, POLL_SLEEP)
@@ -107,8 +104,8 @@ def test_exec(self, gcs_hook, dataflow_mock):
             'labels': {'foo': 'bar', 'airflow-version': TEST_VERSION}
         }
         gcs_download_hook.assert_called_once_with(PY_FILE)
-        start_python_hook.assert_called_once_with(TASK_ID, expected_options,
-                                                  mock.ANY, PY_OPTIONS)
+        start_python_hook.assert_called_once_with(JOB_NAME, expected_options, mock.ANY,
+                                                  PY_OPTIONS)
         self.assertTrue(self.dataflow.py_file.startswith('/tmp/dataflow'))
 
 
@@ -118,6 +115,7 @@ def setUp(self):
         self.dataflow = DataFlowJavaOperator(
             task_id=TASK_ID,
             jar=JAR_FILE,
+            job_name=JOB_NAME,
             job_class=JOB_CLASS,
             dataflow_default_options=DEFAULT_OPTIONS_JAVA,
             options=ADDITIONAL_OPTIONS,
@@ -126,6 +124,7 @@ def setUp(self):
     def test_init(self):
         """Test DataflowTemplateOperator instance is properly initialized."""
         self.assertEqual(self.dataflow.task_id, TASK_ID)
+        self.assertEqual(self.dataflow.job_name, JOB_NAME)
         self.assertEqual(self.dataflow.poll_sleep, POLL_SLEEP)
         self.assertEqual(self.dataflow.dataflow_default_options,
                          DEFAULT_OPTIONS_JAVA)
@@ -146,7 +145,7 @@ def test_exec(self, gcs_hook, dataflow_mock):
         self.dataflow.execute(None)
         self.assertTrue(dataflow_mock.called)
         gcs_download_hook.assert_called_once_with(JAR_FILE)
-        start_java_hook.assert_called_once_with(TASK_ID, mock.ANY,
+        start_java_hook.assert_called_once_with(JOB_NAME, mock.ANY,
                                                 mock.ANY, JOB_CLASS)
 
 
@@ -156,6 +155,7 @@ def setUp(self):
         self.dataflow = DataflowTemplateOperator(
             task_id=TASK_ID,
             template=TEMPLATE,
+            job_name=JOB_NAME,
             parameters=PARAMETERS,
             dataflow_default_options=DEFAULT_OPTIONS_TEMPLATE,
             poll_sleep=POLL_SLEEP)
@@ -163,6 +163,7 @@ def setUp(self):
     def test_init(self):
         """Test DataflowTemplateOperator instance is properly initialized."""
         self.assertEqual(self.dataflow.task_id, TASK_ID)
+        self.assertEqual(self.dataflow.job_name, JOB_NAME)
         self.assertEqual(self.dataflow.template, TEMPLATE)
         self.assertEqual(self.dataflow.parameters, PARAMETERS)
         self.assertEqual(self.dataflow.poll_sleep, POLL_SLEEP)
@@ -184,5 +185,75 @@ def test_exec(self, dataflow_mock):
             'tempLocation': 'gs://test/temp',
             'zone': 'us-central1-f'
         }
-        start_template_hook.assert_called_once_with(TASK_ID, expected_options,
+        start_template_hook.assert_called_once_with(JOB_NAME, expected_options,
                                                     PARAMETERS, TEMPLATE)
+
+
+class GoogleCloudBucketHelperTest(unittest.TestCase):
+
+    @mock.patch(
+        'airflow.contrib.operators.dataflow_operator.GoogleCloudBucketHelper.__init__'
+    )
+    def test_invalid_object_path(self, mock_parent_init):
+
+        # This is just the path of a bucket hence invalid filename
+        file_name = 'gs://test-bucket'
+        mock_parent_init.return_value = None
+
+        gcs_bucket_helper = GoogleCloudBucketHelper()
+        gcs_bucket_helper._gcs_hook = mock.Mock()
+
+        with self.assertRaises(Exception) as context:
+            gcs_bucket_helper.google_cloud_to_local(file_name)
+
+        self.assertEqual(
+            'Invalid Google Cloud Storage (GCS) object path: {}'.format(file_name),
+            str(context.exception))
+
+    @mock.patch(
+        'airflow.contrib.operators.dataflow_operator.GoogleCloudBucketHelper.__init__'
+    )
+    def test_valid_object(self, mock_parent_init):
+
+        file_name = 'gs://test-bucket/path/to/obj.jar'
+        mock_parent_init.return_value = None
+
+        gcs_bucket_helper = GoogleCloudBucketHelper()
+        gcs_bucket_helper._gcs_hook = mock.Mock()
+
+        def _mock_download(bucket, object, filename=None):
+            text_file_contents = 'text file contents'
+            with open(filename, 'w') as text_file:
+                text_file.write(text_file_contents)
+            return text_file_contents
+
+        gcs_bucket_helper._gcs_hook.download.side_effect = _mock_download
+
+        local_file = gcs_bucket_helper.google_cloud_to_local(file_name)
+        self.assertIn('obj.jar', local_file)
+
+    @mock.patch(
+        'airflow.contrib.operators.dataflow_operator.GoogleCloudBucketHelper.__init__'
+    )
+    def test_empty_object(self, mock_parent_init):
+
+        file_name = 'gs://test-bucket/path/to/obj.jar'
+        mock_parent_init.return_value = None
+
+        gcs_bucket_helper = GoogleCloudBucketHelper()
+        gcs_bucket_helper._gcs_hook = mock.Mock()
+
+        def _mock_download(bucket, object, filename=None):
+            text_file_contents = ''
+            with open(filename, 'w') as text_file:
+                text_file.write(text_file_contents)
+            return text_file_contents
+
+        gcs_bucket_helper._gcs_hook.download.side_effect = _mock_download
+
+        with self.assertRaises(Exception) as context:
+            gcs_bucket_helper.google_cloud_to_local(file_name)
+
+        self.assertEqual(
+            'Failed to download Google Cloud Storage (GCS) object: {}'.format(file_name),
+            str(context.exception))
diff --git a/tests/contrib/operators/test_dataproc_operator.py b/tests/contrib/operators/test_dataproc_operator.py
index 65ff5cd756c89..47a105ff52789 100644
--- a/tests/contrib/operators/test_dataproc_operator.py
+++ b/tests/contrib/operators/test_dataproc_operator.py
@@ -21,6 +21,7 @@
 import datetime
 import re
 import unittest
+from typing import Dict
 
 from airflow import DAG, AirflowException
 from airflow.contrib.operators.dataproc_operator import \
@@ -34,38 +35,34 @@
     DataprocWorkflowTemplateInstantiateOperator, \
     DataprocClusterScaleOperator
 from airflow.version import version
+from tests.compat import mock
 
 from copy import deepcopy
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
-from mock import Mock
+from mock import MagicMock, Mock
 from mock import patch
 
 TASK_ID = 'test-dataproc-operator'
 CLUSTER_NAME = 'test-cluster-name'
-PROJECT_ID = 'test-project-id'
+GCP_PROJECT_ID = 'test-project-id'
 NUM_WORKERS = 123
-ZONE = 'us-central1-a'
+GCE_ZONE = 'us-central1-a'
 NETWORK_URI = '/projects/project_id/regions/global/net'
 SUBNETWORK_URI = '/projects/project_id/regions/global/subnet'
 INTERNAL_IP_ONLY = True
 TAGS = ['tag1', 'tag2']
 STORAGE_BUCKET = 'gs://airflow-test-bucket/'
 IMAGE_VERSION = '1.1'
+CUSTOM_IMAGE = 'test-custom-image'
 MASTER_MACHINE_TYPE = 'n1-standard-2'
 MASTER_DISK_SIZE = 100
+MASTER_DISK_TYPE = 'pd-standard'
 WORKER_MACHINE_TYPE = 'n1-standard-2'
 WORKER_DISK_SIZE = 100
+WORKER_DISK_TYPE = 'pd-standard'
 NUM_PREEMPTIBLE_WORKERS = 2
 GET_INIT_ACTION_TIMEOUT = "600s"  # 10m
-LABEL1 = {}
+LABEL1 = {}  # type: Dict
 LABEL2 = {'application': 'test', 'year': 2017}
 SERVICE_ACCOUNT_SCOPES = [
     'https://www.googleapis.com/auth/bigquery',
@@ -75,11 +72,32 @@
 AUTO_DELETE_TIME = datetime.datetime(2017, 6, 7)
 AUTO_DELETE_TTL = 654
 DEFAULT_DATE = datetime.datetime(2017, 6, 6)
-REGION = 'test-region'
+GCP_REGION = 'test-region'
 MAIN_URI = 'test-uri'
 TEMPLATE_ID = 'template-id'
 
 HOOK = 'airflow.contrib.operators.dataproc_operator.DataProcHook'
+DATAPROC_JOB_ID = 'dataproc_job_id'
+DATAPROC_JOB_TO_SUBMIT = {
+    'job': {
+        'reference': {
+            'projectId': GCP_PROJECT_ID,
+            'jobId': DATAPROC_JOB_ID,
+        },
+        'placement': {
+            'clusterName': CLUSTER_NAME
+        }
+    }
+}
+
+
+def _assert_dataproc_job_id(mock_hook, dataproc_task):
+    hook = mock_hook.return_value
+    job = MagicMock()
+    job.build.return_value = DATAPROC_JOB_TO_SUBMIT
+    hook.create_job_template.return_value = job
+    dataproc_task.execute(None)
+    assert dataproc_task.dataproc_job_id == DATAPROC_JOB_ID
 
 
 class DataprocClusterCreateOperatorTest(unittest.TestCase):
@@ -90,13 +108,13 @@ def setUp(self):
         self.dataproc_operators = []
         self.mock_conn = Mock()
         for labels in self.labels:
-             self.dataproc_operators.append(
+            self.dataproc_operators.append(
                 DataprocClusterCreateOperator(
                     task_id=TASK_ID,
                     cluster_name=CLUSTER_NAME,
-                    project_id=PROJECT_ID,
+                    project_id=GCP_PROJECT_ID,
                     num_workers=NUM_WORKERS,
-                    zone=ZONE,
+                    zone=GCE_ZONE,
                     network_uri=NETWORK_URI,
                     subnetwork_uri=SUBNETWORK_URI,
                     internal_ip_only=INTERNAL_IP_ONLY,
@@ -104,8 +122,10 @@ def setUp(self):
                     storage_bucket=STORAGE_BUCKET,
                     image_version=IMAGE_VERSION,
                     master_machine_type=MASTER_MACHINE_TYPE,
+                    master_disk_type=MASTER_DISK_TYPE,
                     master_disk_size=MASTER_DISK_SIZE,
                     worker_machine_type=WORKER_MACHINE_TYPE,
+                    worker_disk_type=WORKER_DISK_TYPE,
                     worker_disk_size=WORKER_DISK_SIZE,
                     num_preemptible_workers=NUM_PREEMPTIBLE_WORKERS,
                     labels=deepcopy(labels),
@@ -114,7 +134,7 @@ def setUp(self):
                     auto_delete_time=AUTO_DELETE_TIME,
                     auto_delete_ttl=AUTO_DELETE_TTL
                 )
-             )
+            )
         self.dag = DAG(
             'test_dag',
             default_args={
@@ -128,9 +148,9 @@ def test_init(self):
         """Test DataProcClusterOperator instance is properly initialized."""
         for suffix, dataproc_operator in enumerate(self.dataproc_operators):
             self.assertEqual(dataproc_operator.cluster_name, CLUSTER_NAME)
-            self.assertEqual(dataproc_operator.project_id, PROJECT_ID)
+            self.assertEqual(dataproc_operator.project_id, GCP_PROJECT_ID)
             self.assertEqual(dataproc_operator.num_workers, NUM_WORKERS)
-            self.assertEqual(dataproc_operator.zone, ZONE)
+            self.assertEqual(dataproc_operator.zone, GCE_ZONE)
             self.assertEqual(dataproc_operator.network_uri, NETWORK_URI)
             self.assertEqual(dataproc_operator.subnetwork_uri, SUBNETWORK_URI)
             self.assertEqual(dataproc_operator.tags, TAGS)
@@ -138,8 +158,10 @@ def test_init(self):
             self.assertEqual(dataproc_operator.image_version, IMAGE_VERSION)
             self.assertEqual(dataproc_operator.master_machine_type, MASTER_MACHINE_TYPE)
             self.assertEqual(dataproc_operator.master_disk_size, MASTER_DISK_SIZE)
+            self.assertEqual(dataproc_operator.master_disk_type, MASTER_DISK_TYPE)
             self.assertEqual(dataproc_operator.worker_machine_type, WORKER_MACHINE_TYPE)
             self.assertEqual(dataproc_operator.worker_disk_size, WORKER_DISK_SIZE)
+            self.assertEqual(dataproc_operator.worker_disk_type, WORKER_DISK_TYPE)
             self.assertEqual(dataproc_operator.num_preemptible_workers,
                              NUM_PREEMPTIBLE_WORKERS)
             self.assertEqual(dataproc_operator.labels, self.labels[suffix])
@@ -158,7 +180,7 @@ def test_build_cluster_data(self):
         for suffix, dataproc_operator in enumerate(self.dataproc_operators):
             cluster_data = dataproc_operator._build_cluster_data()
             self.assertEqual(cluster_data['clusterName'], CLUSTER_NAME)
-            self.assertEqual(cluster_data['projectId'], PROJECT_ID)
+            self.assertEqual(cluster_data['projectId'], GCP_PROJECT_ID)
             self.assertEqual(cluster_data['config']['softwareConfig'],
                              {'imageVersion': IMAGE_VERSION})
             self.assertEqual(cluster_data['config']['configBucket'], STORAGE_BUCKET)
@@ -186,7 +208,7 @@ def test_build_cluster_data(self):
             # set to the dataproc operator.
             merged_labels = {}
             merged_labels.update(self.labels[suffix])
-            merged_labels.update({'airflow-version': 'v' + version.replace('.', '-').replace('+','-')})
+            merged_labels.update({'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')})
             self.assertTrue(re.match(r'[a-z]([-a-z0-9]*[a-z0-9])?',
                                      cluster_data['labels']['airflow-version']))
             self.assertEqual(cluster_data['labels'], merged_labels)
@@ -195,9 +217,9 @@ def test_build_cluster_data_with_autoDeleteTime(self):
         dataproc_operator = DataprocClusterCreateOperator(
             task_id=TASK_ID,
             cluster_name=CLUSTER_NAME,
-            project_id=PROJECT_ID,
+            project_id=GCP_PROJECT_ID,
             num_workers=NUM_WORKERS,
-            zone=ZONE,
+            zone=GCE_ZONE,
             dag=self.dag,
             auto_delete_time=AUTO_DELETE_TIME,
         )
@@ -209,9 +231,9 @@ def test_build_cluster_data_with_autoDeleteTtl(self):
         dataproc_operator = DataprocClusterCreateOperator(
             task_id=TASK_ID,
             cluster_name=CLUSTER_NAME,
-            project_id=PROJECT_ID,
+            project_id=GCP_PROJECT_ID,
             num_workers=NUM_WORKERS,
-            zone=ZONE,
+            zone=GCE_ZONE,
             dag=self.dag,
             auto_delete_ttl=AUTO_DELETE_TTL,
         )
@@ -223,9 +245,9 @@ def test_build_cluster_data_with_autoDeleteTime_and_autoDeleteTtl(self):
         dataproc_operator = DataprocClusterCreateOperator(
             task_id=TASK_ID,
             cluster_name=CLUSTER_NAME,
-            project_id=PROJECT_ID,
+            project_id=GCP_PROJECT_ID,
             num_workers=NUM_WORKERS,
-            zone=ZONE,
+            zone=GCE_ZONE,
             dag=self.dag,
             auto_delete_time=AUTO_DELETE_TIME,
             auto_delete_ttl=AUTO_DELETE_TTL,
@@ -237,33 +259,92 @@ def test_build_cluster_data_with_autoDeleteTime_and_autoDeleteTtl(self):
         self.assertEqual(cluster_data['config']['lifecycleConfig']['autoDeleteTime'],
                          "2017-06-07T00:00:00.000000Z")
 
+    def test_init_with_image_version_and_custom_image_both_set(self):
+        with self.assertRaises(AssertionError):
+            DataprocClusterCreateOperator(
+                task_id=TASK_ID,
+                cluster_name=CLUSTER_NAME,
+                project_id=GCP_PROJECT_ID,
+                num_workers=NUM_WORKERS,
+                zone=GCE_ZONE,
+                dag=self.dag,
+                image_version=IMAGE_VERSION,
+                custom_image=CUSTOM_IMAGE
+            )
+
+    def test_init_with_custom_image(self):
+        dataproc_operator = DataprocClusterCreateOperator(
+            task_id=TASK_ID,
+            cluster_name=CLUSTER_NAME,
+            project_id=GCP_PROJECT_ID,
+            num_workers=NUM_WORKERS,
+            zone=GCE_ZONE,
+            dag=self.dag,
+            custom_image=CUSTOM_IMAGE
+        )
+
+        cluster_data = dataproc_operator._build_cluster_data()
+        expected_custom_image_url = \
+            'https://www.googleapis.com/compute/beta/projects/' \
+            '{}/global/images/{}'.format(GCP_PROJECT_ID, CUSTOM_IMAGE)
+        self.assertEqual(cluster_data['config']['masterConfig']['imageUri'],
+                         expected_custom_image_url)
+        self.assertEqual(cluster_data['config']['workerConfig']['imageUri'],
+                         expected_custom_image_url)
+
+    def test_build_single_node_cluster(self):
+        dataproc_operator = DataprocClusterCreateOperator(
+            task_id=TASK_ID,
+            cluster_name=CLUSTER_NAME,
+            project_id=GCP_PROJECT_ID,
+            num_workers=0,
+            num_preemptible_workers=0,
+            zone=GCE_ZONE,
+            dag=self.dag
+        )
+        cluster_data = dataproc_operator._build_cluster_data()
+        self.assertEqual(
+            cluster_data['config']['softwareConfig']['properties']
+            ['dataproc:dataproc.allow.zero.workers'], "true")
+
+    def test_init_cluster_with_zero_workers_and_not_non_zero_preemtibles(self):
+        with self.assertRaises(AssertionError):
+            DataprocClusterCreateOperator(
+                task_id=TASK_ID,
+                cluster_name=CLUSTER_NAME,
+                project_id=GCP_PROJECT_ID,
+                num_workers=0,
+                num_preemptible_workers=2,
+                zone=GCE_ZONE,
+                dag=self.dag,
+                image_version=IMAGE_VERSION,
+            )
+
     def test_cluster_name_log_no_sub(self):
-        with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') \
-                as mock_hook:
+        with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
             mock_hook.return_value.get_conn = self.mock_conn
             dataproc_task = DataprocClusterCreateOperator(
                 task_id=TASK_ID,
                 cluster_name=CLUSTER_NAME,
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 num_workers=NUM_WORKERS,
-                zone=ZONE,
+                zone=GCE_ZONE,
                 dag=self.dag
             )
             with patch.object(dataproc_task.log, 'info') as mock_info:
-                with self.assertRaises(TypeError) as _:
+                with self.assertRaises(TypeError):
                     dataproc_task.execute(None)
                 mock_info.assert_called_with('Creating cluster: %s', CLUSTER_NAME)
 
     def test_cluster_name_log_sub(self):
-        with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') \
-                as mock_hook:
+        with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
             mock_hook.return_value.get_conn = self.mock_conn
             dataproc_task = DataprocClusterCreateOperator(
                 task_id=TASK_ID,
                 cluster_name='smoke-cluster-{{ ts_nodash }}',
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 num_workers=NUM_WORKERS,
-                zone=ZONE,
+                zone=GCE_ZONE,
                 dag=self.dag
             )
             with patch.object(dataproc_task.log, 'info') as mock_info:
@@ -275,20 +356,18 @@ def test_cluster_name_log_sub(self):
                 setattr(dataproc_task, 'cluster_name', rendered)
                 with self.assertRaises(TypeError):
                     dataproc_task.execute(None)
-                mock_info.assert_called_with('Creating cluster: %s',
-                                             u'smoke-cluster-testnodash')
+                mock_info.assert_called_with('Creating cluster: %s', u'smoke-cluster-testnodash')
 
     def test_build_cluster_data_internal_ip_only_without_subnetwork(self):
 
         def create_cluster_with_invalid_internal_ip_only_setup():
-
             # Given
             create_cluster = DataprocClusterCreateOperator(
                 task_id=TASK_ID,
                 cluster_name=CLUSTER_NAME,
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 num_workers=NUM_WORKERS,
-                zone=ZONE,
+                zone=GCE_ZONE,
                 dag=self.dag,
                 internal_ip_only=True)
 
@@ -300,8 +379,7 @@ def create_cluster_with_invalid_internal_ip_only_setup():
             create_cluster_with_invalid_internal_ip_only_setup()
 
         self.assertEqual(str(cm.exception),
-                         "Set internal_ip_only to true only when"
-                         " you pass a subnetwork_uri.")
+                         "Set internal_ip_only to true only when you pass a subnetwork_uri.")
 
 
 class DataprocClusterScaleOperatorTest(unittest.TestCase):
@@ -334,7 +412,7 @@ def test_cluster_name_log_no_sub(self):
             dataproc_task = DataprocClusterScaleOperator(
                 task_id=TASK_ID,
                 cluster_name=CLUSTER_NAME,
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 num_workers=NUM_WORKERS,
                 num_preemptible_workers=NUM_PREEMPTIBLE_WORKERS,
                 dag=self.dag
@@ -345,13 +423,12 @@ def test_cluster_name_log_no_sub(self):
                 mock_info.assert_called_with('Scaling cluster: %s', CLUSTER_NAME)
 
     def test_cluster_name_log_sub(self):
-        with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') \
-                as mock_hook:
+        with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
             mock_hook.return_value.get_conn = self.mock_conn
             dataproc_task = DataprocClusterScaleOperator(
                 task_id=TASK_ID,
                 cluster_name='smoke-cluster-{{ ts_nodash }}',
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 num_workers=NUM_WORKERS,
                 num_preemptible_workers=NUM_PREEMPTIBLE_WORKERS,
                 dag=self.dag
@@ -366,22 +443,21 @@ def test_cluster_name_log_sub(self):
                 setattr(dataproc_task, 'cluster_name', rendered)
                 with self.assertRaises(TypeError):
                     dataproc_task.execute(None)
-                mock_info.assert_called_with('Scaling cluster: %s',
-                                             u'smoke-cluster-testnodash')
+                mock_info.assert_called_with('Scaling cluster: %s', u'smoke-cluster-testnodash')
 
 
 class DataprocClusterDeleteOperatorTest(unittest.TestCase):
     # Unit test for the DataprocClusterDeleteOperator
     def setUp(self):
         self.mock_execute = Mock()
-        self.mock_execute.execute = Mock(return_value={'done' : True})
+        self.mock_execute.execute = Mock(return_value={'done': True})
         self.mock_get = Mock()
         self.mock_get.get = Mock(return_value=self.mock_execute)
         self.mock_operations = Mock()
         self.mock_operations.get = Mock(return_value=self.mock_get)
         self.mock_regions = Mock()
         self.mock_regions.operations = Mock(return_value=self.mock_operations)
-        self.mock_projects=Mock()
+        self.mock_projects = Mock()
         self.mock_projects.regions = Mock(return_value=self.mock_regions)
         self.mock_conn = Mock()
         self.mock_conn.projects = Mock(return_value=self.mock_projects)
@@ -400,11 +476,11 @@ def test_cluster_name_log_no_sub(self):
             dataproc_task = DataprocClusterDeleteOperator(
                 task_id=TASK_ID,
                 cluster_name=CLUSTER_NAME,
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 dag=self.dag
             )
             with patch.object(dataproc_task.log, 'info') as mock_info:
-                with self.assertRaises(TypeError) as _:
+                with self.assertRaises(TypeError):
                     dataproc_task.execute(None)
                 mock_info.assert_called_with('Deleting cluster: %s', CLUSTER_NAME)
 
@@ -414,7 +490,7 @@ def test_cluster_name_log_sub(self):
             dataproc_task = DataprocClusterDeleteOperator(
                 task_id=TASK_ID,
                 cluster_name='smoke-cluster-{{ ts_nodash }}',
-                project_id=PROJECT_ID,
+                project_id=GCP_PROJECT_ID,
                 dag=self.dag
             )
 
@@ -433,52 +509,100 @@ def test_cluster_name_log_sub(self):
 
 class DataProcHadoopOperatorTest(unittest.TestCase):
     # Unit test for the DataProcHadoopOperator
-    def test_hook_correct_region(self):
-       with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
+    @staticmethod
+    def test_hook_correct_region():
+        with patch(HOOK) as mock_hook:
             dataproc_task = DataProcHadoopOperator(
                 task_id=TASK_ID,
-                region=REGION
+                region=GCP_REGION
             )
 
             dataproc_task.execute(None)
-            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY, REGION)
+            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY,
+                                                                  GCP_REGION, mock.ANY)
+
+    @staticmethod
+    def test_dataproc_job_id_is_set():
+        with patch(HOOK) as mock_hook:
+            dataproc_task = DataProcHadoopOperator(
+                task_id=TASK_ID
+            )
+
+            _assert_dataproc_job_id(mock_hook, dataproc_task)
+
 
 class DataProcHiveOperatorTest(unittest.TestCase):
     # Unit test for the DataProcHiveOperator
-    def test_hook_correct_region(self):
-       with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
+    @staticmethod
+    def test_hook_correct_region():
+        with patch(HOOK) as mock_hook:
             dataproc_task = DataProcHiveOperator(
                 task_id=TASK_ID,
-                region=REGION
+                region=GCP_REGION
             )
 
             dataproc_task.execute(None)
-            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY, REGION)
+            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY,
+                                                                  GCP_REGION, mock.ANY)
+
+    @staticmethod
+    def test_dataproc_job_id_is_set():
+        with patch(HOOK) as mock_hook:
+            dataproc_task = DataProcHiveOperator(
+                task_id=TASK_ID
+            )
+
+            _assert_dataproc_job_id(mock_hook, dataproc_task)
+
 
 class DataProcPySparkOperatorTest(unittest.TestCase):
     # Unit test for the DataProcPySparkOperator
-    def test_hook_correct_region(self):
-       with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
+    @staticmethod
+    def test_hook_correct_region():
+        with patch(HOOK) as mock_hook:
             dataproc_task = DataProcPySparkOperator(
                 task_id=TASK_ID,
                 main=MAIN_URI,
-                region=REGION
+                region=GCP_REGION
             )
 
             dataproc_task.execute(None)
-            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY, REGION)
+            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY,
+                                                                  GCP_REGION, mock.ANY)
+
+    @staticmethod
+    def test_dataproc_job_id_is_set():
+        with patch(HOOK) as mock_hook:
+            dataproc_task = DataProcPySparkOperator(
+                task_id=TASK_ID,
+                main=MAIN_URI
+            )
+
+            _assert_dataproc_job_id(mock_hook, dataproc_task)
+
 
 class DataProcSparkOperatorTest(unittest.TestCase):
     # Unit test for the DataProcSparkOperator
-    def test_hook_correct_region(self):
-       with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') as mock_hook:
+    @staticmethod
+    def test_hook_correct_region():
+        with patch(HOOK) as mock_hook:
             dataproc_task = DataProcSparkOperator(
                 task_id=TASK_ID,
-                region=REGION
+                region=GCP_REGION
             )
 
             dataproc_task.execute(None)
-            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY, REGION)
+            mock_hook.return_value.submit.assert_called_once_with(mock.ANY, mock.ANY,
+                                                                  GCP_REGION, mock.ANY)
+
+    @staticmethod
+    def test_dataproc_job_id_is_set():
+        with patch(HOOK) as mock_hook:
+            dataproc_task = DataProcSparkOperator(
+                task_id=TASK_ID
+            )
+
+            _assert_dataproc_job_id(mock_hook, dataproc_task)
 
 
 class DataprocWorkflowTemplateInstantiateOperatorTest(unittest.TestCase):
@@ -508,12 +632,12 @@ def test_workflow(self):
         with patch(HOOK) as MockHook:
             hook = MockHook()
             hook.get_conn.return_value = self.mock_conn
-            hook.await.return_value = None
+            hook.wait.return_value = None
 
             dataproc_task = DataprocWorkflowTemplateInstantiateOperator(
                 task_id=TASK_ID,
-                project_id=PROJECT_ID,
-                region=REGION,
+                project_id=GCP_PROJECT_ID,
+                region=GCP_REGION,
                 template_id=TEMPLATE_ID,
                 dag=self.dag
             )
@@ -525,7 +649,7 @@ def test_workflow(self):
             self.mock_workflows.instantiate.assert_called_once_with(
                 name=template_name,
                 body=mock.ANY)
-            hook.await.assert_called_once_with(self.operation)
+            hook.wait.assert_called_once_with(self.operation)
 
 
 class DataprocWorkflowTemplateInstantiateInlineOperatorTest(unittest.TestCase):
@@ -556,7 +680,7 @@ def test_iniline_workflow(self):
         with patch(HOOK) as MockHook:
             hook = MockHook()
             hook.get_conn.return_value = self.mock_conn
-            hook.await.return_value = None
+            hook.wait.return_value = None
 
             template = {
                 "placement": {
@@ -564,7 +688,7 @@ def test_iniline_workflow(self):
                         "cluster_name": CLUSTER_NAME,
                         "config": {
                             "gce_cluster_config": {
-                                "zone_uri": ZONE,
+                                "zone_uri": GCE_ZONE,
                             }
                         }
                     }
@@ -580,8 +704,8 @@ def test_iniline_workflow(self):
 
             dataproc_task = DataprocWorkflowTemplateInstantiateInlineOperator(
                 task_id=TASK_ID,
-                project_id=PROJECT_ID,
-                region=REGION,
+                project_id=GCP_PROJECT_ID,
+                region=GCP_REGION,
                 template=template,
                 dag=self.dag
             )
@@ -591,4 +715,4 @@ def test_iniline_workflow(self):
                 parent='projects/test-project-id/regions/test-region',
                 instanceId=mock.ANY,
                 body=template)
-            hook.await.assert_called_once_with(self.operation)
+            hook.wait.assert_called_once_with(self.operation)
diff --git a/tests/contrib/operators/test_dingding_operator.py b/tests/contrib/operators/test_dingding_operator.py
new file mode 100644
index 0000000000000..a744e337d24d3
--- /dev/null
+++ b/tests/contrib/operators/test_dingding_operator.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import mock
+
+from airflow import DAG, configuration
+from airflow.contrib.operators.dingding_operator import DingdingOperator
+from airflow.utils import timezone
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+
+class TestDingdingOperator(unittest.TestCase):
+    _config = {
+        'dingding_conn_id': 'dingding_default',
+        'message_type': 'text',
+        'message': 'Airflow dingding webhook test',
+        'at_mobiles': ['123', '456'],
+        'at_all': False
+    }
+
+    def setUp(self):
+        configuration.load_test_config()
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG('test_dag_id', default_args=args)
+
+    @mock.patch('airflow.contrib.operators.dingding_operator.DingdingHook')
+    def test_execute(self, mock_hook):
+        operator = DingdingOperator(
+            task_id='dingding_task',
+            dag=self.dag,
+            **self._config
+        )
+
+        self.assertIsNotNone(operator)
+        self.assertEqual(self._config['dingding_conn_id'], operator.dingding_conn_id)
+        self.assertEqual(self._config['message_type'], operator.message_type)
+        self.assertEqual(self._config['message'], operator.message)
+        self.assertEqual(self._config['at_mobiles'], operator.at_mobiles)
+        self.assertEqual(self._config['at_all'], operator.at_all)
+
+        operator.execute(None)
+        mock_hook.assert_called_once_with(
+            self._config['dingding_conn_id'],
+            self._config['message_type'],
+            self._config['message'],
+            self._config['at_mobiles'],
+            self._config['at_all']
+        )
+        mock_hook.return_value.send.assert_called_once()
diff --git a/tests/contrib/operators/test_discord_webhook_operator.py b/tests/contrib/operators/test_discord_webhook_operator.py
index 0fa5838351ef9..9816f0e223199 100644
--- a/tests/contrib/operators/test_discord_webhook_operator.py
+++ b/tests/contrib/operators/test_discord_webhook_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/operators/test_druid_operator.py b/tests/contrib/operators/test_druid_operator.py
index d3e51dddba5e6..e7d502c9cf0af 100644
--- a/tests/contrib/operators/test_druid_operator.py
+++ b/tests/contrib/operators/test_druid_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/operators/test_ecs_operator.py b/tests/contrib/operators/test_ecs_operator.py
index b5d3141ec86ce..e65663df6b3ef 100644
--- a/tests/contrib/operators/test_ecs_operator.py
+++ b/tests/contrib/operators/test_ecs_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,18 +22,12 @@
 import unittest
 from copy import deepcopy
 
+from parameterized import parameterized
+
 from airflow import configuration
 from airflow.exceptions import AirflowException
 from airflow.contrib.operators.ecs_operator import ECSOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
+from tests.compat import mock
 
 RESPONSE_WITHOUT_FAILURES = {
     "failures": [],
@@ -41,7 +35,8 @@
         {
             "containers": [
                 {
-                    "containerArn": "arn:aws:ecs:us-east-1:012345678910:container/e1ed7aac-d9b2-4315-8726-d2432bf11868",
+                    "containerArn":
+                        "arn:aws:ecs:us-east-1:012345678910:container/e1ed7aac-d9b2-4315-8726-d2432bf11868",
                     "lastStatus": "PENDING",
                     "name": "wordpress",
                     "taskArn": "arn:aws:ecs:us-east-1:012345678910:task/d8c67b3c-ac87-4ffe-a847-4785bc3a8b55"
@@ -63,16 +58,28 @@ def setUp(self, aws_hook_mock):
         configuration.load_test_config()
 
         self.aws_hook_mock = aws_hook_mock
-        self.ecs = ECSOperator(
-            task_id='task',
-            task_definition='t',
-            cluster='c',
-            overrides={},
-            aws_conn_id=None,
-            region_name='eu-west-1')
+        self.ecs_operator_args = {
+            'task_id': 'task',
+            'task_definition': 't',
+            'cluster': 'c',
+            'overrides': {},
+            'aws_conn_id': None,
+            'region_name': 'eu-west-1',
+            'group': 'group',
+            'placement_constraints': [{
+                'expression': 'attribute:ecs.instance-type =~ t2.*',
+                'type': 'memberOf'
+            }],
+            'network_configuration': {
+                'awsvpcConfiguration': {
+                    'securityGroups': ['sg-123abc'],
+                    'subnets': ['subnet-123456ab']
+                }
+            }
+        }
+        self.ecs = ECSOperator(**self.ecs_operator_args)
 
     def test_init(self):
-
         self.assertEqual(self.ecs.region_name, 'eu-west-1')
         self.assertEqual(self.ecs.task_definition, 't')
         self.assertEqual(self.ecs.aws_conn_id, None)
@@ -85,30 +92,55 @@ def test_init(self):
     def test_template_fields_overrides(self):
         self.assertEqual(self.ecs.template_fields, ('overrides',))
 
+    @parameterized.expand([
+        ['EC2'],
+        ['FARGATE']
+    ])
     @mock.patch.object(ECSOperator, '_wait_for_task_ended')
     @mock.patch.object(ECSOperator, '_check_success_task')
-    def test_execute_without_failures(self, check_mock, wait_mock):
-
-        client_mock = self.aws_hook_mock.return_value.get_client_type.return_value
+    @mock.patch('airflow.contrib.operators.ecs_operator.AwsHook')
+    def test_execute_without_failures(self, launch_type, aws_hook_mock,
+                                      check_mock, wait_mock):
+        client_mock = aws_hook_mock.return_value.get_client_type.return_value
         client_mock.run_task.return_value = RESPONSE_WITHOUT_FAILURES
 
-        self.ecs.execute(None)
+        ecs = ECSOperator(launch_type=launch_type, **self.ecs_operator_args)
+        ecs.execute(None)
+
+        aws_hook_mock.return_value.get_client_type.assert_called_once_with('ecs',
+                                                                           region_name='eu-west-1')
+        extend_args = {}
+        if launch_type == 'FARGATE':
+            extend_args['platformVersion'] = 'LATEST'
 
-        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('ecs', region_name='eu-west-1')
         client_mock.run_task.assert_called_once_with(
             cluster='c',
-            launchType='EC2',
+            launchType=launch_type,
             overrides={},
             startedBy=mock.ANY,  # Can by 'airflow' or 'Airflow'
-            taskDefinition='t'
+            taskDefinition='t',
+            group='group',
+            placementConstraints=[
+                {
+                    'expression': 'attribute:ecs.instance-type =~ t2.*',
+                    'type': 'memberOf'
+                }
+            ],
+            networkConfiguration={
+                'awsvpcConfiguration': {
+                    'securityGroups': ['sg-123abc'],
+                    'subnets': ['subnet-123456ab']
+                }
+            },
+            **extend_args
         )
 
         wait_mock.assert_called_once_with()
         check_mock.assert_called_once_with()
-        self.assertEqual(self.ecs.arn, 'arn:aws:ecs:us-east-1:012345678910:task/d8c67b3c-ac87-4ffe-a847-4785bc3a8b55')
+        self.assertEqual(ecs.arn,
+                         'arn:aws:ecs:us-east-1:012345678910:task/d8c67b3c-ac87-4ffe-a847-4785bc3a8b55')
 
     def test_execute_with_failures(self):
-
         client_mock = self.aws_hook_mock.return_value.get_client_type.return_value
         resp_failures = deepcopy(RESPONSE_WITHOUT_FAILURES)
         resp_failures['failures'].append('dummy error')
@@ -117,25 +149,40 @@ def test_execute_with_failures(self):
         with self.assertRaises(AirflowException):
             self.ecs.execute(None)
 
-        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('ecs', region_name='eu-west-1')
+        self.aws_hook_mock.return_value.get_client_type.assert_called_once_with('ecs',
+                                                                                region_name='eu-west-1')
         client_mock.run_task.assert_called_once_with(
             cluster='c',
             launchType='EC2',
             overrides={},
             startedBy=mock.ANY,  # Can by 'airflow' or 'Airflow'
-            taskDefinition='t'
+            taskDefinition='t',
+            group='group',
+            placementConstraints=[
+                {
+                    'expression': 'attribute:ecs.instance-type =~ t2.*',
+                    'type': 'memberOf'
+                }
+            ],
+            networkConfiguration={
+                'awsvpcConfiguration': {
+                    'securityGroups': ['sg-123abc'],
+                    'subnets': ['subnet-123456ab'],
+                }
+            }
         )
 
     def test_wait_end_tasks(self):
-
         client_mock = mock.Mock()
         self.ecs.arn = 'arn'
         self.ecs.client = client_mock
 
         self.ecs._wait_for_task_ended()
         client_mock.get_waiter.assert_called_once_with('tasks_stopped')
-        client_mock.get_waiter.return_value.wait.assert_called_once_with(cluster='c', tasks=['arn'])
-        self.assertEquals(sys.maxsize, client_mock.get_waiter.return_value.config.max_attempts)
+        client_mock.get_waiter.return_value.wait.assert_called_once_with(
+            cluster='c', tasks=['arn'])
+        self.assertEqual(
+            sys.maxsize, client_mock.get_waiter.return_value.config.max_attempts)
 
     def test_check_success_tasks_raises(self):
         client_mock = mock.Mock()
@@ -159,7 +206,8 @@ def test_check_success_tasks_raises(self):
         self.assertIn("'name': 'foo'", str(e.exception))
         self.assertIn("'lastStatus': 'STOPPED'", str(e.exception))
         self.assertIn("'exitCode': 1", str(e.exception))
-        client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn'])
+        client_mock.describe_tasks.assert_called_once_with(
+            cluster='c', tasks=['arn'])
 
     def test_check_success_tasks_raises_pending(self):
         client_mock = mock.Mock()
@@ -179,9 +227,10 @@ def test_check_success_tasks_raises_pending(self):
         self.assertIn("This task is still pending ", str(e.exception))
         self.assertIn("'name': 'container-name'", str(e.exception))
         self.assertIn("'lastStatus': 'PENDING'", str(e.exception))
-        client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn'])
+        client_mock.describe_tasks.assert_called_once_with(
+            cluster='c', tasks=['arn'])
 
-    def test_check_success_tasks_raises_mutliple(self):
+    def test_check_success_tasks_raises_multiple(self):
         client_mock = mock.Mock()
         self.ecs.client = client_mock
         self.ecs.arn = 'arn'
@@ -198,7 +247,42 @@ def test_check_success_tasks_raises_mutliple(self):
             }]
         }
         self.ecs._check_success_task()
-        client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn'])
+        client_mock.describe_tasks.assert_called_once_with(
+            cluster='c', tasks=['arn'])
+
+    def test_host_terminated_raises(self):
+        client_mock = mock.Mock()
+        self.ecs.client = client_mock
+        self.ecs.arn = 'arn'
+        client_mock.describe_tasks.return_value = {
+            'tasks': [{
+                'stoppedReason': 'Host EC2 (instance i-1234567890abcdef) terminated.',
+                "containers": [
+                    {
+                        "containerArn": "arn:aws:ecs:us-east-1:012345678910:container/e1ed7aac-d9b2-4315-8726-d2432bf11868",  # noqa: E501
+                        "lastStatus": "RUNNING",
+                        "name": "wordpress",
+                        "taskArn": "arn:aws:ecs:us-east-1:012345678910:task/d8c67b3c-ac87-4ffe-a847-4785bc3a8b55"  # noqa: E501
+                    }
+                ],
+                "desiredStatus": "STOPPED",
+                "lastStatus": "STOPPED",
+                "taskArn": "arn:aws:ecs:us-east-1:012345678910:task/d8c67b3c-ac87-4ffe-a847-4785bc3a8b55",  # noqa: E501
+                "taskDefinitionArn": "arn:aws:ecs:us-east-1:012345678910:task-definition/hello_world:11"  # noqa: E501
+
+            }]
+        }
+
+        with self.assertRaises(AirflowException) as e:
+            self.ecs._check_success_task()
+
+        self.assertIn(
+            "The task was stopped because the host instance terminated:",
+            str(e.exception))
+        self.assertIn("Host EC2 (", str(e.exception))
+        self.assertIn(") terminated", str(e.exception))
+        client_mock.describe_tasks.assert_called_once_with(
+            cluster='c', tasks=['arn'])
 
     def test_check_success_task_not_raises(self):
         client_mock = mock.Mock()
@@ -214,7 +298,8 @@ def test_check_success_task_not_raises(self):
             }]
         }
         self.ecs._check_success_task()
-        client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn'])
+        client_mock.describe_tasks.assert_called_once_with(
+            cluster='c', tasks=['arn'])
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/operators/test_emr_add_steps_operator.py b/tests/contrib/operators/test_emr_add_steps_operator.py
index 1315f93769c6b..5caa79cf36766 100644
--- a/tests/contrib/operators/test_emr_add_steps_operator.py
+++ b/tests/contrib/operators/test_emr_add_steps_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/operators/test_emr_create_job_flow_operator.py b/tests/contrib/operators/test_emr_create_job_flow_operator.py
index 7dae30d43a809..fd95f78fd2d01 100644
--- a/tests/contrib/operators/test_emr_create_job_flow_operator.py
+++ b/tests/contrib/operators/test_emr_create_job_flow_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -71,12 +71,14 @@ def setUp(self):
             aws_conn_id='aws_default',
             emr_conn_id='emr_default',
             job_flow_overrides=self._config,
+            region_name='ap-southeast-2',
             dag=DAG('test_dag_id', default_args=args)
         )
 
     def test_init(self):
         self.assertEqual(self.operator.aws_conn_id, 'aws_default')
         self.assertEqual(self.operator.emr_conn_id, 'emr_default')
+        self.assertEqual(self.operator.region_name, 'ap-southeast-2')
 
     def test_render_template(self):
         ti = TaskInstance(self.operator, DEFAULT_DATE)
diff --git a/tests/contrib/operators/test_emr_terminate_job_flow_operator.py b/tests/contrib/operators/test_emr_terminate_job_flow_operator.py
index 7eb27ec16ec79..d25b02adb5bd9 100644
--- a/tests/contrib/operators/test_emr_terminate_job_flow_operator.py
+++ b/tests/contrib/operators/test_emr_terminate_job_flow_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -44,10 +44,8 @@ def setUp(self):
         # Mock out the emr_client creator
         self.boto3_session_mock = MagicMock(return_value=mock_emr_session)
 
-
     def test_execute_terminates_the_job_flow_and_does_not_error(self):
         with patch('boto3.session.Session', self.boto3_session_mock):
-
             operator = EmrTerminateJobFlowOperator(
                 task_id='test_task',
                 job_flow_id='j-8989898989',
@@ -56,5 +54,6 @@ def test_execute_terminates_the_job_flow_and_does_not_error(self):
 
             operator.execute(None)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/operators/test_file_to_gcs.py b/tests/contrib/operators/test_file_to_gcs.py
new file mode 100644
index 0000000000000..29ceae5d52bde
--- /dev/null
+++ b/tests/contrib/operators/test_file_to_gcs.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import datetime
+import unittest
+
+from airflow import DAG, configuration
+from airflow.contrib.operators.file_to_gcs import FileToGoogleCloudStorageOperator
+from tests.compat import mock
+
+
+class TestFileToGcsOperator(unittest.TestCase):
+
+    _config = {
+        'src': '/tmp/fake.csv',
+        'dst': 'fake.csv',
+        'bucket': 'dummy',
+        'mime_type': 'application/octet-stream',
+        'gzip': False
+    }
+
+    def setUp(self):
+        configuration.load_test_config()
+        args = {
+            'owner': 'airflow',
+            'start_date': datetime.datetime(2017, 1, 1)
+        }
+        self.dag = DAG('test_dag_id', default_args=args)
+
+    def test_init(self):
+        operator = FileToGoogleCloudStorageOperator(
+            task_id='file_to_gcs_operator',
+            dag=self.dag,
+            **self._config
+        )
+        self.assertEqual(operator.src, self._config['src'])
+        self.assertEqual(operator.dst, self._config['dst'])
+        self.assertEqual(operator.bucket, self._config['bucket'])
+        self.assertEqual(operator.mime_type, self._config['mime_type'])
+        self.assertEqual(operator.gzip, self._config['gzip'])
+
+    @mock.patch('airflow.contrib.operators.file_to_gcs.GoogleCloudStorageHook',
+                autospec=True)
+    def test_execute(self, mock_hook):
+        mock_instance = mock_hook.return_value
+        operator = FileToGoogleCloudStorageOperator(
+            task_id='gcs_to_file_sensor',
+            dag=self.dag,
+            **self._config
+        )
+        operator.execute(None)
+        mock_instance.upload.assert_called_once_with(
+            bucket=self._config['bucket'],
+            filename=self._config['src'],
+            gzip=self._config['gzip'],
+            mime_type=self._config['mime_type'],
+            object=self._config['dst']
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_file_to_wasb.py b/tests/contrib/operators/test_file_to_wasb.py
index f508be1574d26..61fcc7cba3243 100644
--- a/tests/contrib/operators/test_file_to_wasb.py
+++ b/tests/contrib/operators/test_file_to_wasb.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,14 +23,7 @@
 
 from airflow import DAG, configuration
 from airflow.contrib.operators.file_to_wasb import FileToWasbOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class TestFileToWasbOperator(unittest.TestCase):
diff --git a/tests/contrib/operators/test_gcp_bigtable_operator.py b/tests/contrib/operators/test_gcp_bigtable_operator.py
new file mode 100644
index 0000000000000..ed839bd231dfb
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_bigtable_operator.py
@@ -0,0 +1,705 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from typing import List, Dict
+import google.api_core.exceptions
+from google.cloud.bigtable.column_family import MaxVersionsGCRule
+from google.cloud.bigtable.instance import Instance
+from google.cloud.bigtable.table import ClusterState
+from parameterized import parameterized
+
+from airflow import AirflowException
+from airflow.contrib.operators.gcp_bigtable_operator import \
+    BigtableInstanceDeleteOperator, \
+    BigtableTableDeleteOperator, \
+    BigtableTableCreateOperator, \
+    BigtableTableWaitForReplicationSensor, \
+    BigtableClusterUpdateOperator, \
+    BigtableInstanceCreateOperator
+from tests.compat import mock
+
+PROJECT_ID = 'test_project_id'
+INSTANCE_ID = 'test-instance-id'
+CLUSTER_ID = 'test-cluster-id'
+CLUSTER_ZONE = 'us-central1-f'
+NODES = 5
+TABLE_ID = 'test-table-id'
+INITIAL_SPLIT_KEYS = []  # type: List
+EMPTY_COLUMN_FAMILIES = {}  # type: Dict
+
+
+class BigtableInstanceCreateTest(unittest.TestCase):
+    @parameterized.expand([
+        ('instance_id', PROJECT_ID, '', CLUSTER_ID, CLUSTER_ZONE),
+        ('main_cluster_id', PROJECT_ID, INSTANCE_ID, '', CLUSTER_ZONE),
+        ('main_cluster_zone', PROJECT_ID, INSTANCE_ID, CLUSTER_ID, ''),
+    ], testcase_func_name=lambda f, n, p: 'test_empty_attribute.empty_' + p.args[0])
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_empty_attribute(self, missing_attribute, project_id, instance_id,
+                             main_cluster_id,
+                             main_cluster_zone, mock_hook):
+        with self.assertRaises(AirflowException) as e:
+            BigtableInstanceCreateOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                main_cluster_id=main_cluster_id,
+                main_cluster_zone=main_cluster_zone,
+                task_id="id"
+            )
+        err = e.exception
+        self.assertEqual(str(err), 'Empty parameter: {}'.format(missing_attribute))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_create_instance_that_exists(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+
+        op = BigtableInstanceCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            main_cluster_id=CLUSTER_ID,
+            main_cluster_zone=CLUSTER_ZONE,
+            task_id="id"
+        )
+        op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.create_instance.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_create_instance_that_exists_empty_project_id(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+
+        op = BigtableInstanceCreateOperator(
+            instance_id=INSTANCE_ID,
+            main_cluster_id=CLUSTER_ID,
+            main_cluster_zone=CLUSTER_ZONE,
+            task_id="id"
+        )
+        op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.create_instance.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_different_error_reraised(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+        op = BigtableInstanceCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            main_cluster_id=CLUSTER_ID,
+            main_cluster_zone=CLUSTER_ZONE,
+            task_id="id"
+        )
+
+        mock_hook.return_value.create_instance.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.GoogleAPICallError('error'))
+
+        with self.assertRaises(google.api_core.exceptions.GoogleAPICallError):
+            op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.create_instance.assert_called_once_with(
+            cluster_nodes=None,
+            cluster_storage_type=None,
+            instance_display_name=None,
+            instance_id=INSTANCE_ID,
+            instance_labels=None,
+            instance_type=None,
+            main_cluster_id=CLUSTER_ID,
+            main_cluster_zone=CLUSTER_ZONE,
+            project_id=PROJECT_ID,
+            replica_cluster_id=None,
+            replica_cluster_zone=None,
+            timeout=None
+        )
+
+
+class BigtableClusterUpdateTest(unittest.TestCase):
+    @parameterized.expand([
+        ('instance_id', PROJECT_ID, '', CLUSTER_ID, NODES),
+        ('cluster_id', PROJECT_ID, INSTANCE_ID, '', NODES),
+        ('nodes', PROJECT_ID, INSTANCE_ID, CLUSTER_ID, ''),
+    ], testcase_func_name=lambda f, n, p: 'test_empty_attribute.empty_' + p.args[0])
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_empty_attribute(self, missing_attribute, project_id, instance_id,
+                             cluster_id, nodes, mock_hook):
+        with self.assertRaises(AirflowException) as e:
+            BigtableClusterUpdateOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                cluster_id=cluster_id,
+                nodes=nodes,
+                task_id="id"
+            )
+        err = e.exception
+        self.assertEqual(str(err), 'Empty parameter: {}'.format(missing_attribute))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_updating_cluster_but_instance_does_not_exists(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+
+        with self.assertRaises(AirflowException) as e:
+            op = BigtableClusterUpdateOperator(
+                project_id=PROJECT_ID,
+                instance_id=INSTANCE_ID,
+                cluster_id=CLUSTER_ID,
+                nodes=NODES,
+                task_id="id"
+            )
+            op.execute(None)
+
+        err = e.exception
+        self.assertEqual(str(err), "Dependency: instance '{}' does not exist.".format(
+            INSTANCE_ID))
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.update_cluster.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_updating_cluster_but_instance_does_not_exists_empty_project_id(self,
+                                                                            mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+
+        with self.assertRaises(AirflowException) as e:
+            op = BigtableClusterUpdateOperator(
+                instance_id=INSTANCE_ID,
+                cluster_id=CLUSTER_ID,
+                nodes=NODES,
+                task_id="id"
+            )
+            op.execute(None)
+
+        err = e.exception
+        self.assertEqual(str(err), "Dependency: instance '{}' does not exist.".format(
+            INSTANCE_ID))
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.update_cluster.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_updating_cluster_that_does_not_exists(self, mock_hook):
+        instance = mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.update_cluster.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Cluster not found."))
+
+        with self.assertRaises(AirflowException) as e:
+            op = BigtableClusterUpdateOperator(
+                project_id=PROJECT_ID,
+                instance_id=INSTANCE_ID,
+                cluster_id=CLUSTER_ID,
+                nodes=NODES,
+                task_id="id"
+            )
+            op.execute(None)
+
+        err = e.exception
+        self.assertEqual(
+            str(err),
+            "Dependency: cluster '{}' does not exist for instance '{}'.".format(
+                CLUSTER_ID, INSTANCE_ID)
+        )
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.update_cluster.assert_called_once_with(
+            instance=instance, cluster_id=CLUSTER_ID, nodes=NODES)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_updating_cluster_that_does_not_exists_empty_project_id(self, mock_hook):
+        instance = mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.update_cluster.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Cluster not found."))
+
+        with self.assertRaises(AirflowException) as e:
+            op = BigtableClusterUpdateOperator(
+                instance_id=INSTANCE_ID,
+                cluster_id=CLUSTER_ID,
+                nodes=NODES,
+                task_id="id"
+            )
+            op.execute(None)
+
+        err = e.exception
+        self.assertEqual(
+            str(err),
+            "Dependency: cluster '{}' does not exist for instance '{}'.".format(
+                CLUSTER_ID, INSTANCE_ID)
+        )
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.update_cluster.assert_called_once_with(
+            instance=instance, cluster_id=CLUSTER_ID, nodes=NODES)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_different_error_reraised(self, mock_hook):
+        op = BigtableClusterUpdateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            cluster_id=CLUSTER_ID,
+            nodes=NODES,
+            task_id="id"
+        )
+        instance = mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.update_cluster.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.GoogleAPICallError('error'))
+
+        with self.assertRaises(google.api_core.exceptions.GoogleAPICallError):
+            op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.update_cluster.assert_called_once_with(
+            instance=instance, cluster_id=CLUSTER_ID, nodes=NODES)
+
+
+class BigtableInstanceDeleteTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_delete_execute(self, mock_hook):
+        op = BigtableInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_delete_execute_empty_project_id(self, mock_hook):
+        op = BigtableInstanceDeleteOperator(
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=None,
+            instance_id=INSTANCE_ID)
+
+    @parameterized.expand([
+        ('instance_id', PROJECT_ID, ''),
+    ], testcase_func_name=lambda f, n, p: 'test_empty_attribute.empty_' + p.args[0])
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_empty_attribute(self, missing_attribute, project_id, instance_id, mock_hook):
+        with self.assertRaises(AirflowException) as e:
+            BigtableInstanceDeleteOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                task_id="id"
+            )
+        err = e.exception
+        self.assertEqual(str(err), 'Empty parameter: {}'.format(missing_attribute))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_deleting_instance_that_doesnt_exists(self, mock_hook):
+        op = BigtableInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        mock_hook.return_value.delete_instance.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Instance not found."))
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_deleting_instance_that_doesnt_exists_empty_project_id(self, mock_hook):
+        op = BigtableInstanceDeleteOperator(
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        mock_hook.return_value.delete_instance.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Instance not found."))
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=None,
+            instance_id=INSTANCE_ID)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_different_error_reraised(self, mock_hook):
+        op = BigtableInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        mock_hook.return_value.delete_instance.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.GoogleAPICallError('error'))
+
+        with self.assertRaises(google.api_core.exceptions.GoogleAPICallError):
+            op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID)
+
+
+class BigtableTableDeleteTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_delete_execute(self, mock_hook):
+        op = BigtableTableDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_table.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID)
+
+    @parameterized.expand([
+        ('instance_id', PROJECT_ID, '', TABLE_ID),
+        ('table_id', PROJECT_ID, INSTANCE_ID, ''),
+    ], testcase_func_name=lambda f, n, p: 'test_empty_attribute.empty_' + p.args[0])
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_empty_attribute(self, missing_attribute, project_id, instance_id, table_id,
+                             mock_hook):
+        with self.assertRaises(AirflowException) as e:
+            BigtableTableDeleteOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                table_id=table_id,
+                task_id="id"
+            )
+        err = e.exception
+        self.assertEqual(str(err), 'Empty parameter: {}'.format(missing_attribute))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_deleting_table_that_doesnt_exists(self, mock_hook):
+        op = BigtableTableDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+
+        mock_hook.return_value.delete_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Table not found."))
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_table.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_deleting_table_that_doesnt_exists_empty_project_id(self, mock_hook):
+        op = BigtableTableDeleteOperator(
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+
+        mock_hook.return_value.delete_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Table not found."))
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_table.assert_called_once_with(
+            project_id=None,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_deleting_table_when_instance_doesnt_exists(self, mock_hook):
+        op = BigtableTableDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+
+        mock_hook.return_value.get_instance.return_value = None
+        with self.assertRaises(AirflowException) as e:
+            op.execute(None)
+        err = e.exception
+        self.assertEqual(str(err), "Dependency: instance '{}' does not exist.".format(
+            INSTANCE_ID))
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_table.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_different_error_reraised(self, mock_hook):
+        op = BigtableTableDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+        mock_hook.return_value.delete_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.GoogleAPICallError('error'))
+
+        with self.assertRaises(google.api_core.exceptions.GoogleAPICallError):
+            op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.delete_table.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID)
+
+
+class BigtableTableCreateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_create_execute(self, mock_hook):
+        op = BigtableTableCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES,
+            task_id="id"
+        )
+        instance = mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        op.execute(None)
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.create_table.assert_called_once_with(
+            instance=instance,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES)
+
+    @parameterized.expand([
+        ('instance_id', PROJECT_ID, '', TABLE_ID),
+        ('table_id', PROJECT_ID, INSTANCE_ID, ''),
+    ], testcase_func_name=lambda f, n, p: 'test_empty_attribute.empty_' + p.args[0])
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_empty_attribute(self, missing_attribute, project_id, instance_id, table_id,
+                             mock_hook):
+        with self.assertRaises(AirflowException) as e:
+            BigtableTableCreateOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                table_id=table_id,
+                task_id="id"
+            )
+        err = e.exception
+        self.assertEqual(str(err), 'Empty parameter: {}'.format(missing_attribute))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_instance_not_exists(self, mock_hook):
+        op = BigtableTableCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES,
+            task_id="id"
+        )
+        mock_hook.return_value.get_instance.return_value = None
+        with self.assertRaises(AirflowException) as e:
+            op.execute(None)
+        err = e.exception
+        self.assertEqual(
+            str(err),
+            "Dependency: instance '{}' does not exist in project '{}'.".format(
+                INSTANCE_ID, PROJECT_ID)
+        )
+        mock_hook.assert_called_once_with()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_creating_table_that_exists(self, mock_hook):
+        op = BigtableTableCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES,
+            task_id="id"
+        )
+
+        mock_hook.return_value.get_column_families_for_table.return_value = \
+            EMPTY_COLUMN_FAMILIES
+        instance = mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.create_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.AlreadyExists("Table already exists."))
+        op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.create_table.assert_called_once_with(
+            instance=instance,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_creating_table_that_exists_empty_project_id(self, mock_hook):
+        op = BigtableTableCreateOperator(
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES,
+            task_id="id"
+        )
+
+        mock_hook.return_value.get_column_families_for_table.return_value = \
+            EMPTY_COLUMN_FAMILIES
+        instance = mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.create_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.AlreadyExists("Table already exists."))
+        op.execute(None)
+
+        mock_hook.assert_called_once_with()
+        mock_hook.return_value.create_table.assert_called_once_with(
+            instance=instance,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES)
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_creating_table_that_exists_with_different_column_families_ids_in_the_table(
+            self, mock_hook):
+        op = BigtableTableCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families=EMPTY_COLUMN_FAMILIES,
+            task_id="id"
+        )
+
+        mock_hook.return_value.get_column_families_for_table.return_value = {
+            "existing_family": None}
+        mock_hook.return_value.create_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.AlreadyExists("Table already exists."))
+
+        with self.assertRaises(AirflowException) as e:
+            op.execute(None)
+        err = e.exception
+        self.assertEqual(
+            str(err),
+            "Table '{}' already exists with different Column Families.".format(TABLE_ID)
+        )
+        mock_hook.assert_called_once_with()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_creating_table_that_exists_with_different_column_families_gc_rule_in__table(
+            self, mock_hook):
+        op = BigtableTableCreateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            initial_split_keys=INITIAL_SPLIT_KEYS,
+            column_families={"cf-id": MaxVersionsGCRule(1)},
+            task_id="id"
+        )
+
+        cf_mock = mock.Mock()
+        cf_mock.gc_rule = mock.Mock(return_value=MaxVersionsGCRule(2))
+
+        mock_hook.return_value.get_column_families_for_table.return_value = {
+            "cf-id": cf_mock
+        }
+        mock_hook.return_value.create_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.AlreadyExists("Table already exists."))
+
+        with self.assertRaises(AirflowException) as e:
+            op.execute(None)
+        err = e.exception
+        self.assertEqual(
+            str(err),
+            "Table '{}' already exists with different Column Families.".format(TABLE_ID)
+        )
+        mock_hook.assert_called_once_with()
+
+
+class BigtableWaitForTableReplicationTest(unittest.TestCase):
+    @parameterized.expand([
+        ('instance_id', PROJECT_ID, '', TABLE_ID),
+        ('table_id', PROJECT_ID, INSTANCE_ID, ''),
+    ], testcase_func_name=lambda f, n, p: 'test_empty_attribute.empty_' + p.args[0])
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_empty_attribute(self, missing_attribute, project_id, instance_id, table_id,
+                             mock_hook):
+        with self.assertRaises(AirflowException) as e:
+            BigtableTableWaitForReplicationSensor(
+                project_id=project_id,
+                instance_id=instance_id,
+                table_id=table_id,
+                task_id="id"
+            )
+        err = e.exception
+        self.assertEqual(str(err), 'Empty parameter: {}'.format(missing_attribute))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_wait_no_instance(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+
+        op = BigtableTableWaitForReplicationSensor(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+        self.assertFalse(op.poke(None))
+        mock_hook.assert_called_once_with()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_wait_no_table(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.get_cluster_states_for_table.side_effect = mock.Mock(
+            side_effect=google.api_core.exceptions.NotFound("Table not found."))
+
+        op = BigtableTableWaitForReplicationSensor(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+        self.assertFalse(op.poke(None))
+        mock_hook.assert_called_once_with()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_wait_not_ready(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.get_cluster_states_for_table.return_value = {
+            "cl-id": ClusterState(0)
+        }
+        op = BigtableTableWaitForReplicationSensor(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+        self.assertFalse(op.poke(None))
+        mock_hook.assert_called_once_with()
+
+    @mock.patch('airflow.contrib.operators.gcp_bigtable_operator.BigtableHook')
+    def test_wait_ready(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = mock.Mock(Instance)
+        mock_hook.return_value.get_cluster_states_for_table.return_value = {
+            "cl-id": ClusterState(4)
+        }
+        op = BigtableTableWaitForReplicationSensor(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            table_id=TABLE_ID,
+            task_id="id"
+        )
+        self.assertTrue(op.poke(None))
+        mock_hook.assert_called_once_with()
diff --git a/tests/contrib/operators/test_gcp_bigtable_operator_system.py b/tests/contrib/operators/test_gcp_bigtable_operator_system.py
new file mode 100644
index 0000000000000..525da7776c550
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_bigtable_operator_system.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from tests.contrib.operators.test_gcp_bigtable_operator_system_helper import \
+    GCPBigtableTestHelper
+from tests.contrib.utils.base_gcp_system_test_case import \
+    SKIP_TEST_WARNING, DagGcpSystemTestCase
+from tests.contrib.utils.gcp_authenticator import GCP_BIGTABLE_KEY
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_BIGTABLE_KEY), SKIP_TEST_WARNING)
+class BigTableExampleDagsSystemTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(BigTableExampleDagsSystemTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_bigtable_operators',
+            require_local_executor=True,
+            gcp_key=GCP_BIGTABLE_KEY)
+        self.helper = GCPBigtableTestHelper()
+
+    def test_run_example_dag_gcs_bigtable(self):
+        self._run_dag()
+
+    def tearDown(self):
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            self.helper.delete_instance()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+        super(BigTableExampleDagsSystemTest, self).tearDown()
diff --git a/tests/contrib/operators/test_gcp_bigtable_operator_system_helper.py b/tests/contrib/operators/test_gcp_bigtable_operator_system_helper.py
new file mode 100755
index 0000000000000..68d579d3d82a3
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_bigtable_operator_system_helper.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+
+import argparse
+
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator, GCP_SPANNER_KEY
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+CBT_INSTANCE = os.environ.get('CBT_INSTANCE_ID', 'testinstance')
+
+
+class GCPBigtableTestHelper(LoggingCommandExecutor):
+
+    def delete_instance(self):
+        self.execute_cmd([
+            'gcloud', 'bigtable', '--project', GCP_PROJECT_ID,
+            '--quiet', '--verbosity=none',
+            'instances', 'delete', CBT_INSTANCE
+        ])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Create or delete spanner instances for system tests.')
+    parser.add_argument('--action', dest='action', required=True,
+                        choices=('delete-instance',
+                                 'before-tests', 'after-tests'))
+    action = parser.parse_args().action
+
+    helper = GCPBigtableTestHelper()
+    gcp_authenticator = GcpAuthenticator(GCP_SPANNER_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            pass
+        elif action == 'after-tests':
+            pass
+        elif action == 'delete-instance':
+            helper.delete_instance()
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcp_compute_operator.py b/tests/contrib/operators/test_gcp_compute_operator.py
new file mode 100644
index 0000000000000..7613c2a05752a
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_compute_operator.py
@@ -0,0 +1,1119 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import ast
+import unittest
+from copy import deepcopy
+
+import httplib2
+from googleapiclient.errors import HttpError
+
+from airflow import AirflowException, configuration
+from airflow.contrib.operators.gcp_compute_operator import GceInstanceStartOperator, \
+    GceInstanceStopOperator, GceSetMachineTypeOperator, GceInstanceTemplateCopyOperator, \
+    GceInstanceGroupManagerUpdateTemplateOperator
+from airflow.models import TaskInstance, DAG
+from airflow.utils import timezone
+from tests.compat import mock
+
+EMPTY_CONTENT = ''.encode('utf8')
+
+GCP_PROJECT_ID = 'project-id'
+GCE_ZONE = 'zone'
+RESOURCE_ID = 'resource-id'
+GCE_SHORT_MACHINE_TYPE_NAME = 'n1-machine-type'
+SET_MACHINE_TYPE_BODY = {
+    'machineType': 'zones/{}/machineTypes/{}'.format(GCE_ZONE, GCE_SHORT_MACHINE_TYPE_NAME)
+}
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+
+class GceInstanceStartTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_instance_start(self, mock_hook):
+        mock_hook.return_value.start_instance.return_value = True
+        op = GceInstanceStartOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            task_id='id'
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.start_instance.assert_called_once_with(
+            zone=GCE_ZONE, resource_id=RESOURCE_ID, project_id=GCP_PROJECT_ID
+        )
+        self.assertTrue(result)
+
+    # Setting all of the operator's input parameters as template dag_ids
+    # (could be anything else) just to test if the templating works for all fields
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_instance_start_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG(dag_id, default_args=args)
+        op = GceInstanceStartOperator(
+            project_id='{{ dag.dag_id }}',
+            zone='{{ dag.dag_id }}',
+            resource_id='{{ dag.dag_id }}',
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id='id',
+            dag=self.dag
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'project_id'))
+        self.assertEqual(dag_id, getattr(op, 'zone'))
+        self.assertEqual(dag_id, getattr(op, 'resource_id'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_start_should_throw_ex_when_missing_project_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceStartOperator(
+                project_id="",
+                zone=GCE_ZONE,
+                resource_id=RESOURCE_ID,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'project_id' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_start_should_not_throw_ex_when_project_id_None(self, _):
+        op = GceInstanceStartOperator(
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            task_id='id'
+        )
+        op.execute(None)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_start_should_throw_ex_when_missing_zone(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceStartOperator(
+                project_id=GCP_PROJECT_ID,
+                zone="",
+                resource_id=RESOURCE_ID,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'zone' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_start_should_throw_ex_when_missing_resource_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceStartOperator(
+                project_id=GCP_PROJECT_ID,
+                zone=GCE_ZONE,
+                resource_id="",
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'resource_id' is missing", str(err))
+        mock_hook.assert_not_called()
+
+
+class GceInstanceStopTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_instance_stop(self, mock_hook):
+        op = GceInstanceStopOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            task_id='id'
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.stop_instance.assert_called_once_with(
+            zone=GCE_ZONE, resource_id=RESOURCE_ID, project_id=GCP_PROJECT_ID
+        )
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all fields
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_instance_stop_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG(dag_id, default_args=args)
+        op = GceInstanceStopOperator(
+            project_id='{{ dag.dag_id }}',
+            zone='{{ dag.dag_id }}',
+            resource_id='{{ dag.dag_id }}',
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id='id',
+            dag=self.dag
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'project_id'))
+        self.assertEqual(dag_id, getattr(op, 'zone'))
+        self.assertEqual(dag_id, getattr(op, 'resource_id'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_stop_should_throw_ex_when_missing_project_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceStopOperator(
+                project_id="",
+                zone=GCE_ZONE,
+                resource_id=RESOURCE_ID,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'project_id' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_stop_should_not_throw_ex_when_project_id_none(self, mock_hook):
+        op = GceInstanceStopOperator(
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            task_id='id'
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.stop_instance.assert_called_once_with(
+            zone=GCE_ZONE, resource_id=RESOURCE_ID, project_id=None
+        )
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_stop_should_throw_ex_when_missing_zone(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceStopOperator(
+                project_id=GCP_PROJECT_ID,
+                zone="",
+                resource_id=RESOURCE_ID,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'zone' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_stop_should_throw_ex_when_missing_resource_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceStopOperator(
+                project_id=GCP_PROJECT_ID,
+                zone=GCE_ZONE,
+                resource_id="",
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'resource_id' is missing", str(err))
+        mock_hook.assert_not_called()
+
+
+class GceInstanceSetMachineTypeTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type(self, mock_hook):
+        mock_hook.return_value.set_machine_type.return_value = True
+        op = GceSetMachineTypeOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            body=SET_MACHINE_TYPE_BODY,
+            task_id='id'
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.set_machine_type.assert_called_once_with(
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            body=SET_MACHINE_TYPE_BODY,
+            project_id=GCP_PROJECT_ID
+        )
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all fields
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG(dag_id, default_args=args)
+        op = GceSetMachineTypeOperator(
+            project_id='{{ dag.dag_id }}',
+            zone='{{ dag.dag_id }}',
+            resource_id='{{ dag.dag_id }}',
+            body={},
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id='id',
+            dag=self.dag
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'project_id'))
+        self.assertEqual(dag_id, getattr(op, 'zone'))
+        self.assertEqual(dag_id, getattr(op, 'resource_id'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type_should_throw_ex_when_missing_project_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceSetMachineTypeOperator(
+                project_id="",
+                zone=GCE_ZONE,
+                resource_id=RESOURCE_ID,
+                body=SET_MACHINE_TYPE_BODY,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'project_id' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type_should_not_throw_ex_when_project_id_none(self, mock_hook):
+        op = GceSetMachineTypeOperator(
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            body=SET_MACHINE_TYPE_BODY,
+            task_id='id'
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.set_machine_type.assert_called_once_with(
+            zone=GCE_ZONE,
+            resource_id=RESOURCE_ID,
+            body=SET_MACHINE_TYPE_BODY,
+            project_id=None
+        )
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type_should_throw_ex_when_missing_zone(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceSetMachineTypeOperator(
+                project_id=GCP_PROJECT_ID,
+                zone="",
+                resource_id=RESOURCE_ID,
+                body=SET_MACHINE_TYPE_BODY,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'zone' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type_should_throw_ex_when_missing_resource_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceSetMachineTypeOperator(
+                project_id=GCP_PROJECT_ID,
+                zone=GCE_ZONE,
+                resource_id="",
+                body=SET_MACHINE_TYPE_BODY,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'resource_id' is missing", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_set_machine_type_should_throw_ex_when_missing_machine_type(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GceSetMachineTypeOperator(
+                project_id=GCP_PROJECT_ID,
+                zone=GCE_ZONE,
+                resource_id=RESOURCE_ID,
+                body={},
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn(
+            "The required body field 'machineType' is missing. Please add it.", str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+
+    MOCK_OP_RESPONSE = "{'kind': 'compute#operation', 'id': '8529919847974922736', " \
+                       "'name': " \
+                       "'operation-1538578207537-577542784f769-7999ab71-94f9ec1d', " \
+                       "'zone': 'https://www.googleapis.com/compute/v1/projects/example" \
+                       "-project/zones/europe-west3-b', 'operationType': " \
+                       "'setMachineType', 'targetLink': " \
+                       "'https://www.googleapis.com/compute/v1/projects/example-project" \
+                       "/zones/europe-west3-b/instances/pa-1', 'targetId': " \
+                       "'2480086944131075860', 'status': 'DONE', 'user': " \
+                       "'service-account@example-project.iam.gserviceaccount.com', " \
+                       "'progress': 100, 'insertTime': '2018-10-03T07:50:07.951-07:00', "\
+                       "'startTime': '2018-10-03T07:50:08.324-07:00', 'endTime': " \
+                       "'2018-10-03T07:50:08.484-07:00', 'error': {'errors': [{'code': " \
+                       "'UNSUPPORTED_OPERATION', 'message': \"Machine type with name " \
+                       "'machine-type-1' does not exist in zone 'europe-west3-b'.\"}]}, "\
+                       "'httpErrorStatusCode': 400, 'httpErrorMessage': 'BAD REQUEST', " \
+                       "'selfLink': " \
+                       "'https://www.googleapis.com/compute/v1/projects/example-project" \
+                       "/zones/europe-west3-b/operations/operation-1538578207537" \
+                       "-577542784f769-7999ab71-94f9ec1d'} "
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook'
+                '._check_zone_operation_status')
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook'
+                '._execute_set_machine_type')
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook.get_conn')
+    def test_set_machine_type_should_handle_and_trim_gce_error(
+            self, get_conn, _execute_set_machine_type, _check_zone_operation_status):
+        get_conn.return_value = {}
+        _execute_set_machine_type.return_value = {"name": "test-operation"}
+        _check_zone_operation_status.return_value = ast.literal_eval(
+            self.MOCK_OP_RESPONSE)
+        with self.assertRaises(AirflowException) as cm:
+            op = GceSetMachineTypeOperator(
+                project_id=GCP_PROJECT_ID,
+                zone=GCE_ZONE,
+                resource_id=RESOURCE_ID,
+                body=SET_MACHINE_TYPE_BODY,
+                task_id='id'
+            )
+            op.execute(None)
+        err = cm.exception
+        _check_zone_operation_status.assert_called_once_with(
+            {}, "test-operation", GCP_PROJECT_ID, GCE_ZONE)
+        _execute_set_machine_type.assert_called_once_with(
+            GCE_ZONE, RESOURCE_ID, SET_MACHINE_TYPE_BODY, GCP_PROJECT_ID)
+        # Checking the full message was sometimes failing due to different order
+        # of keys in the serialized JSON
+        self.assertIn("400 BAD REQUEST: {", str(err))  # checking the square bracket trim
+        self.assertIn("UNSUPPORTED_OPERATION", str(err))
+
+
+GCE_INSTANCE_TEMPLATE_NAME = "instance-template-test"
+GCE_INSTANCE_TEMPLATE_NEW_NAME = "instance-template-test-new"
+GCE_INSTANCE_TEMPLATE_REQUEST_ID = "e12d5b48-4826-4ba9-ada6-0cff1e0b36a6"
+
+GCE_INSTANCE_TEMPLATE_BODY_GET = {
+    "kind": "compute#instanceTemplate",
+    "id": "6950321349997439715",
+    "creationTimestamp": "2018-10-15T06:20:12.777-07:00",
+    "name": GCE_INSTANCE_TEMPLATE_NAME,
+    "description": "",
+    "properties": {
+        "machineType": "n1-standard-1",
+        "networkInterfaces": [
+            {
+                "kind": "compute#networkInterface",
+                "network": "https://www.googleapis.com/compute/v1/"
+                           "projects/project/global/networks/default",
+                "accessConfigs": [
+                    {
+                        "kind": "compute#accessConfig",
+                        "type": "ONE_TO_ONE_NAT",
+                    }
+                ]
+            },
+            {
+                "network": "https://www.googleapis.com/compute/v1/"
+                           "projects/project/global/networks/default",
+                "accessConfigs": [
+                    {
+                        "kind": "compute#accessConfig",
+                        "networkTier": "PREMIUM"
+                    }
+                ]
+            }
+        ],
+        "disks": [
+            {
+                "kind": "compute#attachedDisk",
+                "type": "PERSISTENT",
+                "licenses": [
+                    "A String",
+                ]
+            }
+        ],
+        "metadata": {
+            "kind": "compute#metadata",
+            "fingerprint": "GDPUYxlwHe4="
+        },
+    },
+    "selfLink": "https://www.googleapis.com/compute/v1/projects/project"
+                "/global/instanceTemplates/instance-template-test"
+}
+
+GCE_INSTANCE_TEMPLATE_BODY_INSERT = {
+    "name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
+    "description": "",
+    "properties": {
+        "machineType": "n1-standard-1",
+        "networkInterfaces": [
+            {
+                "network": "https://www.googleapis.com/compute/v1/"
+                           "projects/project/global/networks/default",
+                "accessConfigs": [
+                    {
+                        "type": "ONE_TO_ONE_NAT",
+                    }
+                ]
+            },
+            {
+                "network": "https://www.googleapis.com/compute/v1/"
+                           "projects/project/global/networks/default",
+                "accessConfigs": [
+                    {
+                        "networkTier": "PREMIUM"
+                    }
+                ]
+            }
+        ],
+        "disks": [
+            {
+                "type": "PERSISTENT",
+            }
+        ],
+        "metadata": {
+            "fingerprint": "GDPUYxlwHe4="
+        },
+    },
+}
+
+GCE_INSTANCE_TEMPLATE_BODY_GET_NEW = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_GET)
+GCE_INSTANCE_TEMPLATE_BODY_GET_NEW['name'] = GCE_INSTANCE_TEMPLATE_NEW_NAME
+
+
+class GceInstanceTemplateCopyTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={"name": GCE_INSTANCE_TEMPLATE_NEW_NAME}
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=GCE_INSTANCE_TEMPLATE_BODY_INSERT,
+            request_id=None
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={"name": GCE_INSTANCE_TEMPLATE_NEW_NAME}
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=None,
+            body=GCE_INSTANCE_TEMPLATE_BODY_INSERT,
+            request_id=None
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_idempotent_copy_template_when_already_copied(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={"name": GCE_INSTANCE_TEMPLATE_NEW_NAME}
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.insert_instance_template.assert_not_called()
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template_with_request_id(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            request_id=GCE_INSTANCE_TEMPLATE_REQUEST_ID,
+            task_id='id',
+            body_patch={"name": GCE_INSTANCE_TEMPLATE_NEW_NAME}
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=GCE_INSTANCE_TEMPLATE_BODY_INSERT,
+            request_id=GCE_INSTANCE_TEMPLATE_REQUEST_ID,
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template_with_description_fields(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            request_id=GCE_INSTANCE_TEMPLATE_REQUEST_ID,
+            task_id='id',
+            body_patch={"name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
+                        "description": "New description"}
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+
+        body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
+        body_insert["description"] = "New description"
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=body_insert,
+            request_id=GCE_INSTANCE_TEMPLATE_REQUEST_ID,
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_copy_with_some_validation_warnings(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={"name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
+                        "some_wrong_field": "test",
+                        "properties": {
+                            "some_other_wrong_field": "test"
+                        }}
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
+        body_insert["some_wrong_field"] = "test"
+        body_insert["properties"]["some_other_wrong_field"] = "test"
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=body_insert,
+            request_id=None,
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template_with_updated_nested_fields(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={
+                "name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
+                "properties": {
+                    "machineType": "n1-standard-2",
+                }
+            }
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
+        body_insert["properties"]["machineType"] = "n1-standard-2"
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=body_insert,
+            request_id=None
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template_with_smaller_array_fields(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={
+                "name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
+                "properties": {
+                    "machineType": "n1-standard-1",
+                    "networkInterfaces": [
+                        {
+                            "network": "https://www.googleapis.com/compute/v1/"
+                                       "projects/project/global/networks/default",
+                            "accessConfigs": [
+                                {
+                                    "type": "ONE_TO_ONE_NAT",
+                                    "natIP": "8.8.8.8"
+                                }
+                            ]
+                        }
+                    ]
+                }
+            }
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
+        body_insert["properties"]["networkInterfaces"] = [
+            {
+                "network": "https://www.googleapis.com/compute/v1/"
+                           "projects/project/global/networks/default",
+                "accessConfigs": [
+                    {
+                        "type": "ONE_TO_ONE_NAT",
+                        "natIP": "8.8.8.8"
+                    }
+                ]
+            }
+        ]
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=body_insert,
+            request_id=None
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_copy_template_with_bigger_array_fields(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        op = GceInstanceTemplateCopyOperator(
+            project_id=GCP_PROJECT_ID,
+            resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+            task_id='id',
+            body_patch={
+                "name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
+                "properties": {
+                    "disks": [
+                        {
+                            "kind": "compute#attachedDisk",
+                            "type": "SCRATCH",
+                            "licenses": [
+                                "Updated String",
+                            ]
+                        },
+                        {
+                            "kind": "compute#attachedDisk",
+                            "type": "PERSISTENT",
+                            "licenses": [
+                                "Another String",
+                            ]
+                        }
+                    ],
+                }
+            }
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+
+        body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
+        body_insert["properties"]["disks"] = [
+            {
+                "kind": "compute#attachedDisk",
+                "type": "SCRATCH",
+                "licenses": [
+                    "Updated String",
+                ]
+            },
+            {
+                "kind": "compute#attachedDisk",
+                "type": "PERSISTENT",
+                "licenses": [
+                    "Another String",
+                ]
+            }
+        ]
+        mock_hook.return_value.insert_instance_template.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            body=body_insert,
+            request_id=None,
+        )
+        self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_missing_name(self, mock_hook):
+        mock_hook.return_value.get_instance_template.side_effect = [
+            HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
+            GCE_INSTANCE_TEMPLATE_BODY_GET,
+            GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
+        ]
+        with self.assertRaises(AirflowException) as cm:
+            op = GceInstanceTemplateCopyOperator(
+                project_id=GCP_PROJECT_ID,
+                resource_id=GCE_INSTANCE_TEMPLATE_NAME,
+                request_id=GCE_INSTANCE_TEMPLATE_REQUEST_ID,
+                task_id='id',
+                body_patch={"description": "New description"}
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("should contain at least name for the new operator "
+                      "in the 'name' field", str(err))
+        mock_hook.assert_not_called()
+
+
+GCE_INSTANCE_GROUP_MANAGER_NAME = "instance-group-test"
+GCE_INSTANCE_TEMPLATE_SOURCE_URL = \
+    "https://www.googleapis.com/compute/beta/projects/project" \
+    "/global/instanceTemplates/instance-template-test"
+
+GCE_INSTANCE_TEMPLATE_OTHER_URL = \
+    "https://www.googleapis.com/compute/beta/projects/project" \
+    "/global/instanceTemplates/instance-template-other"
+
+GCE_INSTANCE_TEMPLATE_NON_EXISTING_URL = \
+    "https://www.googleapis.com/compute/beta/projects/project" \
+    "/global/instanceTemplates/instance-template-non-existing"
+
+GCE_INSTANCE_TEMPLATE_DESTINATION_URL = \
+    "https://www.googleapis.com/compute/beta/projects/project" \
+    "/global/instanceTemplates/instance-template-new"
+
+GCE_INSTANCE_GROUP_MANAGER_GET = {
+    "kind": "compute#instanceGroupManager",
+    "id": "2822359583810032488",
+    "creationTimestamp": "2018-10-17T05:39:35.793-07:00",
+    "name": GCE_INSTANCE_GROUP_MANAGER_NAME,
+    "zone": "https://www.googleapis.com/compute/beta/projects/project/zones/zone",
+    "instanceTemplate": GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+    "versions": [
+        {
+            "name": "v1",
+            "instanceTemplate": GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            "targetSize": {
+                "calculated": 1
+            }
+        },
+        {
+            "name": "v2",
+            "instanceTemplate": GCE_INSTANCE_TEMPLATE_OTHER_URL,
+        }
+    ],
+    "instanceGroup": GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+    "baseInstanceName": GCE_INSTANCE_GROUP_MANAGER_NAME,
+    "fingerprint": "BKWB_igCNbQ=",
+    "currentActions": {
+        "none": 1,
+        "creating": 0,
+        "creatingWithoutRetries": 0,
+        "verifying": 0,
+        "recreating": 0,
+        "deleting": 0,
+        "abandoning": 0,
+        "restarting": 0,
+        "refreshing": 0
+    },
+    "pendingActions": {
+        "creating": 0,
+        "deleting": 0,
+        "recreating": 0,
+        "restarting": 0
+    },
+    "targetSize": 1,
+    "selfLink": "https://www.googleapis.com/compute/beta/projects/project/zones/"
+                "zone/instanceGroupManagers/" + GCE_INSTANCE_GROUP_MANAGER_NAME,
+    "autoHealingPolicies": [
+        {
+            "initialDelaySec": 300
+        }
+    ],
+    "serviceAccount": "198907790164@cloudservices.gserviceaccount.com"
+}
+
+GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH = {
+    "instanceTemplate": GCE_INSTANCE_TEMPLATE_DESTINATION_URL,
+    "versions": [
+        {
+            "name": "v1",
+            "instanceTemplate": GCE_INSTANCE_TEMPLATE_DESTINATION_URL,
+            "targetSize": {
+                "calculated": 1
+            }
+        },
+        {
+            "name": "v2",
+            "instanceTemplate": GCE_INSTANCE_TEMPLATE_OTHER_URL,
+        }
+    ],
+}
+
+GCE_INSTANCE_GROUP_MANAGER_REQUEST_ID = "e12d5b48-4826-4ba9-ada6-0cff1e0b36a6"
+
+GCE_INSTANCE_GROUP_MANAGER_UPDATE_POLICY = {
+    "type": "OPPORTUNISTIC",
+    "minimalAction": "RESTART",
+    "maxSurge": {
+        "fixed": 1
+    },
+    "maxUnavailable": {
+        "percent": 10
+    },
+    "minReadySec": 1800
+}
+
+
+class GceInstanceGroupManagerUpdateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_instance_group_update(self, mock_hook):
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.patch_instance_group_manager.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            body=GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH,
+            request_id=None
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_instance_group_update_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.patch_instance_group_manager.assert_called_once_with(
+            project_id=None,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            body=GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH,
+            request_id=None
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_instance_group_update_no_instance_template_field(self, mock_hook):
+        instance_group_manager_no_template = deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        del instance_group_manager_no_template['instanceTemplate']
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            instance_group_manager_no_template
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        expected_patch_no_instance_template = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH)
+        del expected_patch_no_instance_template['instanceTemplate']
+        mock_hook.return_value.patch_instance_group_manager.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            body=expected_patch_no_instance_template,
+            request_id=None
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_instance_group_update_no_versions_field(self, mock_hook):
+        instance_group_manager_no_versions = deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        del instance_group_manager_no_versions['versions']
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            instance_group_manager_no_versions
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        expected_patch_no_versions = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH)
+        del expected_patch_no_versions['versions']
+        mock_hook.return_value.patch_instance_group_manager.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            body=expected_patch_no_versions,
+            request_id=None
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_instance_group_update_with_update_policy(self, mock_hook):
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            update_policy=GCE_INSTANCE_GROUP_MANAGER_UPDATE_POLICY,
+            source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        expected_patch_with_update_policy = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH)
+        expected_patch_with_update_policy['updatePolicy'] = GCE_INSTANCE_GROUP_MANAGER_UPDATE_POLICY
+        mock_hook.return_value.patch_instance_group_manager.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            body=expected_patch_with_update_policy,
+            request_id=None
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_successful_instance_group_update_with_request_id(self, mock_hook):
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+            request_id=GCE_INSTANCE_GROUP_MANAGER_REQUEST_ID,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.patch_instance_group_manager.assert_called_once_with(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            body=GCE_INSTANCE_GROUP_MANAGER_EXPECTED_PATCH,
+            request_id=GCE_INSTANCE_GROUP_MANAGER_REQUEST_ID
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_try_to_use_api_v1(self, _):
+        with self.assertRaises(AirflowException) as cm:
+            GceInstanceGroupManagerUpdateTemplateOperator(
+                project_id=GCP_PROJECT_ID,
+                zone=GCE_ZONE,
+                resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+                task_id='id',
+                api_version='v1',
+                source_template=GCE_INSTANCE_TEMPLATE_SOURCE_URL,
+                destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+            )
+        err = cm.exception
+        self.assertIn("Use beta api version or above", str(err))
+
+    @mock.patch('airflow.contrib.operators.gcp_compute_operator.GceHook')
+    def test_try_to_use_non_existing_template(self, mock_hook):
+        mock_hook.return_value.get_instance_group_manager.return_value = \
+            deepcopy(GCE_INSTANCE_GROUP_MANAGER_GET)
+        op = GceInstanceGroupManagerUpdateTemplateOperator(
+            project_id=GCP_PROJECT_ID,
+            zone=GCE_ZONE,
+            resource_id=GCE_INSTANCE_GROUP_MANAGER_NAME,
+            task_id='id',
+            source_template=GCE_INSTANCE_TEMPLATE_NON_EXISTING_URL,
+            destination_template=GCE_INSTANCE_TEMPLATE_DESTINATION_URL
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='beta',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.patch_instance_group_manager.assert_not_called()
+        self.assertTrue(result)
diff --git a/tests/contrib/operators/test_gcp_compute_operator_system.py b/tests/contrib/operators/test_gcp_compute_operator_system.py
new file mode 100644
index 0000000000000..0054df99d7dbb
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_compute_operator_system.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from tests.contrib.utils.base_gcp_system_test_case import \
+    SKIP_TEST_WARNING, DagGcpSystemTestCase
+
+from tests.contrib.operators.test_gcp_compute_operator_system_helper import \
+    GCPComputeTestHelper
+from tests.contrib.utils.gcp_authenticator import GCP_COMPUTE_KEY
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_COMPUTE_KEY), SKIP_TEST_WARNING)
+class GcpComputeExampleDagsSystemTest(DagGcpSystemTestCase):
+
+    def setUp(self):
+        super(GcpComputeExampleDagsSystemTest, self).setUp()
+        self.gcp_authenticator.gcp_authenticate()
+        self.helper.delete_instance()
+        self.helper.create_instance()
+        self.gcp_authenticator.gcp_revoke_authentication()
+
+    def tearDown(self):
+        self.gcp_authenticator.gcp_authenticate()
+        self.helper.delete_instance()
+        self.gcp_authenticator.gcp_revoke_authentication()
+        super(GcpComputeExampleDagsSystemTest, self).tearDown()
+
+    def __init__(self, method_name='runTest'):
+        super(GcpComputeExampleDagsSystemTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_compute',
+            gcp_key=GCP_COMPUTE_KEY)
+        self.helper = GCPComputeTestHelper()
+
+    def test_run_example_dag_compute(self):
+        self._run_dag()
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_COMPUTE_KEY), SKIP_TEST_WARNING)
+class GcpComputeIgmExampleDagsSystemTest(DagGcpSystemTestCase):
+
+    def setUp(self):
+        super(GcpComputeIgmExampleDagsSystemTest, self).setUp()
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            self.helper.delete_instance_group_and_template(silent=True)
+            self.helper.create_instance_group_and_template()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+
+    def tearDown(self):
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            self.helper.delete_instance_group_and_template()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+        super(GcpComputeIgmExampleDagsSystemTest, self).tearDown()
+
+    def __init__(self, method_name='runTest'):
+        super(GcpComputeIgmExampleDagsSystemTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_compute_igm',
+            gcp_key=GCP_COMPUTE_KEY)
+        self.helper = GCPComputeTestHelper()
+
+    def test_run_example_dag_compute_igm(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_compute_operator_system_helper.py b/tests/contrib/operators/test_gcp_compute_operator_system_helper.py
new file mode 100755
index 0000000000000..60de2381d196c
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_compute_operator_system_helper.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import argparse
+import os
+
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator, GCP_COMPUTE_KEY
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+
+GCE_INSTANCE = os.environ.get('GCE_INSTANCE', 'testinstance')
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCE_INSTANCE_GROUP_MANAGER_NAME = os.environ.get('GCE_INSTANCE_GROUP_MANAGER_NAME',
+                                                 'instance-group-test')
+GCE_ZONE = os.environ.get('GCE_ZONE', 'europe-west1-b')
+GCE_TEMPLATE_NAME = os.environ.get('GCE_TEMPLATE_NAME',
+                                   'instance-template-test')
+GCE_NEW_TEMPLATE_NAME = os.environ.get('GCE_NEW_TEMPLATE_NAME',
+                                       'instance-template-test-new')
+
+
+class GCPComputeTestHelper(LoggingCommandExecutor):
+
+    def delete_instance(self):
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID,
+            '--quiet', '--verbosity=none',
+            'instances', 'delete', GCE_INSTANCE, '--zone', GCE_ZONE,
+        ])
+
+    def create_instance(self):
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID, '--quiet',
+            'instances', 'create', GCE_INSTANCE,
+            '--zone', GCE_ZONE
+        ])
+
+    def delete_instance_group_and_template(self, silent=False):
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID,
+            '--quiet', '--verbosity=none',
+            'instance-groups', 'managed', 'delete', GCE_INSTANCE_GROUP_MANAGER_NAME,
+            '--zone', GCE_ZONE
+        ], silent=silent)
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID,
+            '--quiet', '--verbosity=none',
+            'instance-templates', 'delete', GCE_NEW_TEMPLATE_NAME
+        ], silent=silent)
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute',
+            '--project', GCP_PROJECT_ID,
+            '--quiet', '--verbosity=none',
+            'instance-templates', 'delete', GCE_TEMPLATE_NAME
+        ], silent=silent)
+
+    def create_instance_group_and_template(self):
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID, '--quiet',
+            'instance-templates', 'create', GCE_TEMPLATE_NAME
+        ])
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID, '--quiet',
+            'instance-groups', 'managed', 'create', GCE_INSTANCE_GROUP_MANAGER_NAME,
+            '--template', GCE_TEMPLATE_NAME,
+            '--zone', GCE_ZONE, '--size=1'
+        ])
+        self.execute_cmd([
+            'gcloud', 'beta', 'compute', '--project', GCP_PROJECT_ID, '--quiet',
+            'instance-groups', 'managed', 'wait-until-stable',
+            GCE_INSTANCE_GROUP_MANAGER_NAME,
+            '--zone', GCE_ZONE
+        ])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Create or delete GCE instances/instance groups for system tests.')
+    parser.add_argument('--action', dest='action', required=True,
+                        choices=('create-instance', 'delete-instance',
+                                 'create-instance-group', 'delete-instance-group',
+                                 'before-tests', 'after-tests'))
+    action = parser.parse_args().action
+
+    helper = GCPComputeTestHelper()
+    gcp_authenticator = GcpAuthenticator(GCP_COMPUTE_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            pass
+        elif action == 'after-tests':
+            pass
+        elif action == 'create-instance':
+            helper.create_instance()
+        elif action == 'delete-instance':
+            helper.delete_instance()
+        elif action == 'create-instance-group':
+            helper.create_instance_group_and_template()
+        elif action == 'delete-instance-group':
+            helper.delete_instance_group_and_template()
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcp_container_operator.py b/tests/contrib/operators/test_gcp_container_operator.py
index 0f67290cbf350..8fe322dbbb947 100644
--- a/tests/contrib/operators/test_gcp_container_operator.py
+++ b/tests/contrib/operators/test_gcp_container_operator.py
@@ -17,21 +17,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 import unittest
 
 from airflow import AirflowException
 from airflow.contrib.operators.gcp_container_operator import GKEClusterCreateOperator, \
-    GKEClusterDeleteOperator
+    GKEClusterDeleteOperator, GKEPodOperator
+from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
+from tests.compat import mock
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
-PROJECT_ID = 'test-id'
+TEST_GCP_PROJECT_ID = 'test-id'
 PROJECT_LOCATION = 'test-location'
 PROJECT_TASK_ID = 'test-task-id'
 CLUSTER_NAME = 'test-cluster-name'
@@ -39,12 +34,21 @@
 PROJECT_BODY = {'name': 'test-name'}
 PROJECT_BODY_CREATE = {'name': 'test-name', 'initial_node_count': 1}
 
+TASK_NAME = 'test-task-name'
+NAMESPACE = 'default',
+IMAGE = 'bash'
+
+GCLOUD_COMMAND = "gcloud container clusters get-credentials {} --zone {} --project {}"
+KUBE_ENV_VAR = 'KUBECONFIG'
+GAC_ENV_VAR = 'GOOGLE_APPLICATION_CREDENTIALS'
+FILE_NAME = '/tmp/mock_name'
+
 
 class GoogleCloudPlatformContainerOperatorTest(unittest.TestCase):
 
     @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEClusterHook')
     def test_create_execute(self, mock_hook):
-        operator = GKEClusterCreateOperator(project_id=PROJECT_ID,
+        operator = GKEClusterCreateOperator(project_id=TEST_GCP_PROJECT_ID,
                                             location=PROJECT_LOCATION,
                                             body=PROJECT_BODY_CREATE,
                                             task_id=PROJECT_TASK_ID)
@@ -56,7 +60,7 @@ def test_create_execute(self, mock_hook):
     @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEClusterHook')
     def test_create_execute_error_body(self, mock_hook):
         with self.assertRaises(AirflowException):
-            operator = GKEClusterCreateOperator(project_id=PROJECT_ID,
+            operator = GKEClusterCreateOperator(project_id=TEST_GCP_PROJECT_ID,
                                                 location=PROJECT_LOCATION,
                                                 body=None,
                                                 task_id=PROJECT_TASK_ID)
@@ -77,7 +81,7 @@ def test_create_execute_error_project_id(self, mock_hook):
     @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEClusterHook')
     def test_create_execute_error_location(self, mock_hook):
         with self.assertRaises(AirflowException):
-            operator = GKEClusterCreateOperator(project_id=PROJECT_ID,
+            operator = GKEClusterCreateOperator(project_id=TEST_GCP_PROJECT_ID,
                                                 body=PROJECT_BODY,
                                                 task_id=PROJECT_TASK_ID)
 
@@ -86,7 +90,7 @@ def test_create_execute_error_location(self, mock_hook):
 
     @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEClusterHook')
     def test_delete_execute(self, mock_hook):
-        operator = GKEClusterDeleteOperator(project_id=PROJECT_ID,
+        operator = GKEClusterDeleteOperator(project_id=TEST_GCP_PROJECT_ID,
                                             name=CLUSTER_NAME,
                                             location=PROJECT_LOCATION,
                                             task_id=PROJECT_TASK_ID)
@@ -107,7 +111,7 @@ def test_delete_execute_error_project_id(self, mock_hook):
     @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEClusterHook')
     def test_delete_execute_error_cluster_name(self, mock_hook):
         with self.assertRaises(AirflowException):
-            operator = GKEClusterDeleteOperator(project_id=PROJECT_ID,
+            operator = GKEClusterDeleteOperator(project_id=TEST_GCP_PROJECT_ID,
                                                 location=PROJECT_LOCATION,
                                                 task_id=PROJECT_TASK_ID)
 
@@ -117,9 +121,170 @@ def test_delete_execute_error_cluster_name(self, mock_hook):
     @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEClusterHook')
     def test_delete_execute_error_location(self, mock_hook):
         with self.assertRaises(AirflowException):
-            operator = GKEClusterDeleteOperator(project_id=PROJECT_ID,
+            operator = GKEClusterDeleteOperator(project_id=TEST_GCP_PROJECT_ID,
                                                 name=CLUSTER_NAME,
                                                 task_id=PROJECT_TASK_ID)
 
             operator.execute(None)
             mock_hook.return_value.delete_cluster.assert_not_called()
+
+
+class GKEPodOperatorTest(unittest.TestCase):
+    def setUp(self):
+        self.gke_op = GKEPodOperator(project_id=TEST_GCP_PROJECT_ID,
+                                     location=PROJECT_LOCATION,
+                                     cluster_name=CLUSTER_NAME,
+                                     task_id=PROJECT_TASK_ID,
+                                     name=TASK_NAME,
+                                     namespace=NAMESPACE,
+                                     image=IMAGE)
+        if GAC_ENV_VAR in os.environ:
+            del os.environ[GAC_ENV_VAR]
+
+    def test_template_fields(self):
+        self.assertTrue(set(KubernetesPodOperator.template_fields).issubset(
+            GKEPodOperator.template_fields))
+
+    @mock.patch(
+        'airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator.execute')
+    @mock.patch('tempfile.NamedTemporaryFile')
+    @mock.patch("subprocess.check_call")
+    def test_execute_conn_id_none(self, proc_mock, file_mock, exec_mock):
+        self.gke_op.gcp_conn_id = None
+
+        file_mock.return_value.__enter__.return_value.name = FILE_NAME
+
+        self.gke_op.execute(None)
+
+        # Assert Environment Variable is being set correctly
+        self.assertIn(KUBE_ENV_VAR, os.environ)
+        self.assertEqual(os.environ[KUBE_ENV_VAR], FILE_NAME)
+
+        # Assert the gcloud command being called correctly
+        proc_mock.assert_called_with(
+            GCLOUD_COMMAND.format(CLUSTER_NAME, PROJECT_LOCATION, TEST_GCP_PROJECT_ID).split())
+
+        self.assertEqual(self.gke_op.config_file, FILE_NAME)
+
+    @mock.patch('airflow.hooks.base_hook.BaseHook.get_connection')
+    @mock.patch(
+        'airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator.execute')
+    @mock.patch('tempfile.NamedTemporaryFile')
+    @mock.patch("subprocess.check_call")
+    @mock.patch.dict(os.environ, {})
+    def test_execute_conn_id_path(self, proc_mock, file_mock, exec_mock, get_con_mock):
+        # gcp_conn_id is defaulted to `google_cloud_default`
+
+        FILE_PATH = '/path/to/file'
+        KEYFILE_DICT = {"extra__google_cloud_platform__key_path": FILE_PATH}
+        get_con_mock.return_value.extra_dejson = KEYFILE_DICT
+        file_mock.return_value.__enter__.return_value.name = FILE_NAME
+
+        self.gke_op.execute(None)
+
+        # Assert Environment Variable is being set correctly
+        self.assertIn(KUBE_ENV_VAR, os.environ)
+        self.assertEqual(os.environ[KUBE_ENV_VAR], FILE_NAME)
+
+        self.assertIn(GAC_ENV_VAR, os.environ)
+        # since we passed in keyfile_path we should get a file
+        self.assertEqual(os.environ[GAC_ENV_VAR], FILE_PATH)
+
+        # Assert the gcloud command being called correctly
+        proc_mock.assert_called_with(
+            GCLOUD_COMMAND.format(CLUSTER_NAME, PROJECT_LOCATION, TEST_GCP_PROJECT_ID).split())
+
+        self.assertEqual(self.gke_op.config_file, FILE_NAME)
+
+    @mock.patch.dict(os.environ, {})
+    @mock.patch('airflow.hooks.base_hook.BaseHook.get_connection')
+    @mock.patch(
+        'airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator.execute')
+    @mock.patch('tempfile.NamedTemporaryFile')
+    @mock.patch("subprocess.check_call")
+    def test_execute_conn_id_dict(self, proc_mock, file_mock, exec_mock, get_con_mock):
+        # gcp_conn_id is defaulted to `google_cloud_default`
+        FILE_PATH = '/path/to/file'
+
+        # This is used in the _set_env_from_extras method
+        file_mock.return_value.name = FILE_PATH
+        # This is used in the execute method
+        file_mock.return_value.__enter__.return_value.name = FILE_NAME
+
+        KEYFILE_DICT = {"extra__google_cloud_platform__keyfile_dict":
+                        '{"private_key": "r4nd0m_k3y"}'}
+        get_con_mock.return_value.extra_dejson = KEYFILE_DICT
+
+        self.gke_op.execute(None)
+
+        # Assert Environment Variable is being set correctly
+        self.assertIn(KUBE_ENV_VAR, os.environ)
+        self.assertEqual(os.environ[KUBE_ENV_VAR], FILE_NAME)
+
+        self.assertIn(GAC_ENV_VAR, os.environ)
+        # since we passed in keyfile_path we should get a file
+        self.assertEqual(os.environ[GAC_ENV_VAR], FILE_PATH)
+
+        # Assert the gcloud command being called correctly
+        proc_mock.assert_called_with(
+            GCLOUD_COMMAND.format(CLUSTER_NAME, PROJECT_LOCATION, TEST_GCP_PROJECT_ID).split())
+
+        self.assertEqual(self.gke_op.config_file, FILE_NAME)
+
+    @mock.patch.dict(os.environ, {})
+    def test_set_env_from_extras_none(self):
+        extras = {}
+        self.gke_op._set_env_from_extras(extras)
+        # _set_env_from_extras should not edit os.environ if extras does not specify
+        self.assertNotIn(GAC_ENV_VAR, os.environ)
+
+    @mock.patch.dict(os.environ, {})
+    @mock.patch('tempfile.NamedTemporaryFile')
+    def test_set_env_from_extras_dict(self, file_mock):
+        file_mock.return_value.name = FILE_NAME
+
+        KEYFILE_DICT_STR = '{ \"test\": \"cluster\" }'
+        extras = {
+            'extra__google_cloud_platform__keyfile_dict': KEYFILE_DICT_STR,
+        }
+
+        self.gke_op._set_env_from_extras(extras)
+        self.assertEqual(os.environ[GAC_ENV_VAR], FILE_NAME)
+
+        file_mock.return_value.write.assert_called_once_with(KEYFILE_DICT_STR)
+
+    @mock.patch.dict(os.environ, {})
+    def test_set_env_from_extras_path(self):
+        TEST_PATH = '/test/path'
+
+        extras = {
+            'extra__google_cloud_platform__key_path': TEST_PATH,
+        }
+
+        self.gke_op._set_env_from_extras(extras)
+        self.assertEqual(os.environ[GAC_ENV_VAR], TEST_PATH)
+
+    def test_get_field(self):
+        FIELD_NAME = 'test_field'
+        FIELD_VALUE = 'test_field_value'
+        extras = {
+            'extra__google_cloud_platform__{}'.format(FIELD_NAME):
+                FIELD_VALUE
+        }
+
+        ret_val = self.gke_op._get_field(extras, FIELD_NAME)
+        self.assertEqual(FIELD_VALUE, ret_val)
+
+    @mock.patch('airflow.contrib.operators.gcp_container_operator.GKEPodOperator.log')
+    def test_get_field_fail(self, log_mock):
+        log_mock.info = mock.Mock()
+        LOG_STR = 'Field %s not found in extras.'
+        FIELD_NAME = 'test_field'
+        FIELD_VALUE = 'test_field_value'
+
+        extras = {}
+
+        ret_val = self.gke_op._get_field(extras, FIELD_NAME, default=FIELD_VALUE)
+        # Assert default is returned upon failure
+        self.assertEqual(FIELD_VALUE, ret_val)
+        log_mock.info.assert_called_with(LOG_STR, FIELD_NAME)
diff --git a/tests/contrib/operators/test_gcp_function_operator.py b/tests/contrib/operators/test_gcp_function_operator.py
new file mode 100644
index 0000000000000..c1abff0a3e889
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_function_operator.py
@@ -0,0 +1,648 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from googleapiclient.errors import HttpError
+from parameterized import parameterized
+
+from airflow.contrib.operators.gcp_function_operator import \
+    GcfFunctionDeployOperator, GcfFunctionDeleteOperator, FUNCTION_NAME_PATTERN
+from airflow import AirflowException
+from airflow.version import version
+from tests.compat import mock
+
+from copy import deepcopy
+
+EMPTY_CONTENT = ''.encode('utf8')
+MOCK_RESP_404 = type('', (object,), {"status": 404})()
+
+GCP_PROJECT_ID = 'test_project_id'
+GCP_LOCATION = 'test_region'
+GCF_SOURCE_ARCHIVE_URL = 'gs://folder/file.zip'
+GCF_ENTRYPOINT = 'helloWorld'
+FUNCTION_NAME = 'projects/{}/locations/{}/functions/{}'.format(GCP_PROJECT_ID,
+                                                               GCP_LOCATION,
+                                                               GCF_ENTRYPOINT)
+GCF_RUNTIME = 'nodejs6'
+VALID_RUNTIMES = ['nodejs6', 'nodejs8', 'python37']
+VALID_BODY = {
+    "name": FUNCTION_NAME,
+    "entryPoint": GCF_ENTRYPOINT,
+    "runtime": GCF_RUNTIME,
+    "httpsTrigger": {},
+    "sourceArchiveUrl": GCF_SOURCE_ARCHIVE_URL
+}
+
+
+def _prepare_test_bodies():
+    body_no_name = deepcopy(VALID_BODY)
+    body_no_name.pop('name', None)
+    body_empty_entry_point = deepcopy(VALID_BODY)
+    body_empty_entry_point['entryPoint'] = ''
+    body_empty_runtime = deepcopy(VALID_BODY)
+    body_empty_runtime['runtime'] = ''
+    body_values = [
+        ({}, "The required parameter 'body' is missing"),
+        (body_no_name, "The required body field 'name' is missing"),
+        (body_empty_entry_point,
+         "The body field 'entryPoint' of value '' does not match"),
+        (body_empty_runtime, "The body field 'runtime' of value '' does not match"),
+    ]
+    return body_values
+
+
+class GcfFunctionDeployTest(unittest.TestCase):
+    @parameterized.expand(_prepare_test_bodies())
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_body_empty_or_missing_fields(self, body, message, mock_hook):
+        mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
+        with self.assertRaises(AirflowException) as cm:
+            op = GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=body,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn(message, str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_deploy_execute(self, mock_hook):
+        mock_hook.return_value.get_function.side_effect = mock.Mock(
+            side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
+        mock_hook.return_value.create_new_function.return_value = True
+        op = GcfFunctionDeployOperator(
+            project_id=GCP_PROJECT_ID,
+            location=GCP_LOCATION,
+            body=deepcopy(VALID_BODY),
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.get_function.assert_called_once_with(
+            'projects/test_project_id/locations/test_region/functions/helloWorld'
+        )
+        expected_body = deepcopy(VALID_BODY)
+        expected_body['labels'] = {
+            'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')
+        }
+        mock_hook.return_value.create_new_function.assert_called_once_with(
+            project_id='test_project_id',
+            location='test_region',
+            body=expected_body
+        )
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_update_function_if_exists(self, mock_hook):
+        mock_hook.return_value.get_function.return_value = True
+        mock_hook.return_value.update_function.return_value = True
+        op = GcfFunctionDeployOperator(
+            project_id=GCP_PROJECT_ID,
+            location=GCP_LOCATION,
+            body=deepcopy(VALID_BODY),
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.get_function.assert_called_once_with(
+            'projects/test_project_id/locations/test_region/functions/helloWorld'
+        )
+        expected_body = deepcopy(VALID_BODY)
+        expected_body['labels'] = {
+            'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')
+        }
+        mock_hook.return_value.update_function.assert_called_once_with(
+            'projects/test_project_id/locations/test_region/functions/helloWorld',
+            expected_body, expected_body.keys())
+        mock_hook.return_value.create_new_function.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_empty_project_id_is_ok(self, mock_hook):
+        operator = GcfFunctionDeployOperator(
+            location="test_region",
+            body=deepcopy(VALID_BODY),
+            task_id="id"
+        )
+        operator._hook.get_function.side_effect = \
+            HttpError(resp=MOCK_RESP_404, content=b'not found')
+        operator.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        new_body = deepcopy(VALID_BODY)
+        new_body['labels'] = {
+            'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')}
+        mock_hook.return_value.create_new_function.assert_called_once_with(
+            project_id=None,
+            location="test_region",
+            body=new_body)
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_empty_location(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="",
+                body=None,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter 'location' is missing", str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_empty_body(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=None,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter 'body' is missing", str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+
+    @parameterized.expand([
+        (runtime,) for runtime in VALID_RUNTIMES
+    ])
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_correct_runtime_field(self, runtime, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body['runtime'] = runtime
+        op = GcfFunctionDeployOperator(
+            project_id="test_project_id",
+            location="test_region",
+            body=body,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        (network,) for network in [
+            "network-01",
+            "n-0-2-3-4",
+            "projects/PROJECT/global/networks/network-01"
+            "projects/PRÓJECT/global/networks/netwórk-01"
+        ]
+    ])
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_valid_network_field(self, network, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body['network'] = network
+        op = GcfFunctionDeployOperator(
+            project_id="test_project_id",
+            location="test_region",
+            body=body,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        (labels,) for labels in [
+            {},
+            {"label": 'value-01'},
+            {"label_324234_a_b_c": 'value-01_93'},
+        ]
+    ])
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_valid_labels_field(self, labels, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body['labels'] = labels
+        op = GcfFunctionDeployOperator(
+            project_id="test_project_id",
+            location="test_region",
+            body=body,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_validation_disabled(self, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = {
+            "name": "function_name",
+            "some_invalid_body_field": "some_invalid_body_field_value"
+        }
+        op = GcfFunctionDeployOperator(
+            project_id="test_project_id",
+            location="test_region",
+            body=body,
+            validate_body=False,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_body_validation_simple(self, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body['name'] = ''
+        with self.assertRaises(AirflowException) as cm:
+            op = GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=body,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The body field 'name' of value '' does not match",
+                      str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        ('name', '',
+         "The body field 'name' of value '' does not match"),
+        ('description', '', "The body field 'description' of value '' does not match"),
+        ('entryPoint', '', "The body field 'entryPoint' of value '' does not match"),
+        ('availableMemoryMb', '0',
+         "The available memory has to be greater than 0"),
+        ('availableMemoryMb', '-1',
+         "The available memory has to be greater than 0"),
+        ('availableMemoryMb', 'ss',
+         "invalid literal for int() with base 10: 'ss'"),
+        ('network', '', "The body field 'network' of value '' does not match"),
+        ('maxInstances', '0', "The max instances parameter has to be greater than 0"),
+        ('maxInstances', '-1', "The max instances parameter has to be greater than 0"),
+        ('maxInstances', 'ss', "invalid literal for int() with base 10: 'ss'"),
+    ])
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_invalid_field_values(self, key, value, message, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body[key] = value
+        with self.assertRaises(AirflowException) as cm:
+            op = GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=body,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn(message, str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        ({'sourceArchiveUrl': ''},
+         "The body field 'source_code.sourceArchiveUrl' of value '' does not match"),
+        ({'sourceArchiveUrl': '', 'zip_path': '/path/to/file'},
+         "Only one of 'sourceArchiveUrl' in body or 'zip_path' argument allowed."),
+        ({'sourceArchiveUrl': 'gs://url', 'zip_path': '/path/to/file'},
+         "Only one of 'sourceArchiveUrl' in body or 'zip_path' argument allowed."),
+        ({'sourceArchiveUrl': '', 'sourceUploadUrl': ''},
+         "Parameter 'sourceUploadUrl' is empty in the body and argument "
+         "'zip_path' is missing or empty."),
+        ({'sourceArchiveUrl': 'gs://adasda', 'sourceRepository': ''},
+         "The field 'source_code.sourceRepository' should be of dictionary type"),
+        ({'sourceUploadUrl': '', 'sourceRepository': ''},
+         "Parameter 'sourceUploadUrl' is empty in the body and argument 'zip_path' "
+         "is missing or empty."),
+        ({'sourceArchiveUrl': '', 'sourceUploadUrl': '', 'sourceRepository': ''},
+         "Parameter 'sourceUploadUrl' is empty in the body and argument 'zip_path' "
+         "is missing or empty."),
+        ({'sourceArchiveUrl': 'gs://url', 'sourceUploadUrl': 'https://url'},
+         "The mutually exclusive fields 'sourceUploadUrl' and 'sourceArchiveUrl' "
+         "belonging to the union 'source_code' are both present. Please remove one"),
+        ({'sourceUploadUrl': 'https://url', 'zip_path': '/path/to/file'},
+         "Only one of 'sourceUploadUrl' in body "
+         "or 'zip_path' argument allowed. Found both."),
+        ({'sourceUploadUrl': ''}, "Parameter 'sourceUploadUrl' is empty in the body "
+                                  "and argument 'zip_path' is missing or empty."),
+        ({'sourceRepository': ''}, "The field 'source_code.sourceRepository' "
+                                   "should be of dictionary type"),
+        ({'sourceRepository': {}}, "The required body field "
+                                   "'source_code.sourceRepository.url' is missing"),
+        ({'sourceRepository': {'url': ''}},
+         "The body field 'source_code.sourceRepository.url' of value '' does not match"),
+    ]
+    )
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_invalid_source_code_union_field(self, source_code, message, mock_hook):
+        mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
+        body = deepcopy(VALID_BODY)
+        body.pop('sourceUploadUrl', None)
+        body.pop('sourceArchiveUrl', None)
+        zip_path = source_code.pop('zip_path', None)
+        body.update(source_code)
+        with self.assertRaises(AirflowException) as cm:
+            op = GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=body,
+                task_id="id",
+                zip_path=zip_path
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn(message, str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        ({'sourceArchiveUrl': 'gs://url'}, 'test_project_id'),
+        ({'zip_path': '/path/to/file', 'sourceUploadUrl': None}, 'test_project_id'),
+        ({'zip_path': '/path/to/file', 'sourceUploadUrl': None}, None),
+        ({'sourceUploadUrl':
+         'https://source.developers.google.com/projects/a/repos/b/revisions/c/paths/d'},
+         'test_project_id'),
+        ({'sourceRepository':
+         {'url': 'https://source.developers.google.com/projects/a/'
+          'repos/b/revisions/c/paths/d'}},
+         'test_project_id'),
+    ])
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_valid_source_code_union_field(self, source_code, project_id, mock_hook):
+        mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
+        mock_hook.return_value.get_function.side_effect = mock.Mock(
+            side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body.pop('sourceUploadUrl', None)
+        body.pop('sourceArchiveUrl', None)
+        body.pop('sourceRepository', None)
+        body.pop('sourceRepositoryUrl', None)
+        zip_path = source_code.pop('zip_path', None)
+        body.update(source_code)
+        if project_id:
+            op = GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=body,
+                task_id="id",
+                zip_path=zip_path
+            )
+        else:
+            op = GcfFunctionDeployOperator(
+                location="test_region",
+                body=body,
+                task_id="id",
+                zip_path=zip_path
+            )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        if zip_path:
+            mock_hook.return_value.upload_function_zip.assert_called_once_with(
+                project_id=project_id,
+                location='test_region',
+                zip_path='/path/to/file'
+            )
+        mock_hook.return_value.get_function.assert_called_once_with(
+            'projects/test_project_id/locations/test_region/functions/helloWorld'
+        )
+        mock_hook.return_value.create_new_function.assert_called_once_with(
+            project_id=project_id,
+            location='test_region',
+            body=body
+        )
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        ({'eventTrigger': {}},
+         "The required body field 'trigger.eventTrigger.eventType' is missing"),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b'}},
+         "The required body field 'trigger.eventTrigger.resource' is missing"),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b', 'resource': ''}},
+         "The body field 'trigger.eventTrigger.resource' of value '' does not match"),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b',
+                           'resource': 'res',
+                           'service': ''}},
+         "The body field 'trigger.eventTrigger.service' of value '' does not match"),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b',
+                           'resource': 'res',
+                           'service': 'service_name',
+                           'failurePolicy': {'retry': ''}}},
+         "The field 'trigger.eventTrigger.failurePolicy.retry' "
+         "should be of dictionary type")
+    ]
+    )
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_invalid_trigger_union_field(self, trigger, message, mock_hook):
+        mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
+        body = deepcopy(VALID_BODY)
+        body.pop('httpsTrigger', None)
+        body.pop('eventTrigger', None)
+        body.update(trigger)
+        with self.assertRaises(AirflowException) as cm:
+            op = GcfFunctionDeployOperator(
+                project_id="test_project_id",
+                location="test_region",
+                body=body,
+                task_id="id",
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn(message, str(err))
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+    @parameterized.expand([
+        ({'httpsTrigger': {}},),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b',
+                           'resource': 'res'}},),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b',
+                           'resource': 'res',
+                           'service': 'service_name'}},),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/ą.b',
+                           'resource': 'reś',
+                           'service': 'service_namę'}},),
+        ({'eventTrigger': {'eventType': 'providers/test/eventTypes/a.b',
+                           'resource': 'res',
+                           'service': 'service_name',
+                           'failurePolicy': {'retry': {}}}},)
+    ])
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_valid_trigger_union_field(self, trigger, mock_hook):
+        mock_hook.return_value.upload_function_zip.return_value = 'https://uploadUrl'
+        mock_hook.return_value.get_function.side_effect = mock.Mock(
+            side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body.pop('httpsTrigger', None)
+        body.pop('eventTrigger', None)
+        body.update(trigger)
+        op = GcfFunctionDeployOperator(
+            project_id="test_project_id",
+            location="test_region",
+            body=body,
+            task_id="id",
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.get_function.assert_called_once_with(
+            'projects/test_project_id/locations/test_region/functions/helloWorld'
+        )
+        mock_hook.return_value.create_new_function.assert_called_once_with(
+            project_id='test_project_id',
+            location='test_region',
+            body=body
+        )
+        mock_hook.reset_mock()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_extra_parameter(self, mock_hook):
+        mock_hook.return_value.create_new_function.return_value = True
+        body = deepcopy(VALID_BODY)
+        body['extra_parameter'] = 'extra'
+        op = GcfFunctionDeployOperator(
+            project_id="test_project_id",
+            location="test_region",
+            body=body,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.reset_mock()
+
+
+class GcfFunctionDeleteTest(unittest.TestCase):
+    _FUNCTION_NAME = 'projects/project_name/locations/project_location/functions' \
+                     '/function_name'
+    _DELETE_FUNCTION_EXPECTED = {
+        '@type': 'type.googleapis.com/google.cloud.functions.v1.CloudFunction',
+        'name': _FUNCTION_NAME,
+        'sourceArchiveUrl': 'gs://functions/hello.zip',
+        'httpsTrigger': {
+            'url': 'https://project_location-project_name.cloudfunctions.net'
+                   '/function_name'},
+        'status': 'ACTIVE', 'entryPoint': 'entry_point', 'timeout': '60s',
+        'availableMemoryMb': 256,
+        'serviceAccountEmail': 'project_name@appspot.gserviceaccount.com',
+        'updateTime': '2018-08-23T00:00:00Z',
+        'versionId': '1', 'runtime': 'nodejs6'}
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_delete_execute(self, mock_hook):
+        mock_hook.return_value.delete_function.return_value = \
+            self._DELETE_FUNCTION_EXPECTED
+        op = GcfFunctionDeleteOperator(
+            name=self._FUNCTION_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.delete_function.assert_called_once_with(
+            'projects/project_name/locations/project_location/functions/function_name'
+        )
+        self.assertEqual(result['name'], self._FUNCTION_NAME)
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_correct_name(self, mock_hook):
+        op = GcfFunctionDeleteOperator(
+            name="projects/project_name/locations/project_location/functions"
+                 "/function_name",
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_invalid_name(self, mock_hook):
+        with self.assertRaises(AttributeError) as cm:
+            op = GcfFunctionDeleteOperator(
+                name="invalid_name",
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertEqual(str(err), 'Parameter name must match pattern: {}'.format(
+            FUNCTION_NAME_PATTERN))
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_empty_name(self, mock_hook):
+        mock_hook.return_value.delete_function.return_value = \
+            self._DELETE_FUNCTION_EXPECTED
+        with self.assertRaises(AttributeError) as cm:
+            GcfFunctionDeleteOperator(
+                name="",
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertEqual(str(err), 'Empty parameter: name')
+        mock_hook.assert_not_called()
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_gcf_error_silenced_when_function_doesnt_exist(self, mock_hook):
+        op = GcfFunctionDeleteOperator(
+            name=self._FUNCTION_NAME,
+            task_id="id"
+        )
+        mock_hook.return_value.delete_function.side_effect = mock.Mock(
+            side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.delete_function.assert_called_once_with(
+            'projects/project_name/locations/project_location/functions/function_name'
+        )
+
+    @mock.patch('airflow.contrib.operators.gcp_function_operator.GcfHook')
+    def test_non_404_gcf_error_bubbled_up(self, mock_hook):
+        op = GcfFunctionDeleteOperator(
+            name=self._FUNCTION_NAME,
+            task_id="id"
+        )
+        resp = type('', (object,), {"status": 500})()
+        mock_hook.return_value.delete_function.side_effect = mock.Mock(
+            side_effect=HttpError(resp=resp, content=b'error'))
+
+        with self.assertRaises(HttpError):
+            op.execute(None)
+
+        mock_hook.assert_called_once_with(api_version='v1',
+                                          gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.delete_function.assert_called_once_with(
+            'projects/project_name/locations/project_location/functions/function_name'
+        )
diff --git a/scripts/ci/run_tests.sh b/tests/contrib/operators/test_gcp_function_operator_system.py
old mode 100755
new mode 100644
similarity index 54%
rename from scripts/ci/run_tests.sh
rename to tests/contrib/operators/test_gcp_function_operator_system.py
index d5e7655803df5..2dc24de00a16e
--- a/scripts/ci/run_tests.sh
+++ b/tests/contrib/operators/test_gcp_function_operator_system.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- coding: utf-8 -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -8,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,30 +17,21 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -o verbose
-
-if [ -z "$HADOOP_HOME" ]; then
-    echo "HADOOP_HOME not set - abort" >&2
-    exit 1
-fi
-
-echo "Using ${HADOOP_DISTRO} distribution of Hadoop from ${HADOOP_HOME}"
-
-pwd
-
-mkdir ~/airflow/
+import unittest
 
-if [ "${TRAVIS}" ]; then
-    echo "Using travis airflow.cfg"
-    DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-    cp -f ${DIR}/airflow_travis.cfg ~/airflow/unittests.cfg
+from tests.contrib.utils.base_gcp_system_test_case import \
+    SKIP_TEST_WARNING, DagGcpSystemTestCase
+from tests.contrib.utils.gcp_authenticator import GCP_FUNCTION_KEY
 
-    ROOTDIR="$(dirname $(dirname $DIR))"
-    export AIRFLOW__CORE__DAGS_FOLDER="$ROOTDIR/tests/dags"
 
-    # kdc init happens in setup_kdc.sh
-    kinit -kt ${KRB5_KTNAME} airflow
-fi
+@unittest.skipIf(
+    DagGcpSystemTestCase.skip_check(GCP_FUNCTION_KEY), SKIP_TEST_WARNING)
+class GcpFunctionExampleDagsSystemTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(GcpFunctionExampleDagsSystemTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_function',
+            gcp_key=GCP_FUNCTION_KEY)
 
-echo Backend: $AIRFLOW__CORE__SQL_ALCHEMY_CONN
-./run_unit_tests.sh $@
+    def test_run_example_dag_function(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_natural_language_operator.py b/tests/contrib/operators/test_gcp_natural_language_operator.py
new file mode 100644
index 0000000000000..ba05c467ca90a
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_natural_language_operator.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import unittest
+
+from google.cloud.language_v1.proto.language_service_pb2 import (
+    AnalyzeEntitiesResponse,
+    AnalyzeEntitySentimentResponse,
+    AnalyzeSentimentResponse,
+    ClassifyTextResponse,
+    Document,
+)
+
+from airflow.contrib.operators.gcp_natural_language_operator import (
+    CloudLanguageAnalyzeEntitiesOperator,
+    CloudLanguageAnalyzeEntitySentimentOperator,
+    CloudLanguageAnalyzeSentimentOperator,
+    CloudLanguageClassifyTextOperator,
+)
+from tests.compat import patch
+
+
+DOCUMENT = Document(
+    content="Airflow is a platform to programmatically author, schedule and monitor workflows."
+)
+
+CLASSIFY_TEXT_RRESPONSE = ClassifyTextResponse()
+ANALYZE_ENTITIES_RESPONSE = AnalyzeEntitiesResponse()
+ANALYZE_ENTITY_SENTIMENT_RESPONSE = AnalyzeEntitySentimentResponse()
+ANALYZE_SENTIMENT_RESPONSE = AnalyzeSentimentResponse()
+
+ENCODING_TYPE = "UTF32"
+
+
+class CloudLanguageAnalyzeEntitiesOperatorTestCase(unittest.TestCase):
+    @patch("airflow.contrib.operators.gcp_natural_language_operator.CloudNaturalLanguageHook")
+    def test_minimal_green_path(self, hook_mock):
+        hook_mock.return_value.analyze_entities.return_value = ANALYZE_ENTITIES_RESPONSE
+        op = CloudLanguageAnalyzeEntitiesOperator(task_id="task-id", document=DOCUMENT)
+        resp = op.execute({})
+        self.assertEquals(resp, {})
+
+
+class CloudLanguageAnalyzeEntitySentimentOperatorTestCase(unittest.TestCase):
+    @patch("airflow.contrib.operators.gcp_natural_language_operator.CloudNaturalLanguageHook")
+    def test_minimal_green_path(self, hook_mock):
+        hook_mock.return_value.analyze_entity_sentiment.return_value = ANALYZE_ENTITY_SENTIMENT_RESPONSE
+        op = CloudLanguageAnalyzeEntitySentimentOperator(task_id="task-id", document=DOCUMENT)
+        resp = op.execute({})
+        self.assertEquals(resp, {})
+
+
+class CloudLanguageAnalyzeSentimentOperatorTestCase(unittest.TestCase):
+    @patch("airflow.contrib.operators.gcp_natural_language_operator.CloudNaturalLanguageHook")
+    def test_minimal_green_path(self, hook_mock):
+        hook_mock.return_value.analyze_sentiment.return_value = ANALYZE_SENTIMENT_RESPONSE
+        op = CloudLanguageAnalyzeSentimentOperator(task_id="task-id", document=DOCUMENT)
+        resp = op.execute({})
+        self.assertEquals(resp, {})
+
+
+class CloudLanguageClassifyTextOperatorTestCase(unittest.TestCase):
+    @patch("airflow.contrib.operators.gcp_natural_language_operator.CloudNaturalLanguageHook")
+    def test_minimal_green_path(self, hook_mock):
+        hook_mock.return_value.classify_text.return_value = CLASSIFY_TEXT_RRESPONSE
+        op = CloudLanguageClassifyTextOperator(task_id="task-id", document=DOCUMENT)
+        resp = op.execute({})
+        self.assertEquals(resp, {})
diff --git a/tests/contrib/operators/test_gcp_natural_language_operator_system.py b/tests/contrib/operators/test_gcp_natural_language_operator_system.py
new file mode 100644
index 0000000000000..a21519fb5deb9
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_natural_language_operator_system.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from tests.contrib.utils.base_gcp_system_test_case import SKIP_TEST_WARNING, DagGcpSystemTestCase
+from tests.contrib.utils.gcp_authenticator import GCP_AI_KEY
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_AI_KEY), SKIP_TEST_WARNING)
+class CloudNaturalLanguageExampleDagsTest(DagGcpSystemTestCase):
+    def __init__(self, method_name="runTest"):
+        super(CloudNaturalLanguageExampleDagsTest, self).__init__(
+            method_name, dag_id="example_gcp_natural_language", gcp_key=GCP_AI_KEY
+        )
+
+    def test_run_example_dagr(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_spanner_operator.py b/tests/contrib/operators/test_gcp_spanner_operator.py
new file mode 100644
index 0000000000000..7c06d4f6629a7
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_spanner_operator.py
@@ -0,0 +1,538 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from parameterized import parameterized
+
+from airflow import AirflowException
+from airflow.contrib.operators.gcp_spanner_operator import \
+    CloudSpannerInstanceDeployOperator, \
+    CloudSpannerInstanceDeleteOperator, \
+    CloudSpannerInstanceDatabaseQueryOperator, \
+    CloudSpannerInstanceDatabaseDeployOperator, \
+    CloudSpannerInstanceDatabaseDeleteOperator, \
+    CloudSpannerInstanceDatabaseUpdateOperator
+from tests.compat import mock
+
+
+PROJECT_ID = 'project-id'
+INSTANCE_ID = 'instance-id'
+DB_ID = 'db1'
+CONFIG_NAME = 'projects/project-id/instanceConfigs/eur3'
+NODE_COUNT = '1'
+DISPLAY_NAME = 'Test Instance'
+INSERT_QUERY = "INSERT my_table1 (id, name) VALUES (1, 'One')"
+INSERT_QUERY_2 = "INSERT my_table2 (id, name) VALUES (1, 'One')"
+CREATE_QUERY = "CREATE TABLE my_table1 (id INT64, name STRING(100))"
+CREATE_QUERY_2 = "CREATE TABLE my_table2 (id INT64, name STRING(100))"
+DDL_STATEMENTS = [CREATE_QUERY, CREATE_QUERY_2]
+
+
+class CloudSpannerTest(unittest.TestCase):
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_create(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+        op = CloudSpannerInstanceDeployOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME
+        )
+        mock_hook.return_value.update_instance.assert_not_called()
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_create_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+        op = CloudSpannerInstanceDeployOperator(
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_instance.assert_called_once_with(
+            project_id=None,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME
+        )
+        mock_hook.return_value.update_instance.assert_not_called()
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_update(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = {"name": INSTANCE_ID}
+        op = CloudSpannerInstanceDeployOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.update_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME
+        )
+        mock_hook.return_value.create_instance.assert_not_called()
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_update_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = {"name": INSTANCE_ID}
+        op = CloudSpannerInstanceDeployOperator(
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.update_instance.assert_called_once_with(
+            project_id=None,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME
+        )
+        mock_hook.return_value.create_instance.assert_not_called()
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_create_aborts_and_succeeds_if_instance_exists(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = {"name": INSTANCE_ID}
+        op = CloudSpannerInstanceDeployOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            configuration_name=CONFIG_NAME,
+            node_count=int(NODE_COUNT),
+            display_name=DISPLAY_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_instance.assert_not_called()
+        self.assertIsNone(result)
+
+    @parameterized.expand([
+        ("", INSTANCE_ID, "project_id"),
+        (PROJECT_ID, "", "instance_id"),
+    ])
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_create_ex_if_param_missing(self, project_id, instance_id,
+                                                 exp_msg, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            CloudSpannerInstanceDeployOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                configuration_name=CONFIG_NAME,
+                node_count=int(NODE_COUNT),
+                display_name=DISPLAY_NAME,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter '{}' is empty".format(exp_msg), str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_delete(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = {"name": INSTANCE_ID}
+        op = CloudSpannerInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_delete_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_instance.return_value = {"name": INSTANCE_ID}
+        op = CloudSpannerInstanceDeleteOperator(
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=None,
+            instance_id=INSTANCE_ID
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_delete_aborts_and_succeeds_if_instance_does_not_exist(self,
+                                                                            mock_hook):
+        mock_hook.return_value.get_instance.return_value = None
+        op = CloudSpannerInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_instance.assert_not_called()
+        self.assertTrue(result)
+
+    @parameterized.expand([
+        ("", INSTANCE_ID, "project_id"),
+        (PROJECT_ID, "", "instance_id"),
+    ])
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_delete_ex_if_param_missing(self, project_id, instance_id, exp_msg,
+                                                 mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            CloudSpannerInstanceDeleteOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter '{}' is empty".format(exp_msg), str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_query(self, mock_hook):
+        mock_hook.return_value.execute_sql.return_value = None
+        op = CloudSpannerInstanceDatabaseQueryOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            query=INSERT_QUERY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.execute_dml.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            queries=[INSERT_QUERY]
+        )
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_query_missing_project_id(self, mock_hook):
+        mock_hook.return_value.execute_sql.return_value = None
+        op = CloudSpannerInstanceDatabaseQueryOperator(
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            query=INSERT_QUERY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.execute_dml.assert_called_once_with(
+            project_id=None, instance_id=INSTANCE_ID,
+            database_id=DB_ID, queries=[INSERT_QUERY]
+        )
+        self.assertIsNone(result)
+
+    @parameterized.expand([
+        ("", INSTANCE_ID, DB_ID, INSERT_QUERY, "project_id"),
+        (PROJECT_ID, "", DB_ID, INSERT_QUERY, "instance_id"),
+        (PROJECT_ID, INSTANCE_ID, "", INSERT_QUERY, "database_id"),
+        (PROJECT_ID, INSTANCE_ID, DB_ID, "", "query"),
+    ])
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_query_ex_if_param_missing(self, project_id, instance_id,
+                                                database_id, query, exp_msg, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            CloudSpannerInstanceDatabaseQueryOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                database_id=database_id,
+                query=query,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter '{}' is empty".format(exp_msg), str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_query_dml(self, mock_hook):
+        mock_hook.return_value.execute_dml.return_value = None
+        op = CloudSpannerInstanceDatabaseQueryOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            query=INSERT_QUERY,
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.execute_dml.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID, database_id=DB_ID, queries=[INSERT_QUERY]
+        )
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_instance_query_dml_list(self, mock_hook):
+        mock_hook.return_value.execute_dml.return_value = None
+        op = CloudSpannerInstanceDatabaseQueryOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            query=[INSERT_QUERY, INSERT_QUERY_2],
+            task_id="id"
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.execute_dml.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID,
+            database_id=DB_ID, queries=[INSERT_QUERY, INSERT_QUERY_2]
+        )
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_create(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = None
+        op = CloudSpannerInstanceDatabaseDeployOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_database.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID, database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS
+        )
+        mock_hook.return_value.update_database.assert_not_called()
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_create_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = None
+        op = CloudSpannerInstanceDatabaseDeployOperator(
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_database.assert_called_once_with(
+            project_id=None, instance_id=INSTANCE_ID, database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS
+        )
+        mock_hook.return_value.update_database.assert_not_called()
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_create_with_pre_existing_db(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = {"name": DB_ID}
+        op = CloudSpannerInstanceDatabaseDeployOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_database.assert_not_called()
+        mock_hook.return_value.update_database.assert_not_called()
+        self.assertTrue(result)
+
+    @parameterized.expand([
+        ("", INSTANCE_ID, DB_ID, DDL_STATEMENTS, 'project_id'),
+        (PROJECT_ID, "", DB_ID, DDL_STATEMENTS, 'instance_id'),
+        (PROJECT_ID, INSTANCE_ID, "", DDL_STATEMENTS, 'database_id'),
+    ])
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_create_ex_if_param_missing(self,
+                                                 project_id, instance_id,
+                                                 database_id, ddl_statements,
+                                                 exp_msg, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            CloudSpannerInstanceDatabaseDeployOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                database_id=database_id,
+                ddl_statements=ddl_statements,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter '{}' is empty".format(exp_msg), str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_update(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = {"name": DB_ID}
+        op = CloudSpannerInstanceDatabaseUpdateOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.update_database.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID, database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS, operation_id=None
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_update_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = {"name": DB_ID}
+        op = CloudSpannerInstanceDatabaseUpdateOperator(
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.update_database.assert_called_once_with(
+            project_id=None, instance_id=INSTANCE_ID, database_id=DB_ID,
+            ddl_statements=DDL_STATEMENTS, operation_id=None
+        )
+        self.assertTrue(result)
+
+    @parameterized.expand([
+        ("", INSTANCE_ID, DB_ID, DDL_STATEMENTS, 'project_id'),
+        (PROJECT_ID, "", DB_ID, DDL_STATEMENTS, 'instance_id'),
+        (PROJECT_ID, INSTANCE_ID, "", DDL_STATEMENTS, 'database_id'),
+    ])
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_update_ex_if_param_missing(self, project_id, instance_id,
+                                                 database_id, ddl_statements,
+                                                 exp_msg, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            CloudSpannerInstanceDatabaseUpdateOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                database_id=database_id,
+                ddl_statements=ddl_statements,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter '{}' is empty".format(exp_msg), str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_update_ex_if_database_not_exist(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = None
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSpannerInstanceDatabaseUpdateOperator(
+                project_id=PROJECT_ID,
+                instance_id=INSTANCE_ID,
+                database_id=DB_ID,
+                ddl_statements=DDL_STATEMENTS,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The Cloud Spanner database 'db1' in project 'project-id' and "
+                      "instance 'instance-id' is missing", str(err))
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_delete(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = {"name": DB_ID}
+        op = CloudSpannerInstanceDatabaseDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_database.assert_called_once_with(
+            project_id=PROJECT_ID, instance_id=INSTANCE_ID, database_id=DB_ID
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_delete_missing_project_id(self, mock_hook):
+        mock_hook.return_value.get_database.return_value = {"name": DB_ID}
+        op = CloudSpannerInstanceDatabaseDeleteOperator(
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_database.assert_called_once_with(
+            project_id=None, instance_id=INSTANCE_ID, database_id=DB_ID
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_delete_exits_and_succeeds_if_database_does_not_exist(self,
+                                                                           mock_hook):
+        mock_hook.return_value.get_database.return_value = None
+        op = CloudSpannerInstanceDatabaseDeleteOperator(
+            project_id=PROJECT_ID,
+            instance_id=INSTANCE_ID,
+            database_id=DB_ID,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_database.assert_not_called()
+        self.assertTrue(result)
+
+    @parameterized.expand([
+        ("", INSTANCE_ID, DB_ID, DDL_STATEMENTS, 'project_id'),
+        (PROJECT_ID, "", DB_ID, DDL_STATEMENTS, 'instance_id'),
+        (PROJECT_ID, INSTANCE_ID, "", DDL_STATEMENTS, 'database_id'),
+    ])
+    @mock.patch("airflow.contrib.operators.gcp_spanner_operator.CloudSpannerHook")
+    def test_database_delete_ex_if_param_missing(self, project_id, instance_id,
+                                                 database_id, ddl_statements,
+                                                 exp_msg, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            CloudSpannerInstanceDatabaseDeleteOperator(
+                project_id=project_id,
+                instance_id=instance_id,
+                database_id=database_id,
+                ddl_statements=ddl_statements,
+                task_id="id"
+            )
+        err = cm.exception
+        self.assertIn("The required parameter '{}' is empty".format(exp_msg), str(err))
+        mock_hook.assert_not_called()
diff --git a/tests/contrib/operators/test_gcp_spanner_operator_system.py b/tests/contrib/operators/test_gcp_spanner_operator_system.py
new file mode 100644
index 0000000000000..fd4c50bd8ab09
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_spanner_operator_system.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from tests.contrib.operators.test_gcp_spanner_operator_system_helper import \
+    GCPSpannerTestHelper
+from tests.contrib.utils.base_gcp_system_test_case import \
+    SKIP_TEST_WARNING, DagGcpSystemTestCase
+from tests.contrib.utils.gcp_authenticator import GCP_SPANNER_KEY
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_SPANNER_KEY), SKIP_TEST_WARNING)
+class CloudSpannerExampleDagsTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(CloudSpannerExampleDagsTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_spanner',
+            gcp_key=GCP_SPANNER_KEY)
+        self.helper = GCPSpannerTestHelper()
+
+    def tearDown(self):
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            self.helper.delete_instance()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+        super(CloudSpannerExampleDagsTest, self).tearDown()
+
+    def test_run_example_dag_spanner(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_spanner_operator_system_helper.py b/tests/contrib/operators/test_gcp_spanner_operator_system_helper.py
new file mode 100755
index 0000000000000..b715bcdbcb47c
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_spanner_operator_system_helper.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+
+import argparse
+
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator, GCP_SPANNER_KEY
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_SPANNER_INSTANCE_ID = os.environ.get('GCP_SPANNER_INSTANCE_ID', 'testinstance')
+
+
+class GCPSpannerTestHelper(LoggingCommandExecutor):
+
+    def delete_instance(self):
+        self.execute_cmd([
+            'gcloud', 'spanner', '--project', GCP_PROJECT_ID,
+            '--quiet', '--verbosity=none',
+            'instances', 'delete', GCP_SPANNER_INSTANCE_ID
+        ])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Create or delete spanner instances for system tests.')
+    parser.add_argument('--action', dest='action', required=True,
+                        choices=('delete-instance',
+                                 'before-tests', 'after-tests'))
+    action = parser.parse_args().action
+
+    helper = GCPSpannerTestHelper()
+    gcp_authenticator = GcpAuthenticator(GCP_SPANNER_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            pass
+        elif action == 'after-tests':
+            pass
+        elif action == 'delete-instance':
+            helper.delete_instance()
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcp_sql_operator.py b/tests/contrib/operators/test_gcp_sql_operator.py
new file mode 100644
index 0000000000000..f8266adbd5449
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_sql_operator.py
@@ -0,0 +1,1045 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import os
+import unittest
+
+from parameterized import parameterized
+
+from airflow import AirflowException
+from airflow.contrib.operators.gcp_sql_operator import CloudSqlInstanceCreateOperator, \
+    CloudSqlInstancePatchOperator, CloudSqlInstanceDeleteOperator, \
+    CloudSqlInstanceDatabaseCreateOperator, CloudSqlInstanceDatabasePatchOperator, \
+    CloudSqlInstanceExportOperator, CloudSqlInstanceImportOperator, \
+    CloudSqlInstanceDatabaseDeleteOperator, CloudSqlQueryOperator
+from airflow.models.connection import Connection
+from tests.compat import mock
+
+PROJECT_ID = os.environ.get('PROJECT_ID', 'project-id')
+INSTANCE_NAME = os.environ.get('INSTANCE_NAME', 'test-name')
+DB_NAME = os.environ.get('DB_NAME', 'db1')
+
+CREATE_BODY = {
+    "name": INSTANCE_NAME,
+    "settings": {
+        "tier": "db-n1-standard-1",
+        "backupConfiguration": {
+            "binaryLogEnabled": True,
+            "enabled": True,
+            "replicationLogArchivingEnabled": True,
+            "startTime": "05:00"
+        },
+        "activationPolicy": "ALWAYS",
+        "authorizedGaeApplications": [],
+        "crashSafeReplicationEnabled": True,
+        "dataDiskSizeGb": 30,
+        "dataDiskType": "PD_SSD",
+        "databaseFlags": [],
+        "ipConfiguration": {
+            "ipv4Enabled": True,
+            "authorizedNetworks": [
+                {
+                    "value": "192.168.100.0/24",
+                    "name": "network1",
+                    "expirationTime": "2012-11-15T16:19:00.094Z"
+                },
+            ],
+            "privateNetwork": "/vpc/resource/link",
+            "requireSsl": True
+        },
+        "locationPreference": {
+            "zone": "europe-west4-a",
+            "followGaeApplication": "/app/engine/application/to/follow"
+        },
+        "maintenanceWindow": {
+            "hour": 5,
+            "day": 7,
+            "updateTrack": "canary"
+        },
+        "pricingPlan": "PER_USE",
+        "replicationType": "ASYNCHRONOUS",
+        "storageAutoResize": False,
+        "storageAutoResizeLimit": 0,
+        "userLabels": {
+            "my-key": "my-value"
+        }
+    },
+    "databaseVersion": "MYSQL_5_7",
+    "failoverReplica": {
+        "name": "replica-1"
+    },
+    "masterInstanceName": "master-instance-1",
+    "onPremisesConfiguration": {},
+    "region": "europe-west4",
+    "replicaConfiguration": {
+        "mysqlReplicaConfiguration": {
+            "caCertificate": "cert-pem",
+            "clientCertificate": "cert-pem",
+            "clientKey": "cert-pem",
+            "connectRetryInterval": 30,
+            "dumpFilePath": "/path/to/dump",
+            "masterHeartbeatPeriod": 100,
+            "password": "secret_pass",
+            "sslCipher": "list-of-ciphers",
+            "username": "user",
+            "verifyServerCertificate": True
+        },
+    }
+}
+PATCH_BODY = {
+    "name": INSTANCE_NAME,
+    "settings": {
+        "tier": "db-n1-standard-2",
+        "dataDiskType": "PD_HDD"
+    },
+    "region": "europe-west4"
+}
+DATABASE_INSERT_BODY = {
+    "name": DB_NAME,            # The name of the database in the Cloud SQL instance.
+                                # This does not include the project ID or instance name.
+
+    "project": PROJECT_ID,      # The project ID of the project containing the Cloud SQL
+                                # database. The Google apps domain is prefixed if
+                                # applicable.
+
+    "instance": INSTANCE_NAME,  # The name of the Cloud SQL instance.
+                                # This does not include the project ID.
+}
+DATABASE_PATCH_BODY = {
+    "charset": "utf16",
+    "collation": "utf16_general_ci"
+}
+EXPORT_BODY = {
+    "exportContext": {
+        "fileType": "CSV",
+        "uri": "gs://bucketName/fileName",
+        "databases": [],
+        "sqlExportOptions": {
+            "tables": [
+                "table1", "table2"
+            ],
+            "schemaOnly": False
+        },
+        "csvExportOptions": {
+            "selectQuery": "SELECT * FROM TABLE"
+        }
+    }
+}
+IMPORT_BODY = {
+    "importContext": {
+        "fileType": "CSV",
+        "uri": "gs://bucketName/fileName",
+        "database": "db1",
+        "importUser": "",
+        "csvImportOptions": {
+            "table": "my_table",
+            "columns": [
+                "col1", "col2"
+            ]
+        }
+    }
+}
+
+
+class CloudSqlTest(unittest.TestCase):
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceCreateOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_create(self, mock_hook, _check_if_instance_exists):
+        _check_if_instance_exists.return_value = False
+        mock_hook.return_value.create_instance.return_value = True
+        op = CloudSqlInstanceCreateOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=CREATE_BODY,
+            task_id="id"
+        )
+        result = op.execute(context={
+            'task_instance': mock.Mock()
+        })
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            body=CREATE_BODY
+        )
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceCreateOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_create_missing_project_id(self, mock_hook, _check_if_instance_exists):
+        _check_if_instance_exists.return_value = False
+        mock_hook.return_value.create_instance.return_value = True
+        op = CloudSqlInstanceCreateOperator(
+            instance=INSTANCE_NAME,
+            body=CREATE_BODY,
+            task_id="id"
+        )
+        result = op.execute(context={
+            'task_instance': mock.Mock()
+        })
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_instance.assert_called_once_with(
+            project_id=None,
+            body=CREATE_BODY
+        )
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceCreateOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_create_idempotent(self, mock_hook, _check_if_instance_exists):
+        _check_if_instance_exists.return_value = True
+        mock_hook.return_value.create_instance.return_value = True
+        op = CloudSqlInstanceCreateOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=CREATE_BODY,
+            task_id="id"
+        )
+        result = op.execute(context={
+            'task_instance': mock.Mock()
+        })
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_instance.assert_not_called()
+        self.assertIsNone(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_create_should_throw_ex_when_empty_project_id(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceCreateOperator(
+                project_id="",
+                body=CREATE_BODY,
+                instance=INSTANCE_NAME,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'project_id' is empty", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_create_should_throw_ex_when_empty_body(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceCreateOperator(
+                project_id=PROJECT_ID,
+                body={},
+                instance=INSTANCE_NAME,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'body' is empty", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_create_should_throw_ex_when_empty_instance(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceCreateOperator(
+                project_id=PROJECT_ID,
+                body=CREATE_BODY,
+                instance="",
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'instance' is empty", str(err))
+        mock_hook.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_create_should_validate_list_type(self, mock_hook):
+        wrong_list_type_body = {
+            "name": INSTANCE_NAME,
+            "settings": {
+                "tier": "db-n1-standard-1",
+                "ipConfiguration": {
+                    "authorizedNetworks": {}  # Should be a list, not a dict.
+                    # Testing if the validation catches this.
+                }
+            }
+        }
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceCreateOperator(
+                project_id=PROJECT_ID,
+                body=wrong_list_type_body,
+                instance=INSTANCE_NAME,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The field 'settings.ipConfiguration.authorizedNetworks' "
+                      "should be of list type according to the specification", str(err))
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_create_should_validate_non_empty_fields(self, mock_hook):
+        empty_tier_body = {
+            "name": INSTANCE_NAME,
+            "settings": {
+                "tier": "",  # Field can't be empty (defined in CLOUD_SQL_VALIDATION).
+                # Testing if the validation catches this.
+            }
+        }
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceCreateOperator(
+                project_id=PROJECT_ID,
+                body=empty_tier_body,
+                instance=INSTANCE_NAME,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The body field 'settings.tier' can't be empty. "
+                      "Please provide a value.", str(err))
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_patch(self, mock_hook):
+        mock_hook.return_value.patch_instance.return_value = True
+        op = CloudSqlInstancePatchOperator(
+            project_id=PROJECT_ID,
+            body=PATCH_BODY,
+            instance=INSTANCE_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.patch_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            body=PATCH_BODY,
+            instance=INSTANCE_NAME
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_patch_missing_project_id(self, mock_hook):
+        mock_hook.return_value.patch_instance.return_value = True
+        op = CloudSqlInstancePatchOperator(
+            body=PATCH_BODY,
+            instance=INSTANCE_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.patch_instance.assert_called_once_with(
+            project_id=None,
+            body=PATCH_BODY,
+            instance=INSTANCE_NAME
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstancePatchOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_patch_should_bubble_up_ex_if_not_exists(self, mock_hook,
+                                                              _check_if_instance_exists):
+        _check_if_instance_exists.return_value = False
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstancePatchOperator(
+                project_id=PROJECT_ID,
+                body=PATCH_BODY,
+                instance=INSTANCE_NAME,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn('specify another instance to patch', str(err))
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.patch_instance.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDeleteOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_delete(self, mock_hook, _check_if_instance_exists):
+        _check_if_instance_exists.return_value = True
+        op = CloudSqlInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME
+        )
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDeleteOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_delete_missing_project_id(self, mock_hook, _check_if_instance_exists):
+        _check_if_instance_exists.return_value = True
+        op = CloudSqlInstanceDeleteOperator(
+            instance=INSTANCE_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_instance.assert_called_once_with(
+            project_id=None,
+            instance=INSTANCE_NAME
+        )
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDeleteOperator._check_if_instance_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_delete_should_abort_and_succeed_if_not_exists(
+            self,
+            mock_hook,
+            _check_if_instance_exists):
+        _check_if_instance_exists.return_value = False
+        op = CloudSqlInstanceDeleteOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_instance.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabaseCreateOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_create(self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = False
+        op = CloudSqlInstanceDatabaseCreateOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=DATABASE_INSERT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_database.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=DATABASE_INSERT_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabaseCreateOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_create_missing_project_id(self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = False
+        op = CloudSqlInstanceDatabaseCreateOperator(
+            instance=INSTANCE_NAME,
+            body=DATABASE_INSERT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_database.assert_called_once_with(
+            project_id=None,
+            instance=INSTANCE_NAME,
+            body=DATABASE_INSERT_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabaseCreateOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_create_should_abort_and_succeed_if_exists(
+            self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = True
+        op = CloudSqlInstanceDatabaseCreateOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=DATABASE_INSERT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.create_database.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabasePatchOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_patch(self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = True
+        op = CloudSqlInstanceDatabasePatchOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            body=DATABASE_PATCH_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.patch_database.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            body=DATABASE_PATCH_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabasePatchOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_patch_missing_project_id(self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = True
+        op = CloudSqlInstanceDatabasePatchOperator(
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            body=DATABASE_PATCH_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.patch_database.assert_called_once_with(
+            project_id=None,
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            body=DATABASE_PATCH_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabasePatchOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_patch_should_throw_ex_if_not_exists(
+            self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = False
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceDatabasePatchOperator(
+                project_id=PROJECT_ID,
+                instance=INSTANCE_NAME,
+                database=DB_NAME,
+                body=DATABASE_PATCH_BODY,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("Cloud SQL instance with ID", str(err))
+        self.assertIn("does not contain database", str(err))
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.patch_database.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_patch_should_throw_ex_when_empty_database(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlInstanceDatabasePatchOperator(
+                project_id=PROJECT_ID,
+                instance=INSTANCE_NAME,
+                database="",
+                body=DATABASE_INSERT_BODY,
+                task_id="id"
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'database' is empty", str(err))
+        mock_hook.assert_not_called()
+        mock_hook.return_value.patch_database.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabaseDeleteOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_delete(self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = True
+        op = CloudSqlInstanceDatabaseDeleteOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_database.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            database=DB_NAME
+        )
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabaseDeleteOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_delete_missing_project_id(self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = True
+        op = CloudSqlInstanceDatabaseDeleteOperator(
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_database.assert_called_once_with(
+            project_id=None,
+            instance=INSTANCE_NAME,
+            database=DB_NAME
+        )
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator"
+                ".CloudSqlInstanceDatabaseDeleteOperator._check_if_db_exists")
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_db_delete_should_abort_and_succeed_if_not_exists(
+            self, mock_hook, _check_if_db_exists):
+        _check_if_db_exists.return_value = False
+        op = CloudSqlInstanceDatabaseDeleteOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            database=DB_NAME,
+            task_id="id"
+        )
+        result = op.execute(None)
+        self.assertTrue(result)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.delete_database.assert_not_called()
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_export(self, mock_hook):
+        mock_hook.return_value.export_instance.return_value = True
+        op = CloudSqlInstanceExportOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=EXPORT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.export_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=EXPORT_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_export_missing_project_id(self, mock_hook):
+        mock_hook.return_value.export_instance.return_value = True
+        op = CloudSqlInstanceExportOperator(
+            instance=INSTANCE_NAME,
+            body=EXPORT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.export_instance.assert_called_once_with(
+            project_id=None,
+            instance=INSTANCE_NAME,
+            body=EXPORT_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_import(self, mock_hook):
+        mock_hook.return_value.export_instance.return_value = True
+        op = CloudSqlInstanceImportOperator(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=IMPORT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.import_instance.assert_called_once_with(
+            project_id=PROJECT_ID,
+            instance=INSTANCE_NAME,
+            body=IMPORT_BODY
+        )
+        self.assertTrue(result)
+
+    @mock.patch("airflow.contrib.operators.gcp_sql_operator.CloudSqlHook")
+    def test_instance_import_missing_project_id(self, mock_hook):
+        mock_hook.return_value.export_instance.return_value = True
+        op = CloudSqlInstanceImportOperator(
+            instance=INSTANCE_NAME,
+            body=IMPORT_BODY,
+            task_id="id"
+        )
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version="v1beta4",
+                                          gcp_conn_id="google_cloud_default")
+        mock_hook.return_value.import_instance.assert_called_once_with(
+            project_id=None,
+            instance=INSTANCE_NAME,
+            body=IMPORT_BODY
+        )
+        self.assertTrue(result)
+
+
+class CloudSqlQueryValidationTest(unittest.TestCase):
+
+    @staticmethod
+    def _setup_connections(get_connections, uri):
+        gcp_connection = mock.MagicMock()
+        gcp_connection.extra_dejson = mock.MagicMock()
+        gcp_connection.extra_dejson.get.return_value = 'empty_project'
+        cloudsql_connection = Connection()
+        cloudsql_connection.parse_from_uri(uri)
+        cloudsql_connection2 = Connection()
+        cloudsql_connection2.parse_from_uri(uri)
+        get_connections.side_effect = [[gcp_connection], [cloudsql_connection],
+                                       [cloudsql_connection2]]
+
+    @parameterized.expand([
+        ('project_id', '', 'instance_name', 'mysql', False, False,
+         'SELECT * FROM TEST',
+         "The required extra 'location' is empty"),
+        ('project_id', 'location', '', 'postgres', False, False,
+         'SELECT * FROM TEST',
+         "The required extra 'instance' is empty"),
+        ('project_id', 'location', 'instance_name', 'wrong', False, False,
+         'SELECT * FROM TEST',
+         "Invalid database type 'wrong'. Must be one of ['postgres', 'mysql']"),
+        ('project_id', 'location', 'instance_name', 'postgres', True, True,
+         'SELECT * FROM TEST',
+         "Cloud SQL Proxy does not support SSL connections. SSL is not needed as"
+         " Cloud SQL Proxy provides encryption on its own"),
+        ('project_id', 'location', 'instance_name', 'postgres', False, True,
+         'SELECT * FROM TEST',
+         "SSL connections requires sslcert to be set"),
+    ])
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_wrong_parameters(self,
+                                                   project_id,
+                                                   location,
+                                                   instance_name,
+                                                   database_type,
+                                                   use_proxy,
+                                                   use_ssl,
+                                                   sql,
+                                                   message,
+                                                   get_connections):
+        uri = \
+            "gcpcloudsql://user:password@127.0.0.1:3200/testdb?" \
+            "database_type={database_type}&" \
+            "project_id={project_id}&location={location}&instance={instance_name}&" \
+            "use_proxy={use_proxy}&use_ssl={use_ssl}".format(
+                database_type=database_type,
+                project_id=project_id,
+                location=location,
+                instance_name=instance_name,
+                use_proxy=use_proxy,
+                use_ssl=use_ssl)
+        self._setup_connections(get_connections, uri)
+        with self.assertRaises(AirflowException) as cm:
+            op = CloudSqlQueryOperator(
+                sql=sql,
+                task_id='task_id'
+            )
+            op.execute(None)
+        err = cm.exception
+        self.assertIn(message, str(err))
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_postgres(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=postgres&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=False&use_ssl=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.postgres_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('postgres', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_postgres_ssl(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=postgres&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=False&use_ssl=True&sslcert=/bin/bash&" \
+              "sslkey=/bin/bash&sslrootcert=/bin/bash"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.postgres_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('postgres', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+        self.assertEqual('/bin/bash', conn.extra_dejson['sslkey'])
+        self.assertEqual('/bin/bash', conn.extra_dejson['sslcert'])
+        self.assertEqual('/bin/bash', conn.extra_dejson['sslrootcert'])
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_postgres_proxy_socket(
+            self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=postgres&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=True&sql_proxy_use_tcp=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.postgres_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('postgres', conn.conn_type)
+        self.assertIn('/tmp', conn.host)
+        self.assertIn('example-project:europe-west1:testdb', conn.host)
+        self.assertIsNone(conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_project_id_missing(self,
+                                                                        get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=mysql&" \
+              "location=europe-west1&instance=testdb&" \
+              "use_proxy=False&use_ssl=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.mysql_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('mysql', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_postgres_proxy_tcp(self,
+                                                                        get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=postgres&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=True&sql_proxy_use_tcp=True"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.postgres_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('postgres', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertNotEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_mysql(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=mysql&" \
+            "project_id=example-project&location=europe-west1&instance=testdb&" \
+            "use_proxy=False&use_ssl=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.mysql_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('mysql', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_mysql_ssl(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=mysql&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=False&use_ssl=True&sslcert=/bin/bash&" \
+              "sslkey=/bin/bash&sslrootcert=/bin/bash"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.mysql_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('mysql', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+        self.assertEqual('/bin/bash', json.loads(conn.extra_dejson['ssl'])['cert'])
+        self.assertEqual('/bin/bash', json.loads(conn.extra_dejson['ssl'])['key'])
+        self.assertEqual('/bin/bash', json.loads(conn.extra_dejson['ssl'])['ca'])
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_mysql_proxy_socket(self,
+                                                                        get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=mysql&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=True&sql_proxy_use_tcp=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.mysql_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('mysql', conn.conn_type)
+        self.assertEqual('localhost', conn.host)
+        self.assertIn('/tmp', conn.extra_dejson['unix_socket'])
+        self.assertIn('example-project:europe-west1:testdb',
+                      conn.extra_dejson['unix_socket'])
+        self.assertIsNone(conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_correct_parameters_mysql_tcp(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=mysql&" \
+              "project_id=example-project&location=europe-west1&instance=testdb&" \
+              "use_proxy=True&sql_proxy_use_tcp=True"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.mysql_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('mysql', conn.conn_type)
+        self.assertEqual('127.0.0.1', conn.host)
+        self.assertNotEqual(3200, conn.port)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_too_long_unix_socket_path(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=postgres&" \
+              "project_id=example-project&location=europe-west1&" \
+              "instance=" \
+              "test_db_with_long_name_a_bit_above" \
+              "_the_limit_of_UNIX_socket_asdadadasadasd&" \
+              "use_proxy=True&sql_proxy_use_tcp=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        with self.assertRaises(AirflowException) as cm:
+            operator.execute(None)
+        err = cm.exception
+        self.assertIn("The UNIX socket path length cannot exceed", str(err))
+
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_create_operator_with_not_too_long_unix_socket_path(self, get_connections):
+        uri = "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=postgres&" \
+              "project_id=example-project&location=europe-west1&" \
+              "instance=" \
+              "test_db_with_longname_but_with_limit_of_UNIX_socket&" \
+              "use_proxy=True&sql_proxy_use_tcp=False"
+        self._setup_connections(get_connections, uri)
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        operator.cloudsql_db_hook.create_connection()
+        try:
+            db_hook = operator.cloudsql_db_hook.get_database_hook()
+            conn = db_hook._get_connections_from_db(db_hook.postgres_conn_id)[0]
+        finally:
+            operator.cloudsql_db_hook.delete_connection()
+        self.assertEqual('postgres', conn.conn_type)
+        self.assertEqual('testdb', conn.schema)
+
+    @mock.patch("airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook."
+                "delete_connection")
+    @mock.patch("airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook."
+                "get_connection")
+    @mock.patch("airflow.hooks.mysql_hook.MySqlHook.run")
+    @mock.patch("airflow.hooks.base_hook.BaseHook.get_connections")
+    def test_cloudsql_hook_delete_connection_on_exception(
+            self, get_connections, run, get_connection, delete_connection):
+        connection = Connection()
+        connection.parse_from_uri(
+            "gcpcloudsql://user:password@127.0.0.1:3200/testdb?database_type=mysql&"
+            "project_id=example-project&location=europe-west1&instance=testdb&"
+            "use_proxy=False")
+        get_connection.return_value = connection
+
+        db_connection = Connection()
+        db_connection.host = "127.0.0.1"
+        db_connection.set_extra(json.dumps({"project_id": "example-project",
+                                            "location": "europe-west1",
+                                            "instance": "testdb",
+                                            "database_type": "mysql"}))
+        get_connections.return_value = [db_connection]
+        run.side_effect = Exception("Exception when running a query")
+        operator = CloudSqlQueryOperator(
+            sql=['SELECT * FROM TABLE'],
+            task_id='task_id'
+        )
+        with self.assertRaises(Exception) as cm:
+            operator.execute(None)
+        err = cm.exception
+        self.assertEqual("Exception when running a query", str(err))
+        delete_connection.assert_called_once_with()
diff --git a/tests/contrib/operators/test_gcp_sql_operator_system.py b/tests/contrib/operators/test_gcp_sql_operator_system.py
new file mode 100644
index 0000000000000..5508233cfe4cd
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_sql_operator_system.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+import unittest
+
+from airflow import AirflowException
+from tests.contrib.utils.base_gcp_system_test_case import \
+    SKIP_TEST_WARNING, DagGcpSystemTestCase
+from tests.contrib.operators.test_gcp_sql_operator_system_helper import \
+    CloudSqlQueryTestHelper
+from tests.contrib.utils.gcp_authenticator import GCP_CLOUDSQL_KEY
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'project-id')
+
+SQL_QUERY_TEST_HELPER = CloudSqlQueryTestHelper()
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_CLOUDSQL_KEY), SKIP_TEST_WARNING)
+class CloudSqlExampleDagsIntegrationTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(CloudSqlExampleDagsIntegrationTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_sql',
+            gcp_key=GCP_CLOUDSQL_KEY)
+
+    def tearDown(self):
+        # Delete instances just in case the test failed and did not cleanup after itself
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            SQL_QUERY_TEST_HELPER.delete_instances(instance_suffix="-failover-replica")
+            SQL_QUERY_TEST_HELPER.delete_instances(instance_suffix="-read-replica")
+            SQL_QUERY_TEST_HELPER.delete_instances()
+            SQL_QUERY_TEST_HELPER.delete_instances(instance_suffix="2")
+            SQL_QUERY_TEST_HELPER.delete_service_account_acls()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+        super(CloudSqlExampleDagsIntegrationTest, self).tearDown()
+
+    def test_run_example_dag_cloudsql(self):
+        try:
+            self._run_dag()
+        except AirflowException as e:
+            self.log.warning(
+                "In case you see 'The instance or operation is not in an appropriate "
+                "state to handle the request' error - you "
+                "can remove '.random' file from airflow folder and re-run "
+                "the test. This will generate random name of the database for next run "
+                "(the problem is that Cloud SQL keeps names of deleted instances in "
+                "short-term cache).")
+            raise e
diff --git a/tests/contrib/operators/test_gcp_sql_operator_system_helper.py b/tests/contrib/operators/test_gcp_sql_operator_system_helper.py
new file mode 100755
index 0000000000000..d2898ed5db38a
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_sql_operator_system_helper.py
@@ -0,0 +1,468 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import errno
+import json
+import os
+from os.path import expanduser
+
+import argparse
+from threading import Thread
+
+import time
+
+from six.moves.urllib.parse import urlsplit
+
+from tests.contrib.utils.base_gcp_system_test_case import RetrieveVariables
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator, GCP_CLOUDSQL_KEY
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+
+retrieve_variables = RetrieveVariables()
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_LOCATION = os.environ.get('GCP_LOCATION', 'europe-west1')
+
+GCSQL_POSTGRES_SERVER_CA_FILE = os.environ.get('GCSQL_POSTGRES_SERVER_CA_FILE',
+                                               ".key/postgres-server-ca.pem")
+GCSQL_POSTGRES_CLIENT_CERT_FILE = os.environ.get('GCSQL_POSTGRES_CLIENT_CERT_FILE',
+                                                 ".key/postgres-client-cert.pem")
+GCSQL_POSTGRES_CLIENT_KEY_FILE = os.environ.get('GCSQL_POSTGRES_CLIENT_KEY_FILE',
+                                                ".key/postgres-client-key.pem")
+GCSQL_POSTGRES_PUBLIC_IP_FILE = os.environ.get('GCSQL_POSTGRES_PUBLIC_IP_FILE',
+                                               ".key/postgres-ip.env")
+GCSQL_POSTGRES_USER = os.environ.get('GCSQL_POSTGRES_USER', 'postgres_user')
+GCSQL_POSTGRES_DATABASE_NAME = os.environ.get('GCSQL_POSTGRES_DATABASE_NAME',
+                                              'postgresdb')
+GCSQL_MYSQL_CLIENT_CERT_FILE = os.environ.get('GCSQL_MYSQL_CLIENT_CERT_FILE',
+                                              ".key/mysql-client-cert.pem")
+GCSQL_MYSQL_CLIENT_KEY_FILE = os.environ.get('GCSQL_MYSQL_CLIENT_KEY_FILE',
+                                             ".key/mysql-client-key.pem")
+GCSQL_MYSQL_SERVER_CA_FILE = os.environ.get('GCSQL_MYSQL_SERVER_CA_FILE',
+                                            ".key/mysql-server-ca.pem")
+GCSQL_MYSQL_PUBLIC_IP_FILE = os.environ.get('GCSQL_MYSQL_PUBLIC_IP_FILE',
+                                            ".key/mysql-ip.env")
+GCSQL_MYSQL_USER = os.environ.get('GCSQL_MYSQL_USER', 'mysql_user')
+GCSQL_MYSQL_DATABASE_NAME = os.environ.get('GCSQL_MYSQL_DATABASE_NAME', 'mysqldb')
+
+GCSQL_MYSQL_EXPORT_URI = os.environ.get('GCSQL_MYSQL_EXPORT_URI',
+                                        'gs://bucketName/fileName')
+DB_VERSION_MYSQL = 'MYSQL_5_7'
+DV_VERSION_POSTGRES = 'POSTGRES_9_6'
+
+HOME_DIR = expanduser("~")
+
+
+def get_absolute_path(path):
+    if path.startswith("/"):
+        return path
+    else:
+        return os.path.join(HOME_DIR, path)
+
+
+server_ca_file_postgres = get_absolute_path(GCSQL_POSTGRES_SERVER_CA_FILE)
+client_cert_file_postgres = get_absolute_path(GCSQL_POSTGRES_CLIENT_CERT_FILE)
+client_key_file_postgres = get_absolute_path(GCSQL_POSTGRES_CLIENT_KEY_FILE)
+
+server_ca_file_mysql = get_absolute_path(GCSQL_MYSQL_SERVER_CA_FILE)
+client_cert_file_mysql = get_absolute_path(GCSQL_MYSQL_CLIENT_CERT_FILE)
+client_key_file_mysql = get_absolute_path(GCSQL_MYSQL_CLIENT_KEY_FILE)
+
+
+def get_postgres_instance_name(instance_suffix=''):
+    return os.environ.get('GCSQL_POSTGRES_INSTANCE_NAME' + instance_suffix,
+                          'testpostgres')
+
+
+def get_mysql_instance_name(instance_suffix=''):
+    return os.environ.get('GCSQL_MYSQL_INSTANCE_NAME' + instance_suffix,
+                          'testmysql')
+
+
+class CloudSqlQueryTestHelper(LoggingCommandExecutor):
+
+    def create_instances(self, instance_suffix=''):
+        thread_mysql = Thread(target=lambda: self.__create_instance(
+            get_mysql_instance_name(instance_suffix), DB_VERSION_MYSQL))
+        thread_postgres = Thread(target=lambda: self.__create_instance(
+            get_postgres_instance_name(instance_suffix), DV_VERSION_POSTGRES))
+        thread_mysql.start()
+        thread_postgres.start()
+        thread_mysql.join()
+        thread_postgres.join()
+
+    def delete_instances(self, instance_suffix=''):
+        thread_mysql = Thread(target=lambda: self.__delete_instance(
+            get_mysql_instance_name(instance_suffix)))
+        thread_postgres = Thread(target=lambda: self.__delete_instance(
+            get_postgres_instance_name(instance_suffix)))
+        thread_mysql.start()
+        thread_postgres.start()
+        thread_mysql.join()
+        thread_postgres.join()
+
+    def get_ip_addresses(self, instance_suffix):
+        with open(GCSQL_MYSQL_PUBLIC_IP_FILE, "w") as f:
+            ip = self.__get_ip_address(get_mysql_instance_name(instance_suffix),
+                                       'GCSQL_MYSQL_PUBLIC_IP')
+            f.write(ip)
+        with open(GCSQL_POSTGRES_PUBLIC_IP_FILE, "w") as f:
+            ip = self.__get_ip_address(get_postgres_instance_name(instance_suffix),
+                                       'GCSQL_POSTGRES_PUBLIC_IP')
+            f.write(ip)
+
+    def raise_database_exception(self, database):
+        raise Exception("The {database} instance does not exist. Make sure to run  "
+                        "`python {f} --action=before-tests` before running the test"
+                        " (and remember to run `python {f} --action=after-tests` "
+                        "after you are done."
+                        .format(f=__file__, database=database))
+
+    def check_if_instances_are_up(self, instance_suffix=''):
+        res_postgres = self.execute_cmd(
+            ['gcloud', 'sql', 'instances', 'describe',
+             get_postgres_instance_name(instance_suffix),
+             "--project={}".format(GCP_PROJECT_ID)])
+        if res_postgres != 0:
+            self.raise_database_exception('postgres')
+        res_postgres = self.execute_cmd(
+            ['gcloud', 'sql', 'instances', 'describe',
+             get_postgres_instance_name(instance_suffix),
+             "--project={}".format(GCP_PROJECT_ID)])
+        if res_postgres != 0:
+            self.raise_database_exception('mysql')
+
+    def authorize_address(self, instance_suffix=''):
+        ip = self.__get_my_public_ip()
+        self.log.info('Authorizing access from IP: %s', ip)
+        postgres_thread = Thread(target=lambda: self.execute_cmd(
+            ['gcloud', 'sql', 'instances', 'patch',
+             get_postgres_instance_name(instance_suffix), '--quiet',
+             "--authorized-networks={}".format(ip),
+             "--project={}".format(GCP_PROJECT_ID)]))
+        mysql_thread = Thread(target=lambda: self.execute_cmd(
+            ['gcloud', 'sql', 'instances', 'patch',
+             get_mysql_instance_name(instance_suffix), '--quiet',
+             "--authorized-networks={}".format(ip),
+             "--project={}".format(GCP_PROJECT_ID)]))
+        postgres_thread.start()
+        mysql_thread.start()
+        postgres_thread.join()
+        mysql_thread.join()
+
+    def setup_instances(self, instance_suffix=''):
+        mysql_thread = Thread(target=lambda: self.__setup_instance_and_certs(
+            get_mysql_instance_name(instance_suffix), DB_VERSION_MYSQL,
+            server_ca_file_mysql,
+            client_key_file_mysql, client_cert_file_mysql, GCSQL_MYSQL_DATABASE_NAME,
+            GCSQL_MYSQL_USER
+        ))
+        postgres_thread = Thread(target=lambda: self.__setup_instance_and_certs(
+            get_postgres_instance_name(instance_suffix), DV_VERSION_POSTGRES,
+            server_ca_file_postgres,
+            client_key_file_postgres, client_cert_file_postgres,
+            GCSQL_POSTGRES_DATABASE_NAME, GCSQL_POSTGRES_USER
+        ))
+        mysql_thread.start()
+        postgres_thread.start()
+        mysql_thread.join()
+        postgres_thread.join()
+        self.get_ip_addresses(instance_suffix)
+        self.authorize_address(instance_suffix)
+
+    def delete_service_account_acls(self):
+        self.__delete_service_accounts_acls()
+
+    def __create_instance(self, instance_name, db_version):
+        self.log.info('Creating a test %s instance "%s"...', db_version, instance_name)
+        try:
+            create_instance_opcode = self.__create_sql_instance(instance_name, db_version)
+            if create_instance_opcode:  # return code 1, some error occurred
+                operation_name = self.__get_operation_name(instance_name)
+                self.log.info('Waiting for operation: %s ...', operation_name)
+                self.__wait_for_create(operation_name)
+                self.log.info('... Done.')
+
+            self.log.info('... Done creating a test %s instance "%s"!\n',
+                          db_version, instance_name)
+        except Exception as ex:
+            self.log.error('Exception occurred. '
+                           'Aborting creating a test instance.\n\n%s', ex)
+            raise ex
+
+    def __delete_service_accounts_acls(self):
+        export_bucket_split = urlsplit(GCSQL_MYSQL_EXPORT_URI)
+        export_bucket_name = export_bucket_split[1]  # netloc (bucket)
+        self.log.info('Deleting temporary service accounts from bucket "%s"...',
+                      export_bucket_name)
+        all_permissions = self.check_output(['gsutil', 'iam', 'get',
+                                             "gs://{}".format(export_bucket_name),
+                                             "--project={}".format(GCP_PROJECT_ID)])
+        all_permissions_dejson = json.loads(all_permissions.decode("utf-8"))
+        for binding in all_permissions_dejson['bindings']:
+            if binding['role'] == 'roles/storage.legacyBucketWriter':
+                for member in binding['members']:
+                    if not member.startswith('serviceAccount:gcp-storage-account'):
+
+                        self.log.info("Remove member: {}".format(member))
+                        member_type, member_email = member.split(':')
+                        if member_type != 'serviceAccount':
+                            self.log.warning("Skip removing member {} as the type {} is "
+                                             "not service account".format(member,
+                                                                          member_type))
+                        self.execute_cmd(['gsutil', 'acl', 'ch', '-d', member_email,
+                                         "gs://{}".format(export_bucket_name)])
+                    else:
+                        self.log.info("Skip removing member {}".format(member))
+
+    @staticmethod
+    def set_ip_addresses_in_env():
+        CloudSqlQueryTestHelper.__set_ip_address_in_env(GCSQL_MYSQL_PUBLIC_IP_FILE)
+        CloudSqlQueryTestHelper.__set_ip_address_in_env(GCSQL_POSTGRES_PUBLIC_IP_FILE)
+
+    @staticmethod
+    def __set_ip_address_in_env(file_name):
+        if os.path.exists(file_name):
+            with open(file_name, "r") as f:
+                env, ip = f.read().split("=")
+                os.environ[env] = ip
+
+    def __setup_instance_and_certs(self, instance_name, db_version, server_ca_file,
+                                   client_key_file, client_cert_file, db_name,
+                                   db_username):
+        self.log.info('Setting up a test %s instance "%s"...', db_version, instance_name)
+        try:
+            self.__remove_keys_and_certs([server_ca_file, client_key_file,
+                                          client_cert_file])
+
+            self.__wait_for_operations(instance_name)
+            self.__write_to_file(server_ca_file, self.__get_server_ca_cert(instance_name))
+            client_cert_name = 'client-cert-name'
+            self.__wait_for_operations(instance_name)
+            self.__delete_client_cert(instance_name, client_cert_name)
+            self.__wait_for_operations(instance_name)
+            self.__create_client_cert(instance_name, client_key_file, client_cert_name)
+            self.__wait_for_operations(instance_name)
+            self.__write_to_file(client_cert_file,
+                                 self.__get_client_cert(instance_name, client_cert_name))
+            self.__wait_for_operations(instance_name)
+            self.__wait_for_operations(instance_name)
+            self.__create_user(instance_name, db_username)
+            self.__wait_for_operations(instance_name)
+            self.__delete_db(instance_name, db_name)
+            self.__create_db(instance_name, db_name)
+            self.log.info('... Done setting up a test %s instance "%s"!\n',
+                          db_version, instance_name)
+        except Exception as ex:
+            self.log.error('Exception occurred. '
+                           'Aborting setting up test instance and certs.\n\n%s', ex)
+            raise ex
+
+    def __delete_instance(self, instance_name):
+        # type: (str) -> None
+        self.log.info('Deleting Cloud SQL instance "%s"...', instance_name)
+        self.execute_cmd(['gcloud', 'sql', 'instances', 'delete',
+                          instance_name, '--quiet'])
+        self.log.info('... Done.')
+
+    def __get_my_public_ip(self):
+        return self.check_output(
+            ['curl', 'https://ipinfo.io/ip']).decode('utf-8').strip()
+
+    def __create_sql_instance(self, instance_name, db_version):
+        # type: (str, str) -> int
+        return self.execute_cmd(
+            ['gcloud', 'sql', 'instances', 'create', instance_name,
+             '--region', GCP_LOCATION,
+             '--project', GCP_PROJECT_ID,
+             '--database-version', db_version,
+             '--tier', 'db-f1-micro'])
+
+    def __get_server_ca_cert(self, instance_name):
+        # type: (str) -> bytes
+        self.log.info('Getting server CA cert for "%s"...', instance_name)
+        output = self.check_output(
+            ['gcloud', 'sql', 'instances', 'describe', instance_name,
+             '--format=value(serverCaCert.cert)'])
+        self.log.info('... Done.')
+        return output
+
+    def __get_client_cert(self, instance_name, client_cert_name):
+        # type: (str, str) -> bytes
+        self.log.info('Getting client cert for "%s"...', instance_name)
+        output = self.check_output(
+            ['gcloud', 'sql', 'ssl', 'client-certs', 'describe', client_cert_name, '-i',
+             instance_name, '--format=get(cert)'])
+        self.log.info('... Done.')
+        return output
+
+    def __create_user(self, instance_name, username):
+        # type: (str, str) -> None
+        self.log.info('Creating user "%s" in Cloud SQL instance "%s"...', username,
+                      instance_name)
+        self.execute_cmd(['gcloud', 'sql', 'users', 'create', username, '-i',
+                          instance_name, '--host', '%', '--password', 'JoxHlwrPzwch0gz9',
+                          '--quiet'])
+        self.log.info('... Done.')
+
+    def __delete_db(self, instance_name, db_name):
+        # type: (str, str) -> None
+        self.log.info('Deleting database "%s" in Cloud SQL instance "%s"...', db_name,
+                      instance_name)
+        self.execute_cmd(['gcloud', 'sql', 'databases', 'delete', db_name, '-i',
+                          instance_name, '--quiet'])
+        self.log.info('... Done.')
+
+    def __create_db(self, instance_name, db_name):
+        # type: (str, str) -> None
+        self.log.info('Creating database "%s" in Cloud SQL instance "%s"...', db_name,
+                      instance_name)
+        self.execute_cmd(['gcloud', 'sql', 'databases', 'create', db_name, '-i',
+                          instance_name, '--quiet'])
+        self.log.info('... Done.')
+
+    def __write_to_file(self, filepath, content):
+        # type: (str, bytes) -> None
+        # https://stackoverflow.com/a/12517490
+        self.log.info("Checking file under: %s", filepath)
+        if not os.path.exists(os.path.dirname(filepath)):
+            self.log.info("File doesn't exits. Creating dir...")
+            try:
+                os.makedirs(os.path.dirname(filepath))
+            except OSError as exc:  # Guard against race condition
+                self.log.info("Error while creating dir.")
+                if exc.errno != errno.EEXIST:
+                    raise
+        self.log.info("... Done. Dir created.")
+
+        with open(filepath, "w") as f:
+            f.write(str(content.decode('utf-8')))
+        self.log.info('Written file in: %s', filepath)
+
+    def __remove_keys_and_certs(self, filepaths):
+        if not len(filepaths):
+            return
+        self.log.info('Removing client keys and certs...')
+
+        for filepath in filepaths:
+            if os.path.exists(filepath):
+                os.remove(filepath)
+        self.log.info('Done ...')
+
+    def __delete_client_cert(self, instance_name, common_name):
+        self.log.info('Deleting client key and cert for "%s"...', instance_name)
+        self.execute_cmd(['gcloud', 'sql', 'ssl', 'client-certs', 'delete', common_name,
+                          '-i', instance_name, '--quiet'])
+        self.log.info('... Done.')
+
+    def __create_client_cert(self, instance_name, client_key_file, common_name):
+        self.log.info('Creating client key and cert for "%s"...', instance_name)
+        try:
+            os.remove(client_key_file)
+        except OSError:
+            pass
+        self.execute_cmd(['gcloud', 'sql', 'ssl', 'client-certs', 'create', common_name,
+                          client_key_file, '-i', instance_name])
+        self.log.info('... Done.')
+
+    def __get_operation_name(self, instance_name):
+        # type: (str) -> str
+        op_name_bytes = self.check_output(
+            ['gcloud', 'sql', 'operations', 'list', '-i',
+             instance_name, '--format=get(name)'])
+        return op_name_bytes.decode('utf-8').strip()
+
+    def __print_operations(self, operations):
+        self.log.info("\n==== OPERATIONS >>>>")
+        self.log.info(operations)
+        self.log.info("<<<< OPERATIONS ====\n")
+
+    def __wait_for_operations(self, instance_name):
+        # type: (str) -> None
+        while True:
+            operations = self.__get_operations(instance_name)
+            self.__print_operations(operations)
+            if "RUNNING" in operations:
+                self.log.info("Found a running operation. Sleeping 5s before retrying...")
+                time.sleep(5)
+            else:
+                break
+
+    def __get_ip_address(self, instance_name, env_var):
+        # type: (str, str) -> str
+        ip = self.check_output(
+            ['gcloud', 'sql', 'instances', 'describe',
+             instance_name,
+             '--format=get(ipAddresses[0].ipAddress)']
+        ).decode('utf-8').strip()
+        os.environ[env_var] = ip
+        return "{}={}".format(env_var, ip)
+
+    def __get_operations(self, instance_name):
+        # type: (str) -> str
+        op_name_bytes = self.check_output(
+            ['gcloud', 'sql', 'operations', 'list', '-i',
+             instance_name, '--format=get(NAME,TYPE,STATUS)'])
+        return op_name_bytes.decode('utf-8').strip()
+
+    def __wait_for_create(self, operation_name):
+        # type: (str) -> None
+        self.execute_cmd(['gcloud', 'beta', 'sql', 'operations', 'wait',
+                          '--project', GCP_PROJECT_ID, operation_name])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Create or delete Cloud SQL instances for system tests.')
+    parser.add_argument('--action', required=True,
+                        choices=('create', 'delete', 'setup-instances',
+                                 'create2', 'delete2', 'setup-instances2',
+                                 'before-tests', 'after-tests',
+                                 'delete-service-accounts-acls'))
+    action = parser.parse_args().action
+
+    helper = CloudSqlQueryTestHelper()
+    gcp_authenticator = GcpAuthenticator(GCP_CLOUDSQL_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            pass
+        elif action == 'after-tests':
+            pass
+        elif action == 'create':
+            helper.create_instances()
+        elif action == 'delete':
+            helper.delete_instances()
+        elif action == 'create2':
+            helper.create_instances(instance_suffix="2")
+        elif action == 'delete2':
+            helper.delete_instances(instance_suffix="2")
+        elif action == 'setup-instances':
+            helper.setup_instances()
+        elif action == 'setup-instances2':
+            helper.setup_instances(instance_suffix="2")
+        elif action == 'delete-service-accounts-acls':
+            helper.delete_service_account_acls()
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcp_sql_operatorquery_system.py b/tests/contrib/operators/test_gcp_sql_operatorquery_system.py
new file mode 100644
index 0000000000000..4e216662f6c6c
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_sql_operatorquery_system.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+import random
+import string
+import unittest
+from os.path import dirname
+
+import time
+
+from airflow import AirflowException
+from airflow.contrib.hooks.gcp_sql_hook import CloudSqlProxyRunner
+from tests.contrib.utils.base_gcp_system_test_case import BaseGcpSystemTestCase, \
+    DagGcpSystemTestCase
+from tests.contrib.operators.test_gcp_sql_operator_system_helper import \
+    CloudSqlQueryTestHelper
+from tests.contrib.utils.gcp_authenticator import GCP_CLOUDSQL_KEY
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'project-id')
+
+SKIP_CLOUDSQL_QUERY_WARNING = """
+    This test is skipped from automated runs intentionally
+    as creating databases in Google Cloud SQL takes a very
+    long time. You can still set GCP_ENABLE_CLOUDSQL_QUERY_TEST
+    environment variable to 'True' and then you should be able to
+    run it manually after you create the database
+    Creating the database can be done by running
+    `{}/test_gcp_sql_operatorquery_system_helper.py \
+--action=before-tests`
+    (you should remember to delete the database with --action=after-tests flag)
+""".format(dirname(__file__))
+
+GCP_ENABLE_CLOUDSQL_QUERY_TEST = os.environ.get('GCP_ENABLE_CLOUDSQL_QUERY_TEST')
+
+if GCP_ENABLE_CLOUDSQL_QUERY_TEST == 'True':
+    enable_cloudsql_query_test = True
+else:
+    enable_cloudsql_query_test = False
+
+
+SQL_QUERY_TEST_HELPER = CloudSqlQueryTestHelper()
+
+
+@unittest.skipIf(not enable_cloudsql_query_test, SKIP_CLOUDSQL_QUERY_WARNING)
+class CloudSqlProxySystemTest(BaseGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(CloudSqlProxySystemTest, self).__init__(
+            method_name,
+            gcp_key='gcp_cloudsql.json')
+
+    def setUp(self):
+        super(CloudSqlProxySystemTest, self).setUp()
+        self.gcp_authenticator.gcp_authenticate()
+        SQL_QUERY_TEST_HELPER.check_if_instances_are_up(instance_suffix="_QUERY")
+        self.gcp_authenticator.gcp_revoke_authentication()
+
+    @staticmethod
+    def generate_unique_path():
+        return ''.join(
+            random.choice(string.ascii_letters + string.digits) for _ in range(8))
+
+    def test_start_proxy_fail_no_parameters(self):
+        runner = CloudSqlProxyRunner(path_prefix='/tmp/' + self.generate_unique_path(),
+                                     project_id=GCP_PROJECT_ID,
+                                     instance_specification='a')
+        with self.assertRaises(AirflowException) as cm:
+            runner.start_proxy()
+        err = cm.exception
+        self.assertIn("invalid instance name", str(err))
+        with self.assertRaises(AirflowException) as cm:
+            runner.start_proxy()
+        err = cm.exception
+        self.assertIn("invalid instance name", str(err))
+        self.assertIsNone(runner.sql_proxy_process)
+
+    def test_start_proxy_with_all_instances(self):
+        runner = CloudSqlProxyRunner(path_prefix='/tmp/' + self.generate_unique_path(),
+                                     project_id=GCP_PROJECT_ID,
+                                     instance_specification='')
+        try:
+            runner.start_proxy()
+            time.sleep(1)
+        finally:
+            runner.stop_proxy()
+        self.assertIsNone(runner.sql_proxy_process)
+
+    def test_start_proxy_with_all_instances_generated_credential_file(self):
+        self.gcp_authenticator.set_dictionary_in_airflow_connection()
+        runner = CloudSqlProxyRunner(path_prefix='/tmp/' + self.generate_unique_path(),
+                                     project_id=GCP_PROJECT_ID,
+                                     instance_specification='')
+        try:
+            runner.start_proxy()
+            time.sleep(1)
+        finally:
+            runner.stop_proxy()
+        self.assertIsNone(runner.sql_proxy_process)
+
+    def test_start_proxy_with_all_instances_specific_version(self):
+        runner = CloudSqlProxyRunner(path_prefix='/tmp/' + self.generate_unique_path(),
+                                     project_id=GCP_PROJECT_ID,
+                                     instance_specification='',
+                                     sql_proxy_version='v1.13')
+        try:
+            runner.start_proxy()
+            time.sleep(1)
+        finally:
+            runner.stop_proxy()
+        self.assertIsNone(runner.sql_proxy_process)
+        self.assertEqual(runner.get_proxy_version(), "1.13")
+
+
+@unittest.skipIf(not enable_cloudsql_query_test, SKIP_CLOUDSQL_QUERY_WARNING)
+class CloudSqlQueryExampleDagsSystemTest(DagGcpSystemTestCase):
+
+    def __init__(self, method_name='runTest'):
+        super(CloudSqlQueryExampleDagsSystemTest, self).__init__(
+            method_name,
+            dag_id='example_gcp_sql_query',
+            gcp_key=GCP_CLOUDSQL_KEY)
+
+    def setUp(self):
+        super(CloudSqlQueryExampleDagsSystemTest, self).setUp()
+        self.gcp_authenticator.gcp_authenticate()
+        SQL_QUERY_TEST_HELPER.check_if_instances_are_up(instance_suffix="_QUERY")
+        SQL_QUERY_TEST_HELPER.setup_instances(instance_suffix="_QUERY")
+        self.gcp_authenticator.gcp_revoke_authentication()
+
+    def test_run_example_dag_cloudsql_query(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_sql_operatorquery_system_helper.py b/tests/contrib/operators/test_gcp_sql_operatorquery_system_helper.py
new file mode 100755
index 0000000000000..d8bc5164c9dce
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_sql_operatorquery_system_helper.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+
+from tests.contrib.operators.test_gcp_sql_operator_system_helper \
+    import CloudSqlQueryTestHelper
+from tests.contrib.utils.gcp_authenticator import GCP_CLOUDSQL_KEY, GcpAuthenticator
+
+QUERY_SUFFIX = "_QUERY"
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Create or delete Cloud SQL instances for system tests.')
+    parser.add_argument('--action', required=True,
+                        choices=('create', 'delete', 'setup-instances',
+                                 'before-tests', 'after-tests'))
+    action = parser.parse_args().action
+
+    helper = CloudSqlQueryTestHelper()
+    gcp_authenticator = GcpAuthenticator(gcp_key=GCP_CLOUDSQL_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            helper.create_instances(instance_suffix=QUERY_SUFFIX)
+            helper.setup_instances(instance_suffix=QUERY_SUFFIX)
+        elif action == 'after-tests':
+            helper.delete_instances(instance_suffix=QUERY_SUFFIX)
+        elif action == 'create':
+            helper.create_instances(instance_suffix=QUERY_SUFFIX)
+        elif action == 'delete':
+            helper.delete_instances(instance_suffix=QUERY_SUFFIX)
+        elif action == 'setup-instances':
+            helper.setup_instances(instance_suffix=QUERY_SUFFIX)
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcp_transfer_operator.py b/tests/contrib/operators/test_gcp_transfer_operator.py
new file mode 100644
index 0000000000000..67fd4649fdfbe
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_transfer_operator.py
@@ -0,0 +1,748 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import itertools
+import unittest
+from copy import deepcopy
+from datetime import date, time
+from typing import Dict
+
+from parameterized import parameterized
+from botocore.credentials import Credentials
+
+from airflow import AirflowException, configuration
+from airflow.contrib.hooks.gcp_transfer_hook import (
+    FILTER_JOB_NAMES,
+    SCHEDULE_START_DATE,
+    SCHEDULE_END_DATE,
+    START_TIME_OF_DAY,
+    STATUS,
+    NAME,
+    AWS_S3_DATA_SOURCE,
+    GCS_DATA_SOURCE,
+    GCS_DATA_SINK,
+    AWS_ACCESS_KEY,
+    ACCESS_KEY_ID,
+    SECRET_ACCESS_KEY,
+    BUCKET_NAME,
+    SCHEDULE,
+    TRANSFER_SPEC,
+    HTTP_DATA_SOURCE,
+    LIST_URL,
+)
+from airflow.contrib.operators.gcp_transfer_operator import (
+    GcpTransferServiceOperationCancelOperator,
+    GcpTransferServiceOperationResumeOperator,
+    GcpTransferServiceOperationsListOperator,
+    TransferJobValidator,
+    TransferJobPreprocessor,
+    GcpTransferServiceJobCreateOperator,
+    GcpTransferServiceJobUpdateOperator,
+    GcpTransferServiceOperationGetOperator,
+    GcpTransferServiceOperationPauseOperator,
+    S3ToGoogleCloudStorageTransferOperator,
+    GoogleCloudStorageToGoogleCloudStorageTransferOperator,
+    GcpTransferServiceJobDeleteOperator,
+)
+from airflow.models import TaskInstance, DAG
+from airflow.utils import timezone
+from tests.compat import mock
+
+try:
+    import boto3
+except ImportError:  # pragma: no cover
+    boto3 = None
+
+GCP_PROJECT_ID = 'project-id'
+TASK_ID = 'task-id'
+
+JOB_NAME = "job-name"
+OPERATION_NAME = "operation-name"
+AWS_BUCKET_NAME = "aws-bucket-name"
+GCS_BUCKET_NAME = "gcp-bucket-name"
+DESCRIPTION = "description"
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+TEST_FILTER = {FILTER_JOB_NAMES: [JOB_NAME]}
+
+TEST_AWS_ACCESS_KEY_ID = "test-key-1"
+TEST_AWS_ACCESS_SECRET = "test-secret-1"
+TEST_AWS_ACCESS_KEY = {ACCESS_KEY_ID: TEST_AWS_ACCESS_KEY_ID, SECRET_ACCESS_KEY: TEST_AWS_ACCESS_SECRET}
+
+NATIVE_DATE = date(2018, 10, 15)
+DICT_DATE = {'day': 15, 'month': 10, 'year': 2018}
+NATIVE_TIME = time(hour=11, minute=42, second=43)
+DICT_TIME = {'hours': 11, 'minutes': 42, 'seconds': 43}
+SCHEDULE_NATIVE = {
+    SCHEDULE_START_DATE: NATIVE_DATE,
+    SCHEDULE_END_DATE: NATIVE_DATE,
+    START_TIME_OF_DAY: NATIVE_TIME,
+}
+
+SCHEDULE_DICT = {
+    SCHEDULE_START_DATE: {'day': 15, 'month': 10, 'year': 2018},
+    SCHEDULE_END_DATE: {'day': 15, 'month': 10, 'year': 2018},
+    START_TIME_OF_DAY: {'hours': 11, 'minutes': 42, 'seconds': 43},
+}
+
+SOURCE_AWS = {AWS_S3_DATA_SOURCE: {BUCKET_NAME: AWS_BUCKET_NAME}}
+SOURCE_GCS = {GCS_DATA_SOURCE: {BUCKET_NAME: GCS_BUCKET_NAME}}
+SOURCE_HTTP = {HTTP_DATA_SOURCE: {LIST_URL: "http://example.com"}}
+
+VALID_TRANSFER_JOB_BASE = {
+    NAME: JOB_NAME,
+    DESCRIPTION: DESCRIPTION,
+    STATUS: 'ENABLED',
+    SCHEDULE: SCHEDULE_DICT,
+    TRANSFER_SPEC: {GCS_DATA_SINK: {BUCKET_NAME: GCS_BUCKET_NAME}},
+}  # type: Dict
+VALID_TRANSFER_JOB_GCS = deepcopy(VALID_TRANSFER_JOB_BASE)
+VALID_TRANSFER_JOB_GCS[TRANSFER_SPEC].update(deepcopy(SOURCE_GCS))
+VALID_TRANSFER_JOB_AWS = deepcopy(VALID_TRANSFER_JOB_BASE)
+VALID_TRANSFER_JOB_AWS[TRANSFER_SPEC].update(deepcopy(SOURCE_AWS))
+
+VALID_TRANSFER_JOB_GCS = {
+    NAME: JOB_NAME,
+    DESCRIPTION: DESCRIPTION,
+    STATUS: 'ENABLED',
+    SCHEDULE: SCHEDULE_NATIVE,
+    TRANSFER_SPEC: {
+        GCS_DATA_SOURCE: {BUCKET_NAME: GCS_BUCKET_NAME},
+        GCS_DATA_SINK: {BUCKET_NAME: GCS_BUCKET_NAME},
+    },
+}
+
+VALID_TRANSFER_JOB_RAW = {
+    DESCRIPTION: DESCRIPTION,
+    STATUS: 'ENABLED',
+    SCHEDULE: SCHEDULE_DICT,
+    TRANSFER_SPEC: {GCS_DATA_SINK: {BUCKET_NAME: GCS_BUCKET_NAME}},
+}  # type: Dict
+
+VALID_TRANSFER_JOB_GCS_RAW = deepcopy(VALID_TRANSFER_JOB_RAW)
+VALID_TRANSFER_JOB_GCS_RAW[TRANSFER_SPEC].update(SOURCE_GCS)
+VALID_TRANSFER_JOB_AWS_RAW = deepcopy(VALID_TRANSFER_JOB_RAW)
+VALID_TRANSFER_JOB_AWS_RAW[TRANSFER_SPEC].update(deepcopy(SOURCE_AWS))
+VALID_TRANSFER_JOB_AWS_RAW[TRANSFER_SPEC][AWS_S3_DATA_SOURCE][AWS_ACCESS_KEY] = TEST_AWS_ACCESS_KEY
+
+
+VALID_OPERATION = {NAME: "operation-name"}
+
+
+class TransferJobPreprocessorTest(unittest.TestCase):
+    def test_should_do_nothing_on_empty(self):
+        body = {}
+        TransferJobPreprocessor(body=body).process_body()
+        self.assertEqual(body, {})
+
+    @unittest.skipIf(boto3 is None, "Skipping test because boto3 is not available")
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.AwsHook')
+    def test_should_inject_aws_credentials(self, mock_hook):
+        mock_hook.return_value.get_credentials.return_value = Credentials(
+            TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
+        )
+
+        body = {TRANSFER_SPEC: deepcopy(SOURCE_AWS)}
+        body = TransferJobPreprocessor(body=body).process_body()
+        self.assertEqual(body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE][AWS_ACCESS_KEY], TEST_AWS_ACCESS_KEY)
+
+    @parameterized.expand([(SCHEDULE_START_DATE,), (SCHEDULE_END_DATE,)])
+    def test_should_format_date_from_python_to_dict(self, field_attr):
+        body = {SCHEDULE: {field_attr: NATIVE_DATE}}
+        TransferJobPreprocessor(body=body).process_body()
+        self.assertEqual(body[SCHEDULE][field_attr], DICT_DATE)
+
+    def test_should_format_time_from_python_to_dict(self):
+        body = {SCHEDULE: {START_TIME_OF_DAY: NATIVE_TIME}}
+        TransferJobPreprocessor(body=body).process_body()
+        self.assertEqual(body[SCHEDULE][START_TIME_OF_DAY], DICT_TIME)
+
+    @parameterized.expand([(SCHEDULE_START_DATE,), (SCHEDULE_END_DATE,)])
+    def test_should_not_change_date_for_dict(self, field_attr):
+        body = {SCHEDULE: {field_attr: DICT_DATE}}
+        TransferJobPreprocessor(body=body).process_body()
+        self.assertEqual(body[SCHEDULE][field_attr], DICT_DATE)
+
+    def test_should_not_change_time_for_dict(self):
+        body = {SCHEDULE: {START_TIME_OF_DAY: DICT_TIME}}
+        TransferJobPreprocessor(body=body).process_body()
+        self.assertEqual(body[SCHEDULE][START_TIME_OF_DAY], DICT_TIME)
+
+
+class TransferJobValidatorTest(unittest.TestCase):
+    def test_should_raise_exception_when_encounters_aws_credentials(self):
+        body = {"transferSpec": {"awsS3DataSource": {"awsAccessKey": TEST_AWS_ACCESS_KEY}}}
+        with self.assertRaises(AirflowException) as cm:
+            TransferJobValidator(body=body).validate_body()
+        err = cm.exception
+        self.assertIn(
+            "AWS credentials detected inside the body parameter (awsAccessKey). This is not allowed, please "
+            "use Airflow connections to store credentials.",
+            str(err),
+        )
+
+    def test_should_raise_exception_when_body_empty(self):
+        body = None
+        with self.assertRaises(AirflowException) as cm:
+            TransferJobValidator(body=body).validate_body()
+        err = cm.exception
+        self.assertIn("The required parameter 'body' is empty or None", str(err))
+
+    @parameterized.expand(
+        [
+            (dict(itertools.chain(SOURCE_AWS.items(), SOURCE_GCS.items(), SOURCE_HTTP.items())),),
+            (dict(itertools.chain(SOURCE_AWS.items(), SOURCE_GCS.items())),),
+            (dict(itertools.chain(SOURCE_AWS.items(), SOURCE_HTTP.items())),),
+            (dict(itertools.chain(SOURCE_GCS.items(), SOURCE_HTTP.items())),),
+        ]
+    )
+    def test_verify_data_source(self, transferSpec):
+        body = {TRANSFER_SPEC: transferSpec}
+
+        with self.assertRaises(AirflowException) as cm:
+            TransferJobValidator(body=body).validate_body()
+        err = cm.exception
+        self.assertIn(
+            "More than one data source detected. Please choose exactly one data source from: "
+            "gcsDataSource, awsS3DataSource and httpDataSource.",
+            str(err),
+        )
+
+    @parameterized.expand([(VALID_TRANSFER_JOB_GCS,), (VALID_TRANSFER_JOB_AWS,)])
+    def test_verify_success(self, body):
+
+        TransferJobValidator(body=body).validate_body()
+
+        self.assertTrue(True)
+
+
+class GcpStorageTransferJobCreateOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_job_create_gcs(self, mock_hook):
+        mock_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_GCS_RAW
+        body = deepcopy(VALID_TRANSFER_JOB_GCS)
+        del (body['name'])
+        op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
+        result = op.execute(None)
+
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+
+        mock_hook.return_value.create_transfer_job.assert_called_once_with(body=VALID_TRANSFER_JOB_GCS_RAW)
+
+        self.assertEqual(result, VALID_TRANSFER_JOB_GCS_RAW)
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.AwsHook')
+    def test_job_create_aws(self, aws_hook, mock_hook):
+        mock_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_AWS_RAW
+        aws_hook.return_value.get_credentials.return_value = Credentials(
+            TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
+        )
+        body = deepcopy(VALID_TRANSFER_JOB_AWS)
+        del (body['name'])
+        op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
+
+        result = op.execute(None)
+
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+
+        mock_hook.return_value.create_transfer_job.assert_called_once_with(body=VALID_TRANSFER_JOB_AWS_RAW)
+
+        self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.AwsHook')
+    def test_job_create_multiple(self, aws_hook, gcp_hook):
+        aws_hook.return_value.get_credentials.return_value = Credentials(
+            TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
+        )
+        gcp_hook.return_value.create_transfer_job.return_value = VALID_TRANSFER_JOB_AWS_RAW
+        body = deepcopy(VALID_TRANSFER_JOB_AWS)
+
+        op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
+        result = op.execute(None)
+        self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
+
+        op = GcpTransferServiceJobCreateOperator(body=body, task_id=TASK_ID)
+        result = op.execute(None)
+        self.assertEqual(result, VALID_TRANSFER_JOB_AWS_RAW)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        self.dag = DAG(dag_id, default_args={'start_date': DEFAULT_DATE})
+        op = GcpTransferServiceJobCreateOperator(
+            body={"description": "{{ dag.dag_id }}"},
+            gcp_conn_id='{{ dag.dag_id }}',
+            aws_conn_id='{{ dag.dag_id }}',
+            task_id='task-id',
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'body')[DESCRIPTION])
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'aws_conn_id'))
+
+
+class GcpStorageTransferJobUpdateOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_job_update(self, mock_hook):
+        mock_hook.return_value.update_transfer_job.return_value = VALID_TRANSFER_JOB_GCS
+        body = {'transferJob': {'description': 'example-name'}, 'updateTransferJobFieldMask': DESCRIPTION}
+
+        op = GcpTransferServiceJobUpdateOperator(job_name=JOB_NAME, body=body, task_id=TASK_ID)
+        result = op.execute(None)
+
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.update_transfer_job.assert_called_once_with(job_name=JOB_NAME, body=body)
+        self.assertEqual(result, VALID_TRANSFER_JOB_GCS)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceJobUpdateOperator(
+            job_name='{{ dag.dag_id }}',
+            body={'transferJob': {"name": "{{ dag.dag_id }}"}},
+            task_id='task-id',
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'body')['transferJob']['name'])
+        self.assertEqual(dag_id, getattr(op, 'job_name'))
+
+
+class GcpStorageTransferJobDeleteOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_job_delete(self, mock_hook):
+        op = GcpTransferServiceJobDeleteOperator(
+            job_name=JOB_NAME, project_id=GCP_PROJECT_ID, task_id='task-id'
+        )
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.delete_transfer_job.assert_called_once_with(
+            job_name=JOB_NAME, project_id=GCP_PROJECT_ID
+        )
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_job_delete_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceJobDeleteOperator(
+            job_name='{{ dag.dag_id }}',
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id=TASK_ID,
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'job_name'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_job_delete_should_throw_ex_when_name_none(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GcpTransferServiceJobDeleteOperator(job_name="", task_id='task-id')
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'job_name' is empty or None", str(err))
+        mock_hook.assert_not_called()
+
+
+class GpcStorageTransferOperationsGetOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_get(self, mock_hook):
+        mock_hook.return_value.get_transfer_operation.return_value = VALID_OPERATION
+        op = GcpTransferServiceOperationGetOperator(operation_name=OPERATION_NAME, task_id=TASK_ID)
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.get_transfer_operation.assert_called_once_with(operation_name=OPERATION_NAME)
+        self.assertEqual(result, VALID_OPERATION)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_get_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceOperationGetOperator(
+            operation_name='{{ dag.dag_id }}', task_id='task-id', dag=self.dag
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'operation_name'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_get_should_throw_ex_when_operation_name_none(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GcpTransferServiceOperationGetOperator(operation_name="", task_id=TASK_ID)
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'operation_name' is empty or None", str(err))
+        mock_hook.assert_not_called()
+
+
+class GcpStorageTransferOperationListOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_list(self, mock_hook):
+        mock_hook.return_value.list_transfer_operations.return_value = [VALID_TRANSFER_JOB_GCS]
+        op = GcpTransferServiceOperationsListOperator(filter=TEST_FILTER, task_id=TASK_ID)
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.list_transfer_operations.assert_called_once_with(filter=TEST_FILTER)
+        self.assertEqual(result, [VALID_TRANSFER_JOB_GCS])
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceOperationsListOperator(
+            filter={"job_names": ['{{ dag.dag_id }}']},
+            gcp_conn_id='{{ dag.dag_id }}',
+            task_id='task-id',
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'filter')['job_names'][0])
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+
+
+class GcpStorageTransferOperationsPauseOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_pause(self, mock_hook):
+        op = GcpTransferServiceOperationPauseOperator(operation_name=OPERATION_NAME, task_id='task-id')
+        op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.pause_transfer_operation.assert_called_once_with(operation_name=OPERATION_NAME)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_pause_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceOperationPauseOperator(
+            operation_name='{{ dag.dag_id }}',
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id=TASK_ID,
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'operation_name'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_pause_should_throw_ex_when_name_none(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GcpTransferServiceOperationPauseOperator(operation_name="", task_id='task-id')
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'operation_name' is empty or None", str(err))
+        mock_hook.assert_not_called()
+
+
+class GcpStorageTransferOperationsResumeOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_resume(self, mock_hook):
+        op = GcpTransferServiceOperationResumeOperator(operation_name=OPERATION_NAME, task_id=TASK_ID)
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.resume_transfer_operation.assert_called_once_with(
+            operation_name=OPERATION_NAME
+        )
+        self.assertIsNone(result)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_resume_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceOperationResumeOperator(
+            operation_name='{{ dag.dag_id }}',
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id=TASK_ID,
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'operation_name'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_resume_should_throw_ex_when_name_none(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GcpTransferServiceOperationResumeOperator(operation_name="", task_id=TASK_ID)
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'operation_name' is empty or None", str(err))
+        mock_hook.assert_not_called()
+
+
+class GcpStorageTransferOperationsCancelOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_cancel(self, mock_hook):
+        op = GcpTransferServiceOperationCancelOperator(operation_name=OPERATION_NAME, task_id=TASK_ID)
+        result = op.execute(None)
+        mock_hook.assert_called_once_with(api_version='v1', gcp_conn_id='google_cloud_default')
+        mock_hook.return_value.cancel_transfer_operation.assert_called_once_with(
+            operation_name=OPERATION_NAME
+        )
+        self.assertIsNone(result)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_cancel_with_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GcpTransferServiceOperationCancelOperator(
+            operation_name='{{ dag.dag_id }}',
+            gcp_conn_id='{{ dag.dag_id }}',
+            api_version='{{ dag.dag_id }}',
+            task_id=TASK_ID,
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'operation_name'))
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+        self.assertEqual(dag_id, getattr(op, 'api_version'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_operation_cancel_should_throw_ex_when_name_none(self, mock_hook):
+        with self.assertRaises(AirflowException) as cm:
+            op = GcpTransferServiceOperationCancelOperator(operation_name="", task_id=TASK_ID)
+            op.execute(None)
+        err = cm.exception
+        self.assertIn("The required parameter 'operation_name' is empty or None", str(err))
+        mock_hook.assert_not_called()
+
+
+class S3ToGoogleCloudStorageTransferOperatorTest(unittest.TestCase):
+    def test_constructor(self):
+        operator = S3ToGoogleCloudStorageTransferOperator(
+            task_id=TASK_ID,
+            s3_bucket=AWS_BUCKET_NAME,
+            gcs_bucket=GCS_BUCKET_NAME,
+            project_id=GCP_PROJECT_ID,
+            description=DESCRIPTION,
+            schedule=SCHEDULE_DICT,
+        )
+
+        self.assertEqual(operator.task_id, TASK_ID)
+        self.assertEqual(operator.s3_bucket, AWS_BUCKET_NAME)
+        self.assertEqual(operator.gcs_bucket, GCS_BUCKET_NAME)
+        self.assertEqual(operator.project_id, GCP_PROJECT_ID)
+        self.assertEqual(operator.description, DESCRIPTION)
+        self.assertEqual(operator.schedule, SCHEDULE_DICT)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = S3ToGoogleCloudStorageTransferOperator(
+            s3_bucket='{{ dag.dag_id }}',
+            gcs_bucket='{{ dag.dag_id }}',
+            description='{{ dag.dag_id }}',
+            object_conditions={'exclude_prefixes': ['{{ dag.dag_id }}']},
+            gcp_conn_id='{{ dag.dag_id }}',
+            task_id=TASK_ID,
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 's3_bucket'))
+        self.assertEqual(dag_id, getattr(op, 'gcs_bucket'))
+        self.assertEqual(dag_id, getattr(op, 'description'))
+        self.assertEqual(dag_id, getattr(op, 'object_conditions')['exclude_prefixes'][0])
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.AwsHook')
+    def test_execute(self, mock_aws_hook, mock_transfer_hook):
+        mock_aws_hook.return_value.get_credentials.return_value = Credentials(
+            TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
+        )
+
+        operator = S3ToGoogleCloudStorageTransferOperator(
+            task_id=TASK_ID,
+            s3_bucket=AWS_BUCKET_NAME,
+            gcs_bucket=GCS_BUCKET_NAME,
+            description=DESCRIPTION,
+            schedule=SCHEDULE_DICT,
+        )
+
+        operator.execute(None)
+
+        mock_transfer_hook.return_value.create_transfer_job.assert_called_once_with(
+            body=VALID_TRANSFER_JOB_AWS_RAW
+        )
+
+        self.assertTrue(mock_transfer_hook.return_value.wait_for_transfer_job.called)
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.AwsHook')
+    def test_execute_skip_wait(self, mock_aws_hook, mock_transfer_hook):
+        mock_aws_hook.return_value.get_credentials.return_value = Credentials(
+            TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCESS_SECRET, None
+        )
+
+        operator = S3ToGoogleCloudStorageTransferOperator(
+            task_id=TASK_ID,
+            s3_bucket=AWS_BUCKET_NAME,
+            gcs_bucket=GCS_BUCKET_NAME,
+            description=DESCRIPTION,
+            schedule=SCHEDULE_DICT,
+            wait=False,
+        )
+
+        operator.execute(None)
+
+        mock_transfer_hook.return_value.create_transfer_job.assert_called_once_with(
+            body=VALID_TRANSFER_JOB_AWS_RAW
+        )
+
+        self.assertFalse(mock_transfer_hook.return_value.wait_for_transfer_job.called)
+
+
+class GoogleCloudStorageToGoogleCloudStorageTransferOperatorTest(unittest.TestCase):
+    def test_constructor(self):
+        operator = GoogleCloudStorageToGoogleCloudStorageTransferOperator(
+            task_id=TASK_ID,
+            source_bucket=GCS_BUCKET_NAME,
+            destination_bucket=GCS_BUCKET_NAME,
+            project_id=GCP_PROJECT_ID,
+            description=DESCRIPTION,
+            schedule=SCHEDULE_DICT,
+        )
+
+        self.assertEqual(operator.task_id, TASK_ID)
+        self.assertEqual(operator.source_bucket, GCS_BUCKET_NAME)
+        self.assertEqual(operator.destination_bucket, GCS_BUCKET_NAME)
+        self.assertEqual(operator.project_id, GCP_PROJECT_ID)
+        self.assertEqual(operator.description, DESCRIPTION)
+        self.assertEqual(operator.schedule, SCHEDULE_DICT)
+
+    # Setting all of the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_templates(self, _):
+        dag_id = 'test_dag_id'
+        configuration.load_test_config()
+        args = {'start_date': DEFAULT_DATE}
+        self.dag = DAG(dag_id, default_args=args)
+        op = GoogleCloudStorageToGoogleCloudStorageTransferOperator(
+            source_bucket='{{ dag.dag_id }}',
+            destination_bucket='{{ dag.dag_id }}',
+            description='{{ dag.dag_id }}',
+            object_conditions={'exclude_prefixes': ['{{ dag.dag_id }}']},
+            gcp_conn_id='{{ dag.dag_id }}',
+            task_id=TASK_ID,
+            dag=self.dag,
+        )
+        ti = TaskInstance(op, DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(dag_id, getattr(op, 'source_bucket'))
+        self.assertEqual(dag_id, getattr(op, 'destination_bucket'))
+        self.assertEqual(dag_id, getattr(op, 'description'))
+        self.assertEqual(dag_id, getattr(op, 'object_conditions')['exclude_prefixes'][0])
+        self.assertEqual(dag_id, getattr(op, 'gcp_conn_id'))
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_execute(self, mock_transfer_hook):
+        operator = GoogleCloudStorageToGoogleCloudStorageTransferOperator(
+            task_id=TASK_ID,
+            source_bucket=GCS_BUCKET_NAME,
+            destination_bucket=GCS_BUCKET_NAME,
+            description=DESCRIPTION,
+            schedule=SCHEDULE_DICT,
+        )
+
+        operator.execute(None)
+
+        mock_transfer_hook.return_value.create_transfer_job.assert_called_once_with(
+            body=VALID_TRANSFER_JOB_GCS_RAW
+        )
+        self.assertTrue(mock_transfer_hook.return_value.wait_for_transfer_job.called)
+
+    @mock.patch('airflow.contrib.operators.gcp_transfer_operator.GCPTransferServiceHook')
+    def test_execute_skip_wait(self, mock_transfer_hook):
+        operator = GoogleCloudStorageToGoogleCloudStorageTransferOperator(
+            task_id=TASK_ID,
+            source_bucket=GCS_BUCKET_NAME,
+            destination_bucket=GCS_BUCKET_NAME,
+            description=DESCRIPTION,
+            wait=False,
+            schedule=SCHEDULE_DICT,
+        )
+
+        operator.execute(None)
+
+        mock_transfer_hook.return_value.create_transfer_job.assert_called_once_with(
+            body=VALID_TRANSFER_JOB_GCS_RAW
+        )
+        self.assertFalse(mock_transfer_hook.return_value.wait_for_transfer_job.called)
diff --git a/tests/contrib/operators/test_gcp_transfer_operator_system.py b/tests/contrib/operators/test_gcp_transfer_operator_system.py
new file mode 100644
index 0000000000000..7a94c3762b600
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_transfer_operator_system.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from tests.contrib.operators.test_gcp_transfer_operator_system_helper import GCPTransferTestHelper
+from tests.contrib.utils.base_gcp_system_test_case import SKIP_TEST_WARNING, DagGcpSystemTestCase
+
+from tests.contrib.utils.gcp_authenticator import GCP_GCS_TRANSFER_KEY
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_GCS_TRANSFER_KEY), SKIP_TEST_WARNING)
+class GcpTransferExampleDagsSystemTest(DagGcpSystemTestCase):
+    def setUp(self):
+        super(GcpTransferExampleDagsSystemTest, self).setUp()
+        self.gcp_authenticator.gcp_authenticate()
+        self.helper.create_gcs_buckets()
+        self.gcp_authenticator.gcp_revoke_authentication()
+
+    def tearDown(self):
+        self.gcp_authenticator.gcp_authenticate()
+        self.helper.delete_gcs_buckets()
+        self.gcp_authenticator.gcp_revoke_authentication()
+        super(GcpTransferExampleDagsSystemTest, self).tearDown()
+
+    def __init__(self, method_name='runTest'):
+        super(GcpTransferExampleDagsSystemTest, self).__init__(
+            method_name, dag_id='example_gcp_transfer', gcp_key=GCP_GCS_TRANSFER_KEY
+        )
+        self.helper = GCPTransferTestHelper()
+
+    def test_run_example_dag_compute(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_transfer_operator_system_helper.py b/tests/contrib/operators/test_gcp_transfer_operator_system_helper.py
new file mode 100755
index 0000000000000..b10bfeab08ade
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_transfer_operator_system_helper.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import argparse
+import os
+import subprocess
+
+from googleapiclient._auth import default_credentials, with_scopes
+
+from tests.contrib.utils.base_gcp_system_test_case import RetrieveVariables
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator, GCP_GCS_TRANSFER_KEY
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+from googleapiclient import discovery
+
+
+retrieve_variables = RetrieveVariables()
+
+SERVICE_EMAIL_FORMAT = "project-%s@storage-transfer-service.iam.gserviceaccount.com"
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_TRANSFER_SOURCE_AWS_BUCKET = os.environ.get('GCP_TRANSFER_SOURCE_AWS_BUCKET', 'instance-bucket-test-2')
+GCP_TRANSFER_FIRST_TARGET_BUCKET = os.environ.get(
+    'GCP_TRANSFER_FIRST_TARGET_BUCKET', 'gcp-transfer-first-target'
+)
+GCP_TRANSFER_SECOND_TARGET_BUCKET = os.environ.get(
+    'GCP_TRANSFER_SECOND_TARGET_BUCKET', 'gcp-transfer-second-target'
+)
+
+# 100 MB
+TEST_FILE_SIZE = 1 * 1024 * 1024
+TEST_FILE_NO = 30000
+# Total: 30000 * 1 MB = 30GB
+
+
+class GCPTransferTestHelper(LoggingCommandExecutor):
+    def create_s3_bucket(self):
+        self.execute_cmd(["aws", "s3", "mb", "s3://%s" % GCP_TRANSFER_SOURCE_AWS_BUCKET])
+
+        self.execute_cmd(
+            [
+                "bash",
+                "-c",
+                "cat /dev/urandom | head -c %s | aws s3 cp - "
+                "s3://%s/file-from-aws.bin" % (TEST_FILE_SIZE, GCP_TRANSFER_SOURCE_AWS_BUCKET),
+            ]
+        )
+
+        self.execute_cmd(
+            [
+                "bash",
+                "-c",
+                "seq 1 %s | xargs -n 1 -P 16 -I {} "
+                "aws s3 cp  s3://%s/file-from-aws.bin s3://%s/file-from-aws-{}.bin"
+                % (TEST_FILE_NO, GCP_TRANSFER_SOURCE_AWS_BUCKET, GCP_TRANSFER_SOURCE_AWS_BUCKET),
+            ]
+        )
+
+    def delete_s3_bucket(self):
+        self.execute_cmd(["aws", "s3", "rb", "s3://%s" % GCP_TRANSFER_SOURCE_AWS_BUCKET, "--force"])
+
+    def create_gcs_buckets(self):
+        self.execute_cmd(
+            [
+                'gsutil',
+                'mb',
+                "-p",
+                GCP_PROJECT_ID,
+                "-c",
+                "regional",
+                "-l",
+                "europe-north1",
+                "gs://%s/" % GCP_TRANSFER_FIRST_TARGET_BUCKET,
+            ]
+        )
+
+        self.execute_cmd(
+            [
+                'gsutil',
+                'mb',
+                "-p",
+                GCP_PROJECT_ID,
+                "-c",
+                "regional",
+                "-l",
+                "asia-east1",
+                "gs://%s/" % GCP_TRANSFER_SECOND_TARGET_BUCKET,
+            ]
+        )
+
+        project_number = (
+            subprocess.check_output(
+                ['gcloud', 'projects', 'describe', GCP_PROJECT_ID, '--format', 'value(projectNumber)']
+            )
+            .decode("utf-8")
+            .strip()
+        )
+
+        account_email = SERVICE_EMAIL_FORMAT % project_number
+
+        self.execute_cmd(
+            [
+                "gsutil",
+                "iam",
+                "ch",
+                "serviceAccount:%s:admin" % account_email,
+                "gs://%s" % GCP_TRANSFER_SECOND_TARGET_BUCKET,
+            ]
+        )
+
+        self.execute_cmd(
+            [
+                "gsutil",
+                "iam",
+                "ch",
+                "serviceAccount:%s:admin" % account_email,
+                "gs://%s" % GCP_TRANSFER_FIRST_TARGET_BUCKET,
+            ]
+        )
+
+    def delete_gcs_buckets(self):
+        self.execute_cmd(['gsutil', 'rm', "-r", "gs://%s/" % GCP_TRANSFER_FIRST_TARGET_BUCKET, "&"], True)
+
+        self.execute_cmd(['gsutil', 'rm', "-r", "gs://%s/" % GCP_TRANSFER_SECOND_TARGET_BUCKET, "&"], True)
+
+    @staticmethod
+    def _get_transfer_service_account():
+        credentials = default_credentials()
+        credentials = with_scopes(credentials, scopes=['https://www.googleapis.com/auth/cloud-platform'])
+        service = discovery.build('storagetransfer', 'v1', cache_discovery=False, credentials=credentials)
+
+        request = service.googleServiceAccounts().get(projectId=GCP_PROJECT_ID)
+        return request.execute()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create and delete buckets for system tests.')
+    parser.add_argument(
+        '--action',
+        dest='action',
+        required=True,
+        choices=(
+            'create-s3-bucket',
+            'delete-s3-bucket',
+            'create-gcs-buckets',
+            'delete-gcs-buckets',
+            'before-tests',
+            'after-tests',
+        ),
+    )
+    action = parser.parse_args().action
+
+    helper = GCPTransferTestHelper()
+    gcp_authenticator = GcpAuthenticator(GCP_GCS_TRANSFER_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            pass
+        elif action == 'after-tests':
+            pass
+        elif action == 'create-s3-bucket':
+            helper.create_s3_bucket()
+        elif action == 'delete-s3-bucket':
+            helper.delete_s3_bucket()
+        elif action == 'create-gcs-buckets':
+            helper.create_gcs_buckets()
+        elif action == 'delete-gcs-buckets':
+            helper.delete_gcs_buckets()
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcp_translate_operator.py b/tests/contrib/operators/test_gcp_translate_operator.py
new file mode 100644
index 0000000000000..5e8f245c182c1
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_translate_operator.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow.contrib.operators.gcp_translate_operator import CloudTranslateTextOperator
+from tests.compat import mock
+
+GCP_CONN_ID = 'google_cloud_default'
+
+
+class CloudTranslateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_translate_operator.CloudTranslateHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.translate.return_value = [
+            {
+                'translatedText': 'Yellowing self Gęśle',
+                'detectedSourceLanguage': 'pl',
+                'model': 'base',
+                'input': 'zażółć gęślą jaźń',
+            }
+        ]
+        op = CloudTranslateTextOperator(
+            values=['zażółć gęślą jaźń'],
+            target_language='en',
+            format_='text',
+            source_language=None,
+            model='base',
+            gcp_conn_id=GCP_CONN_ID,
+            task_id='id',
+        )
+        return_value = op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.translate.assert_called_once_with(
+            values=['zażółć gęślą jaźń'],
+            target_language='en',
+            format_='text',
+            source_language=None,
+            model='base',
+        )
+        self.assertEqual(
+            [
+                {
+                    'translatedText': 'Yellowing self Gęśle',
+                    'detectedSourceLanguage': 'pl',
+                    'model': 'base',
+                    'input': 'zażółć gęślą jaźń',
+                }
+            ],
+            return_value,
+        )
diff --git a/tests/contrib/operators/test_gcp_translate_operator_system.py b/tests/contrib/operators/test_gcp_translate_operator_system.py
new file mode 100644
index 0000000000000..70e774230b05c
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_translate_operator_system.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from tests.contrib.utils.base_gcp_system_test_case import DagGcpSystemTestCase, SKIP_TEST_WARNING
+from tests.contrib.utils.gcp_authenticator import GCP_AI_KEY
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_AI_KEY), SKIP_TEST_WARNING)
+class CloudTranslateExampleDagsSystemTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(CloudTranslateExampleDagsSystemTest, self).__init__(
+            method_name, dag_id='example_gcp_translate', gcp_key=GCP_AI_KEY
+        )
+
+    def test_run_example_dag_function(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcp_vision_operator.py b/tests/contrib/operators/test_gcp_vision_operator.py
new file mode 100644
index 0000000000000..37365d9bec10b
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_vision_operator.py
@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from google.api_core.exceptions import AlreadyExists
+from google.cloud.vision_v1.types import ProductSet, Product, ReferenceImage
+
+from airflow.contrib.operators.gcp_vision_operator import (
+    CloudVisionProductSetCreateOperator,
+    CloudVisionProductSetGetOperator,
+    CloudVisionProductSetUpdateOperator,
+    CloudVisionProductSetDeleteOperator,
+    CloudVisionProductCreateOperator,
+    CloudVisionProductGetOperator,
+    CloudVisionProductUpdateOperator,
+    CloudVisionProductDeleteOperator,
+    CloudVisionReferenceImageCreateOperator,
+    CloudVisionAddProductToProductSetOperator,
+    CloudVisionRemoveProductFromProductSetOperator,
+    CloudVisionAnnotateImageOperator,
+)
+
+from tests.compat import mock
+
+PRODUCTSET_TEST = ProductSet(display_name='Test Product Set')
+PRODUCTSET_ID_TEST = 'my-productset'
+PRODUCT_TEST = Product(display_name='My Product 1', product_category='toys')
+PRODUCT_ID_TEST = 'my-product'
+REFERENCE_IMAGE_TEST = ReferenceImage(uri='gs://bucket_name/file.txt')
+REFERENCE_IMAGE_ID_TEST = 'my-reference-image'
+ANNOTATE_REQUEST_TEST = {'image': {'source': {'image_uri': 'https://foo.com/image.jpg'}}}
+LOCATION_TEST = 'europe-west1'
+GCP_CONN_ID = 'google_cloud_default'
+
+
+class CloudVisionProductSetCreateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.create_product_set.return_value = {}
+        op = CloudVisionProductSetCreateOperator(
+            location=LOCATION_TEST, product_set=PRODUCTSET_TEST, task_id='id'
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.create_product_set.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_set=PRODUCTSET_TEST,
+            product_set_id=None,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook.get_conn')
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook.create_product_set')
+    def test_already_exists(self, create_product_set_mock, get_conn):
+        get_conn.return_value = {}
+        create_product_set_mock.side_effect = AlreadyExists(message='')
+        # Exception AlreadyExists not raised, caught in the operator's execute() - idempotence
+        op = CloudVisionProductSetCreateOperator(
+            location=LOCATION_TEST,
+            product_set=PRODUCTSET_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            project_id='mock-project-id',
+            task_id='id',
+        )
+        result = op.execute(None)
+        self.assertEqual(PRODUCTSET_ID_TEST, result)
+
+
+class CloudVisionProductSetUpdateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.update_product_set.return_value = {}
+        op = CloudVisionProductSetUpdateOperator(
+            location=LOCATION_TEST, product_set=PRODUCTSET_TEST, task_id='id'
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.update_product_set.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_set=PRODUCTSET_TEST,
+            product_set_id=None,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+            update_mask=None,
+        )
+
+
+class CloudVisionProductSetGetTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.get_product_set.return_value = {}
+        op = CloudVisionProductSetGetOperator(
+            location=LOCATION_TEST, product_set_id=PRODUCTSET_ID_TEST, task_id='id'
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.get_product_set.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionProductSetDeleteTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.delete_product_set.return_value = {}
+        op = CloudVisionProductSetDeleteOperator(
+            location=LOCATION_TEST, product_set_id=PRODUCTSET_ID_TEST, task_id='id'
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.delete_product_set.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionProductCreateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.create_product.return_value = {}
+        op = CloudVisionProductCreateOperator(location=LOCATION_TEST, product=PRODUCT_TEST, task_id='id')
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.create_product.assert_called_once_with(
+            location=LOCATION_TEST,
+            product=PRODUCT_TEST,
+            product_id=None,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook.get_conn')
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook.create_product')
+    def test_already_exists(self, create_product_mock, get_conn):
+        get_conn.return_value = {}
+        create_product_mock.side_effect = AlreadyExists(message='')
+        # Exception AlreadyExists not raised, caught in the operator's execute() - idempotence
+        op = CloudVisionProductCreateOperator(
+            location=LOCATION_TEST,
+            product=PRODUCT_TEST,
+            product_id=PRODUCT_ID_TEST,
+            project_id='mock-project-id',
+            task_id='id',
+        )
+        result = op.execute(None)
+        self.assertEqual(PRODUCT_ID_TEST, result)
+
+
+class CloudVisionProductGetTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.get_product.return_value = {}
+        op = CloudVisionProductGetOperator(location=LOCATION_TEST, product_id=PRODUCT_ID_TEST, task_id='id')
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.get_product.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_id=PRODUCT_ID_TEST,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionProductUpdateTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.update_product.return_value = {}
+        op = CloudVisionProductUpdateOperator(location=LOCATION_TEST, product=PRODUCT_TEST, task_id='id')
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.update_product.assert_called_once_with(
+            location=LOCATION_TEST,
+            product=PRODUCT_TEST,
+            product_id=None,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+            update_mask=None,
+        )
+
+
+class CloudVisionProductDeleteTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        mock_hook.return_value.delete_product.return_value = {}
+        op = CloudVisionProductDeleteOperator(
+            location=LOCATION_TEST, product_id=PRODUCT_ID_TEST, task_id='id'
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.delete_product.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_id=PRODUCT_ID_TEST,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionReferenceImageCreateTest(unittest.TestCase):
+    @mock.patch(  # type: ignore
+        'airflow.contrib.operators.gcp_vision_operator.CloudVisionHook',
+        **{'return_value.create_reference_image.return_value': {}}
+    )
+    def test_minimal_green_path(self, mock_hook):
+        op = CloudVisionReferenceImageCreateOperator(
+            location=LOCATION_TEST,
+            product_id=PRODUCT_ID_TEST,
+            reference_image=REFERENCE_IMAGE_TEST,
+            task_id='id',
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.create_reference_image.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_id=PRODUCT_ID_TEST,
+            reference_image=REFERENCE_IMAGE_TEST,
+            reference_image_id=None,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+    @mock.patch(
+        'airflow.contrib.operators.gcp_vision_operator.CloudVisionHook',
+        **{'return_value.create_reference_image.side_effect': AlreadyExists("MESSAGe")}
+    )
+    def test_already_exists(self, mock_hook):
+        # Exception AlreadyExists not raised, caught in the operator's execute() - idempotence
+        op = CloudVisionReferenceImageCreateOperator(
+            location=LOCATION_TEST,
+            product_id=PRODUCT_ID_TEST,
+            reference_image=REFERENCE_IMAGE_TEST,
+            task_id='id',
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.create_reference_image.assert_called_once_with(
+            location=LOCATION_TEST,
+            product_id=PRODUCT_ID_TEST,
+            reference_image=REFERENCE_IMAGE_TEST,
+            reference_image_id=None,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionAddProductToProductSetOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        op = CloudVisionAddProductToProductSetOperator(
+            location=LOCATION_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            task_id='id',
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.add_product_to_product_set.assert_called_once_with(
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            location=LOCATION_TEST,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionRemoveProductFromProductSetOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        op = CloudVisionRemoveProductFromProductSetOperator(
+            location=LOCATION_TEST,
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            task_id='id',
+        )
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.remove_product_from_product_set.assert_called_once_with(
+            product_set_id=PRODUCTSET_ID_TEST,
+            product_id=PRODUCT_ID_TEST,
+            location=LOCATION_TEST,
+            project_id=None,
+            retry=None,
+            timeout=None,
+            metadata=None,
+        )
+
+
+class CloudVisionAnnotateImageOperatorTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcp_vision_operator.CloudVisionHook')
+    def test_minimal_green_path(self, mock_hook):
+        op = CloudVisionAnnotateImageOperator(request=ANNOTATE_REQUEST_TEST, task_id='id')
+        op.execute(context=None)
+        mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID)
+        mock_hook.return_value.annotate_image.assert_called_once_with(
+            request=ANNOTATE_REQUEST_TEST, retry=None, timeout=None
+        )
diff --git a/tests/contrib/operators/test_gcp_vision_operator_system.py b/tests/contrib/operators/test_gcp_vision_operator_system.py
new file mode 100644
index 0000000000000..8618e61bcf6e8
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_vision_operator_system.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from tests.contrib.utils.gcp_authenticator import GCP_AI_KEY
+from tests.contrib.operators.test_gcp_vision_operator_system_helper import GCPVisionTestHelper
+from tests.contrib.utils.base_gcp_system_test_case import SKIP_TEST_WARNING, DagGcpSystemTestCase
+
+VISION_HELPER = GCPVisionTestHelper()
+
+
+@unittest.skipIf(DagGcpSystemTestCase.skip_check(GCP_AI_KEY), SKIP_TEST_WARNING)
+class CloudVisionExampleDagsSystemTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(CloudVisionExampleDagsSystemTest, self).__init__(
+            method_name, dag_name='example_gcp_vision.py', gcp_key=GCP_AI_KEY
+        )
+
+    def setUp(self):
+        super(CloudVisionExampleDagsSystemTest, self).setUp()
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            VISION_HELPER.create_bucket()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+
+    def tearDown(self):
+        self.gcp_authenticator.gcp_authenticate()
+        try:
+            VISION_HELPER.delete_bucket()
+        finally:
+            self.gcp_authenticator.gcp_revoke_authentication()
+        super(CloudVisionExampleDagsSystemTest, self).tearDown()
+
+    def test_run_example_gcp_vision_autogenerated_id_dag(self):
+        self._run_dag('example_gcp_vision_autogenerated_id')
+
+    def test_run_example_gcp_vision_explicit_id_dag(self):
+        self._run_dag('example_gcp_vision_explicit_id')
+
+    def test_run_example_gcp_vision_annotate_image_dag(self):
+        self._run_dag('example_gcp_vision_annotate_image')
diff --git a/tests/contrib/operators/test_gcp_vision_operator_system_helper.py b/tests/contrib/operators/test_gcp_vision_operator_system_helper.py
new file mode 100755
index 0000000000000..3a88286ca25e4
--- /dev/null
+++ b/tests/contrib/operators/test_gcp_vision_operator_system_helper.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+
+import argparse
+from tempfile import NamedTemporaryFile
+
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator, GCP_AI_KEY
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+
+GCP_PROJECT_ID = os.environ.get('GCP_PROJECT_ID', 'example-project')
+GCP_BUCKET_NAME = os.environ.get('GCP_VISION_BUCKET_NAME', 'vision-bucket-system-test')
+GCP_REFERENCE_IMAGE_URL = os.environ.get('GCP_VISION_REFERENCE_IMAGE_URL', 'gs://bucket-name/image.png')
+GCP_ANNOTATE_IMAGE_URL = os.environ.get('GCP_VISION_ANNOTATE_IMAGE_URL', 'gs://bucket-name/image.png')
+GCP_VIDEO_SOURCE_URL = os.environ.get('GCP_VISION_SOURCE_IMAGE_URL', "http://google.com/image.jpg")
+
+
+class GCPVisionTestHelper(LoggingCommandExecutor):
+    def create_bucket(self):
+        self.execute_cmd(
+            [
+                'gsutil',
+                'mb',
+                "-p",
+                GCP_PROJECT_ID,
+                "-c",
+                "regional",
+                "-l",
+                "europe-north1",
+                "gs://%s/" % GCP_BUCKET_NAME,
+            ]
+        )
+
+        with NamedTemporaryFile(suffix=".png") as file:
+            self.execute_cmd(["curl", "-s", GCP_VIDEO_SOURCE_URL, "-o", file.name])
+            self.execute_cmd(['gsutil', 'cp', file.name, GCP_REFERENCE_IMAGE_URL])
+            self.execute_cmd(['gsutil', 'cp', file.name, GCP_ANNOTATE_IMAGE_URL])
+
+    def delete_bucket(self):
+        self.execute_cmd(['gsutil', 'rm', '-r', "gs://%s/" % GCP_BUCKET_NAME])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create and remove a bucket for system tests.')
+    parser.add_argument(
+        '--action',
+        dest='action',
+        required=True,
+        choices=('create-bucket', 'delete-bucket', 'before-tests', 'after-tests'),
+    )
+    action = parser.parse_args().action
+
+    helper = GCPVisionTestHelper()
+    gcp_authenticator = GcpAuthenticator(GCP_AI_KEY)
+    helper.log.info('Starting action: {}'.format(action))
+
+    gcp_authenticator.gcp_store_authentication()
+    try:
+        gcp_authenticator.gcp_authenticate()
+        if action == 'before-tests':
+            pass
+        elif action == 'after-tests':
+            pass
+        elif action == 'create-bucket':
+            helper.create_bucket()
+        elif action == 'delete-bucket':
+            helper.delete_bucket()
+        else:
+            raise Exception("Unknown action: {}".format(action))
+    finally:
+        gcp_authenticator.gcp_restore_authentication()
+
+    helper.log.info('Finishing action: {}'.format(action))
diff --git a/tests/contrib/operators/test_gcs_acl_operator.py b/tests/contrib/operators/test_gcs_acl_operator.py
new file mode 100644
index 0000000000000..cae3484613276
--- /dev/null
+++ b/tests/contrib/operators/test_gcs_acl_operator.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow.contrib.operators.gcs_acl_operator import \
+    GoogleCloudStorageBucketCreateAclEntryOperator, \
+    GoogleCloudStorageObjectCreateAclEntryOperator
+from tests.compat import mock
+
+
+class GoogleCloudStorageAclTest(unittest.TestCase):
+    @mock.patch('airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageHook')
+    def test_bucket_create_acl(self, mock_hook):
+        operator = GoogleCloudStorageBucketCreateAclEntryOperator(
+            bucket="test-bucket",
+            entity="test-entity",
+            role="test-role",
+            user_project="test-user-project",
+            task_id="id"
+        )
+        operator.execute(None)
+        mock_hook.return_value.insert_bucket_acl.assert_called_once_with(
+            bucket="test-bucket",
+            entity="test-entity",
+            role="test-role",
+            user_project="test-user-project"
+        )
+
+    @mock.patch('airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageHook')
+    def test_object_create_acl(self, mock_hook):
+        operator = GoogleCloudStorageObjectCreateAclEntryOperator(
+            bucket="test-bucket",
+            object_name="test-object",
+            entity="test-entity",
+            role="test-role",
+            generation="test-generation",
+            user_project="test-user-project",
+            task_id="id"
+        )
+        operator.execute(None)
+        mock_hook.return_value.insert_object_acl.assert_called_once_with(
+            bucket="test-bucket",
+            object_name="test-object",
+            entity="test-entity",
+            role="test-role",
+            generation="test-generation",
+            user_project="test-user-project"
+        )
diff --git a/tests/contrib/operators/test_gcs_acl_operator_system.py b/tests/contrib/operators/test_gcs_acl_operator_system.py
new file mode 100644
index 0000000000000..0afddcbc9e5a7
--- /dev/null
+++ b/tests/contrib/operators/test_gcs_acl_operator_system.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from tests.contrib.utils.base_gcp_system_test_case import \
+    SKIP_TEST_WARNING, DagGcpSystemTestCase
+from tests.contrib.utils.gcp_authenticator import GCP_GCS_KEY
+
+
+@unittest.skipIf(
+    DagGcpSystemTestCase.skip_check(GCP_GCS_KEY), SKIP_TEST_WARNING)
+class CloudStorageExampleDagsSystemTest(DagGcpSystemTestCase):
+    def __init__(self, method_name='runTest'):
+        super(CloudStorageExampleDagsSystemTest, self).__init__(
+            method_name,
+            dag_id='example_gcs_acl',
+            gcp_key=GCP_GCS_KEY)
+
+    def test_run_example_dag_gcs_acl(self):
+        self._run_dag()
diff --git a/tests/contrib/operators/test_gcs_download_operator.py b/tests/contrib/operators/test_gcs_download_operator.py
index 13b08a4bd7e00..c366afed162ea 100644
--- a/tests/contrib/operators/test_gcs_download_operator.py
+++ b/tests/contrib/operators/test_gcs_download_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,14 +21,7 @@
 
 from airflow.contrib.operators.gcs_download_operator \
     import GoogleCloudStorageDownloadOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-gcs-download-operator'
 TEST_BUCKET = 'test-bucket'
diff --git a/tests/contrib/operators/test_gcs_list_operator.py b/tests/contrib/operators/test_gcs_list_operator.py
index 84612eaaf3a04..7b01b2143085b 100644
--- a/tests/contrib/operators/test_gcs_list_operator.py
+++ b/tests/contrib/operators/test_gcs_list_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,14 +20,7 @@
 import unittest
 
 from airflow.contrib.operators.gcs_list_operator import GoogleCloudStorageListOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-gcs-list-operator'
 TEST_BUCKET = 'test-bucket'
diff --git a/tests/contrib/operators/test_gcs_operator.py b/tests/contrib/operators/test_gcs_operator.py
index cd51933de7268..67c86d62d6f6e 100644
--- a/tests/contrib/operators/test_gcs_operator.py
+++ b/tests/contrib/operators/test_gcs_operator.py
@@ -21,14 +21,7 @@
 
 from airflow.contrib.operators.gcs_operator import GoogleCloudStorageCreateBucketOperator
 from airflow.version import version
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-gcs-operator'
 TEST_BUCKET = 'test-bucket'
@@ -42,6 +35,7 @@ def test_execute(self, mock_hook):
         operator = GoogleCloudStorageCreateBucketOperator(
             task_id=TASK_ID,
             bucket_name=TEST_BUCKET,
+            resource={"lifecycle": {"rule": [{"action": {"type": "Delete"}, "condition": {"age": 7}}]}},
             storage_class='MULTI_REGIONAL',
             location='EU',
             labels={'env': 'prod'},
@@ -54,5 +48,6 @@ def test_execute(self, mock_hook):
             location='EU', labels={
                 'airflow-version': 'v' + version.replace('.', '-').replace('+', '-'),
                 'env': 'prod'
-            }, project_id=TEST_PROJECT
+            }, project_id=TEST_PROJECT,
+            resource={'lifecycle': {'rule': [{'action': {'type': 'Delete'}, 'condition': {'age': 7}}]}}
         )
diff --git a/tests/contrib/operators/test_gcs_to_gcs_operator.py b/tests/contrib/operators/test_gcs_to_gcs_operator.py
index 6b866d11e1796..bcf56af9765c0 100644
--- a/tests/contrib/operators/test_gcs_to_gcs_operator.py
+++ b/tests/contrib/operators/test_gcs_to_gcs_operator.py
@@ -18,17 +18,11 @@
 # under the License.
 
 import unittest
+from datetime import datetime
 
 from airflow.contrib.operators.gcs_to_gcs import \
     GoogleCloudStorageToGoogleCloudStorageOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-gcs-to-gcs-operator'
 TEST_BUCKET = 'test-bucket'
@@ -38,6 +32,7 @@
 SOURCE_OBJECT_2 = 'test_object*'
 SOURCE_OBJECT_3 = 'test*object'
 SOURCE_OBJECT_4 = 'test_object*.txt'
+SOURCE_OBJECT_5 = 'test_object.txt'
 DESTINATION_BUCKET = 'archive'
 DESTINATION_OBJECT_PREFIX = 'foo/bar'
 SOURCE_FILES_LIST = [
@@ -45,6 +40,7 @@
     'test_object/file2.txt',
     'test_object/file3.json',
 ]
+MOD_TIME_1 = datetime(2016, 1, 1)
 
 
 class GoogleCloudStorageToCloudStorageOperatorTest(unittest.TestCase):
@@ -167,3 +163,113 @@ def test_execute_wildcard_empty_destination_object(self, mock_hook):
                       DESTINATION_BUCKET, '/file2.txt'),
         ]
         mock_hook.return_value.rewrite.assert_has_calls(mock_calls_empty)
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_execute_last_modified_time(self, mock_hook):
+        mock_hook.return_value.list.return_value = SOURCE_FILES_LIST
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_4,
+            destination_bucket=DESTINATION_BUCKET,
+            last_modified_time=None)
+
+        operator.execute(None)
+        mock_calls_none = [
+            mock.call(TEST_BUCKET, 'test_object/file1.txt',
+                      DESTINATION_BUCKET, 'test_object/file1.txt'),
+            mock.call(TEST_BUCKET, 'test_object/file2.txt',
+                      DESTINATION_BUCKET, 'test_object/file2.txt'),
+        ]
+        mock_hook.return_value.rewrite.assert_has_calls(mock_calls_none)
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_wc_with_last_modified_time_with_all_true_cond(self, mock_hook):
+        mock_hook.return_value.list.return_value = SOURCE_FILES_LIST
+        mock_hook.return_value.is_updated_after.side_effect = [True, True, True]
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_4,
+            destination_bucket=DESTINATION_BUCKET,
+            last_modified_time=MOD_TIME_1)
+
+        operator.execute(None)
+        mock_calls_none = [
+            mock.call(TEST_BUCKET, 'test_object/file1.txt',
+                      DESTINATION_BUCKET, 'test_object/file1.txt'),
+            mock.call(TEST_BUCKET, 'test_object/file2.txt',
+                      DESTINATION_BUCKET, 'test_object/file2.txt'),
+        ]
+        mock_hook.return_value.rewrite.assert_has_calls(mock_calls_none)
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_wc_with_last_modified_time_with_one_true_cond(self, mock_hook):
+        mock_hook.return_value.list.return_value = SOURCE_FILES_LIST
+        mock_hook.return_value.is_updated_after.side_effect = [True, False, False]
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_4,
+            destination_bucket=DESTINATION_BUCKET,
+            last_modified_time=MOD_TIME_1)
+
+        operator.execute(None)
+        mock_hook.return_value.rewrite.assert_called_once_with(
+            TEST_BUCKET, 'test_object/file1.txt',
+            DESTINATION_BUCKET, 'test_object/file1.txt')
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_wc_with_no_last_modified_time(self, mock_hook):
+        mock_hook.return_value.list.return_value = SOURCE_FILES_LIST
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_4,
+            destination_bucket=DESTINATION_BUCKET,
+            last_modified_time=None)
+
+        operator.execute(None)
+        mock_calls_none = [
+            mock.call(TEST_BUCKET, 'test_object/file1.txt',
+                      DESTINATION_BUCKET, 'test_object/file1.txt'),
+            mock.call(TEST_BUCKET, 'test_object/file2.txt',
+                      DESTINATION_BUCKET, 'test_object/file2.txt'),
+        ]
+        mock_hook.return_value.rewrite.assert_has_calls(mock_calls_none)
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_no_prefix_with_last_modified_time_with_true_cond(self, mock_hook):
+        mock_hook.return_value.is_updated_after.return_value = True
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_5,
+            destination_bucket=DESTINATION_BUCKET,
+            destination_object=SOURCE_OBJECT_5,
+            last_modified_time=MOD_TIME_1)
+
+        operator.execute(None)
+        mock_hook.return_value.rewrite.assert_called_once_with(
+            TEST_BUCKET, 'test_object.txt', DESTINATION_BUCKET, 'test_object.txt')
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_execute_no_prefix_with_no_last_modified_time(self, mock_hook):
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_5,
+            destination_bucket=DESTINATION_BUCKET,
+            destination_object=SOURCE_OBJECT_5,
+            last_modified_time=None)
+
+        operator.execute(None)
+        mock_hook.return_value.rewrite.assert_called_once_with(
+            TEST_BUCKET, 'test_object.txt', DESTINATION_BUCKET, 'test_object.txt')
+
+    @mock.patch('airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageHook')
+    def test_no_prefix_with_last_modified_time_with_false_cond(self, mock_hook):
+        mock_hook.return_value.is_updated_after.return_value = False
+        operator = GoogleCloudStorageToGoogleCloudStorageOperator(
+            task_id=TASK_ID, source_bucket=TEST_BUCKET,
+            source_object=SOURCE_OBJECT_5,
+            destination_bucket=DESTINATION_BUCKET,
+            destination_object=SOURCE_OBJECT_5,
+            last_modified_time=MOD_TIME_1)
+
+        operator.execute(None)
+        mock_hook.return_value.rewrite.assert_not_called()
diff --git a/tests/contrib/operators/test_gcs_to_s3_operator.py b/tests/contrib/operators/test_gcs_to_s3_operator.py
index 5f386a62fe812..092d4dfadbb7f 100644
--- a/tests/contrib/operators/test_gcs_to_s3_operator.py
+++ b/tests/contrib/operators/test_gcs_to_s3_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,14 +21,7 @@
 
 from airflow.contrib.operators.gcs_to_s3 import GoogleCloudStorageToS3Operator
 from airflow.hooks.S3_hook import S3Hook
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 try:
     from moto import mock_s3
diff --git a/tests/contrib/operators/test_hipchat_operator.py b/tests/contrib/operators/test_hipchat_operator.py
index b6abce9bd2f94..be86a1bb595f4 100644
--- a/tests/contrib/operators/test_hipchat_operator.py
+++ b/tests/contrib/operators/test_hipchat_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,14 +24,7 @@
     HipChatAPISendRoomNotificationOperator
 from airflow.exceptions import AirflowException
 from airflow import configuration
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class HipChatOperatorTest(unittest.TestCase):
@@ -47,7 +40,7 @@ def test_execute(self, request_mock):
 
         operator = HipChatAPISendRoomNotificationOperator(
             task_id='test_hipchat_success',
-            owner = 'airflow',
+            owner='airflow',
             token='abc123',
             room_id='room_id',
             message='hello world!'
diff --git a/tests/contrib/operators/test_hive_to_dynamodb_operator.py b/tests/contrib/operators/test_hive_to_dynamodb_operator.py
index d1f75b50c40eb..ab86e055176fd 100644
--- a/tests/contrib/operators/test_hive_to_dynamodb_operator.py
+++ b/tests/contrib/operators/test_hive_to_dynamodb_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,17 +20,18 @@
 
 import json
 import unittest
+import datetime
 
 import mock
 import pandas as pd
 
 from airflow import configuration, DAG
-
-configuration.load_test_config()
-import datetime
 from airflow.contrib.hooks.aws_dynamodb_hook import AwsDynamoDBHook
+
 import airflow.contrib.operators.hive_to_dynamodb
 
+configuration.load_test_config()
+
 DEFAULT_DATE = datetime.datetime(2015, 1, 1)
 DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat()
 DEFAULT_DATE_DS = DEFAULT_DATE_ISO[:10]
@@ -52,7 +53,8 @@ def setUp(self):
         self.hook = AwsDynamoDBHook(
             aws_conn_id='aws_default', region_name='us-east-1')
 
-    def process_data(self, data, *args, **kwargs):
+    @staticmethod
+    def process_data(data, *args, **kwargs):
         return json.loads(data.to_json(orient='records'))
 
     @unittest.skipIf(mock_dynamodb2 is None, 'mock_dynamodb2 package not present')
@@ -66,9 +68,8 @@ def test_get_conn_returns_a_boto3_connection(self):
     @unittest.skipIf(mock_dynamodb2 is None, 'mock_dynamodb2 package not present')
     @mock_dynamodb2
     def test_get_records_with_schema(self, get_results_mock):
-
         # this table needs to be created in production
-        table = self.hook.get_conn().create_table(
+        self.hook.get_conn().create_table(
             TableName='test_airflow',
             KeySchema=[
                 {
@@ -107,9 +108,8 @@ def test_get_records_with_schema(self, get_results_mock):
     @unittest.skipIf(mock_dynamodb2 is None, 'mock_dynamodb2 package not present')
     @mock_dynamodb2
     def test_pre_process_records_with_schema(self, get_results_mock):
-
-         # this table needs to be created in production
-        table = self.hook.get_conn().create_table(
+        # this table needs to be created in production
+        self.hook.get_conn().create_table(
             TableName='test_airflow',
             KeySchema=[
                 {
@@ -140,8 +140,7 @@ def test_pre_process_records_with_schema(self, get_results_mock):
         operator.execute(None)
 
         table = self.hook.get_conn().Table('test_airflow')
-        table.meta.client.get_waiter(
-            'table_exists').wait(TableName='test_airflow')
+        table.meta.client.get_waiter('table_exists').wait(TableName='test_airflow')
         self.assertEqual(table.item_count, 1)
 
 
diff --git a/tests/contrib/operators/test_imap_attachment_to_s3_operator.py b/tests/contrib/operators/test_imap_attachment_to_s3_operator.py
new file mode 100644
index 0000000000000..66287fc7f4de3
--- /dev/null
+++ b/tests/contrib/operators/test_imap_attachment_to_s3_operator.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from mock import patch
+
+from airflow.contrib.operators.imap_attachment_to_s3_operator import ImapAttachmentToS3Operator
+
+
+class TestImapAttachmentToS3Operator(unittest.TestCase):
+
+    def setUp(self):
+        self.kwargs = dict(
+            imap_attachment_name='test_file',
+            s3_key='test_file',
+            imap_mail_folder='INBOX',
+            imap_check_regex=False,
+            s3_overwrite=False,
+            task_id='test_task',
+            dag=None
+        )
+
+    @patch('airflow.contrib.operators.imap_attachment_to_s3_operator.S3Hook')
+    @patch('airflow.contrib.operators.imap_attachment_to_s3_operator.ImapHook')
+    def test_execute(self, mock_imap_hook, mock_s3_hook):
+        mock_imap_hook.return_value.__enter__ = mock_imap_hook
+        mock_imap_hook.return_value.retrieve_mail_attachments.return_value = [('test_file', b'Hello World')]
+
+        ImapAttachmentToS3Operator(**self.kwargs).execute(context={})
+
+        mock_imap_hook.return_value.retrieve_mail_attachments.assert_called_once_with(
+            name=self.kwargs['imap_attachment_name'],
+            mail_folder=self.kwargs['imap_mail_folder'],
+            check_regex=self.kwargs['imap_check_regex'],
+            latest_only=True
+        )
+        mock_s3_hook.return_value.load_bytes.assert_called_once_with(
+            bytes_data=mock_imap_hook.return_value.retrieve_mail_attachments.return_value[0][1],
+            key=self.kwargs['s3_key']
+        )
diff --git a/tests/contrib/operators/test_jenkins_operator.py b/tests/contrib/operators/test_jenkins_operator.py
index 0d70246fff85d..875d906082d5f 100644
--- a/tests/contrib/operators/test_jenkins_operator.py
+++ b/tests/contrib/operators/test_jenkins_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -26,14 +26,7 @@
 from airflow.contrib.hooks.jenkins_hook import JenkinsHook
 
 from airflow.exceptions import AirflowException
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class JenkinsOperatorTestCase(unittest.TestCase):
@@ -70,7 +63,7 @@ def test_execute(self):
 
             operator.execute(None)
 
-            self.assertEquals(jenkins_mock.get_build_info.call_count, 1)
+            self.assertEqual(jenkins_mock.get_build_info.call_count, 1)
             jenkins_mock.get_build_info.assert_called_with(name='a_job_on_jenkins',
                                                            number='1')
 
@@ -107,7 +100,7 @@ def test_execute_job_polling_loop(self):
                 sleep_time=1)
 
             operator.execute(None)
-            self.assertEquals(jenkins_mock.get_build_info.call_count, 2)
+            self.assertEqual(jenkins_mock.get_build_info.call_count, 2)
 
     @unittest.skipIf(mock is None, 'mock package not present')
     def test_execute_job_failure(self):
diff --git a/tests/contrib/operators/test_jira_operator_test.py b/tests/contrib/operators/test_jira_operator_test.py
index 764458c314dad..5d15832525c77 100644
--- a/tests/contrib/operators/test_jira_operator_test.py
+++ b/tests/contrib/operators/test_jira_operator_test.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,13 +25,13 @@
 
 from airflow import DAG, configuration
 from airflow.contrib.operators.jira_operator import JiraOperator
-from airflow import models
+from airflow.models.connection import Connection
 from airflow.utils import db
 from airflow.utils import timezone
 
 DEFAULT_DATE = timezone.datetime(2017, 1, 1)
 jira_client_mock = Mock(
-        name="jira_client_for_test"
+    name="jira_client_for_test"
 )
 
 minimal_test_ticket = {
@@ -54,14 +54,14 @@ def setUp(self):
         args = {
             'owner': 'airflow',
             'start_date': DEFAULT_DATE
-            }
+        }
         dag = DAG('test_dag_id', default_args=args)
         self.dag = dag
         db.merge_conn(
-                models.Connection(
-                        conn_id='jira_default', conn_type='jira',
-                        host='https://localhost/jira/', port=443,
-                        extra='{"verify": "False", "project": "AIRFLOW"}'))
+            Connection(
+                conn_id='jira_default', conn_type='jira',
+                host='https://localhost/jira/', port=443,
+                extra='{"verify": "False", "project": "AIRFLOW"}'))
 
     @patch("airflow.contrib.hooks.jira_hook.JIRA",
            autospec=True, return_value=jira_client_mock)
diff --git a/tests/contrib/operators/test_mlengine_operator.py b/tests/contrib/operators/test_mlengine_operator.py
index 9a8d56cfb2e43..eb62e79d4c54c 100644
--- a/tests/contrib/operators/test_mlengine_operator.py
+++ b/tests/contrib/operators/test_mlengine_operator.py
@@ -22,7 +22,7 @@
 import unittest
 
 import httplib2
-from apiclient import errors
+from googleapiclient.errors import HttpError
 from mock import ANY, patch
 
 from airflow import DAG, configuration
@@ -85,7 +85,7 @@ def testSuccessWithModel(self):
             success_message['predictionInput'] = input_with_model
 
             hook_instance = mock_hook.return_value
-            hook_instance.get_job.side_effect = errors.HttpError(
+            hook_instance.get_job.side_effect = HttpError(
                 resp=httplib2.Response({
                     'status': 404
                 }), content=b'some bytes')
@@ -109,8 +109,8 @@ def testSuccessWithModel(self):
                     'jobId': 'test_prediction',
                     'predictionInput': input_with_model
                 }, ANY)
-            self.assertEquals(success_message['predictionOutput'],
-                              prediction_output)
+            self.assertEqual(success_message['predictionOutput'],
+                             prediction_output)
 
     def testSuccessWithVersion(self):
         with patch('airflow.contrib.operators.mlengine_operator.MLEngineHook') \
@@ -123,7 +123,7 @@ def testSuccessWithVersion(self):
             success_message['predictionInput'] = input_with_version
 
             hook_instance = mock_hook.return_value
-            hook_instance.get_job.side_effect = errors.HttpError(
+            hook_instance.get_job.side_effect = HttpError(
                 resp=httplib2.Response({
                     'status': 404
                 }), content=b'some bytes')
@@ -148,8 +148,8 @@ def testSuccessWithVersion(self):
                     'jobId': 'test_prediction',
                     'predictionInput': input_with_version
                 }, ANY)
-            self.assertEquals(success_message['predictionOutput'],
-                              prediction_output)
+            self.assertEqual(success_message['predictionOutput'],
+                             prediction_output)
 
     def testSuccessWithURI(self):
         with patch('airflow.contrib.operators.mlengine_operator.MLEngineHook') \
@@ -161,7 +161,7 @@ def testSuccessWithURI(self):
             success_message['predictionInput'] = input_with_uri
 
             hook_instance = mock_hook.return_value
-            hook_instance.get_job.side_effect = errors.HttpError(
+            hook_instance.get_job.side_effect = HttpError(
                 resp=httplib2.Response({
                     'status': 404
                 }), content=b'some bytes')
@@ -185,8 +185,8 @@ def testSuccessWithURI(self):
                     'jobId': 'test_prediction',
                     'predictionInput': input_with_uri
                 }, ANY)
-            self.assertEquals(success_message['predictionOutput'],
-                              prediction_output)
+            self.assertEqual(success_message['predictionOutput'],
+                             prediction_output)
 
     def testInvalidModelOrigin(self):
         # Test that both uri and model is given
@@ -195,9 +195,9 @@ def testInvalidModelOrigin(self):
         task_args['model_name'] = 'fake_model'
         with self.assertRaises(AirflowException) as context:
             MLEngineBatchPredictionOperator(**task_args).execute(None)
-        self.assertEquals('Ambiguous model origin: Both uri and '
-                          'model/version name are provided.',
-                          str(context.exception))
+        self.assertEqual('Ambiguous model origin: Both uri and '
+                         'model/version name are provided.',
+                         str(context.exception))
 
         # Test that both uri and model/version is given
         task_args = self.BATCH_PREDICTION_DEFAULT_ARGS.copy()
@@ -206,24 +206,24 @@ def testInvalidModelOrigin(self):
         task_args['version_name'] = 'fake_version'
         with self.assertRaises(AirflowException) as context:
             MLEngineBatchPredictionOperator(**task_args).execute(None)
-        self.assertEquals('Ambiguous model origin: Both uri and '
-                          'model/version name are provided.',
-                          str(context.exception))
+        self.assertEqual('Ambiguous model origin: Both uri and '
+                         'model/version name are provided.',
+                         str(context.exception))
 
         # Test that a version is given without a model
         task_args = self.BATCH_PREDICTION_DEFAULT_ARGS.copy()
         task_args['version_name'] = 'bare_version'
         with self.assertRaises(AirflowException) as context:
             MLEngineBatchPredictionOperator(**task_args).execute(None)
-        self.assertEquals('Missing model: Batch prediction expects a model '
-                          'name when a version name is provided.',
-                          str(context.exception))
+        self.assertEqual('Missing model: Batch prediction expects a model '
+                         'name when a version name is provided.',
+                         str(context.exception))
 
         # Test that none of uri, model, model/version is given
         task_args = self.BATCH_PREDICTION_DEFAULT_ARGS.copy()
         with self.assertRaises(AirflowException) as context:
             MLEngineBatchPredictionOperator(**task_args).execute(None)
-        self.assertEquals(
+        self.assertEqual(
             'Missing model origin: Batch prediction expects a '
             'model, a model & version combination, or a URI to a savedModel.',
             str(context.exception))
@@ -238,13 +238,13 @@ def testHttpError(self):
                 'projects/experimental/models/test_model'
 
             hook_instance = mock_hook.return_value
-            hook_instance.create_job.side_effect = errors.HttpError(
+            hook_instance.create_job.side_effect = HttpError(
                 resp=httplib2.Response({
                     'status': http_error_code
                 }),
                 content=b'Forbidden')
 
-            with self.assertRaises(errors.HttpError) as context:
+            with self.assertRaises(HttpError) as context:
                 prediction_task = MLEngineBatchPredictionOperator(
                     job_id='test_prediction',
                     project_id='test-project',
@@ -264,7 +264,7 @@ def testHttpError(self):
                         'predictionInput': input_with_model
                     }, ANY)
 
-            self.assertEquals(http_error_code, context.exception.resp.status)
+            self.assertEqual(http_error_code, context.exception.resp.status)
 
     def testFailedJobError(self):
         with patch('airflow.contrib.operators.mlengine_operator.MLEngineHook') \
@@ -280,7 +280,7 @@ def testFailedJobError(self):
             with self.assertRaises(RuntimeError) as context:
                 MLEngineBatchPredictionOperator(**task_args).execute(None)
 
-            self.assertEquals('A failure message', str(context.exception))
+            self.assertEqual('A failure message', str(context.exception))
 
 
 class MLEngineTrainingOperatorTest(unittest.TestCase):
@@ -320,7 +320,7 @@ def testSuccessCreateTrainingJob(self):
             mock_hook.assert_called_with(
                 gcp_conn_id='google_cloud_default', delegate_to=None)
             # Make sure only 'create_job' is invoked on hook instance
-            self.assertEquals(len(hook_instance.mock_calls), 1)
+            self.assertEqual(len(hook_instance.mock_calls), 1)
             hook_instance.create_job.assert_called_with(
                 'test-project', self.TRAINING_INPUT, ANY)
 
@@ -347,7 +347,7 @@ def testSuccessCreateTrainingJobWithOptionalArgs(self):
             mock_hook.assert_called_with(gcp_conn_id='google_cloud_default',
                                          delegate_to=None)
             # Make sure only 'create_job' is invoked on hook instance
-            self.assertEquals(len(hook_instance.mock_calls), 1)
+            self.assertEqual(len(hook_instance.mock_calls), 1)
             hook_instance.create_job.assert_called_with(
                 'test-project', training_input, ANY)
 
@@ -356,13 +356,13 @@ def testHttpError(self):
         with patch('airflow.contrib.operators.mlengine_operator.MLEngineHook') \
                 as mock_hook:
             hook_instance = mock_hook.return_value
-            hook_instance.create_job.side_effect = errors.HttpError(
+            hook_instance.create_job.side_effect = HttpError(
                 resp=httplib2.Response({
                     'status': http_error_code
                 }),
                 content=b'Forbidden')
 
-            with self.assertRaises(errors.HttpError) as context:
+            with self.assertRaises(HttpError) as context:
                 training_op = MLEngineTrainingOperator(
                     **self.TRAINING_DEFAULT_ARGS)
                 training_op.execute(None)
@@ -370,10 +370,10 @@ def testHttpError(self):
             mock_hook.assert_called_with(
                 gcp_conn_id='google_cloud_default', delegate_to=None)
             # Make sure only 'create_job' is invoked on hook instance
-            self.assertEquals(len(hook_instance.mock_calls), 1)
+            self.assertEqual(len(hook_instance.mock_calls), 1)
             hook_instance.create_job.assert_called_with(
                 'test-project', self.TRAINING_INPUT, ANY)
-            self.assertEquals(http_error_code, context.exception.resp.status)
+            self.assertEqual(http_error_code, context.exception.resp.status)
 
     def testFailedJobError(self):
         with patch('airflow.contrib.operators.mlengine_operator.MLEngineHook') \
@@ -392,10 +392,10 @@ def testFailedJobError(self):
             mock_hook.assert_called_with(
                 gcp_conn_id='google_cloud_default', delegate_to=None)
             # Make sure only 'create_job' is invoked on hook instance
-            self.assertEquals(len(hook_instance.mock_calls), 1)
+            self.assertEqual(len(hook_instance.mock_calls), 1)
             hook_instance.create_job.assert_called_with(
                 'test-project', self.TRAINING_INPUT, ANY)
-            self.assertEquals('A failure message', str(context.exception))
+            self.assertEqual('A failure message', str(context.exception))
 
 
 class MLEngineVersionOperatorTest(unittest.TestCase):
@@ -425,7 +425,7 @@ def testSuccessCreateVersion(self):
             mock_hook.assert_called_with(gcp_conn_id='google_cloud_default',
                                          delegate_to=None)
             # Make sure only 'create_version' is invoked on hook instance
-            self.assertEquals(len(hook_instance.mock_calls), 1)
+            self.assertEqual(len(hook_instance.mock_calls), 1)
             hook_instance.create_version.assert_called_with(
                 'test-project', 'test-model', self.VERSION_INPUT)
 
diff --git a/tests/contrib/operators/test_mongo_to_s3_operator.py b/tests/contrib/operators/test_mongo_to_s3_operator.py
index 6f30596ce9985..d1de69e6ae832 100644
--- a/tests/contrib/operators/test_mongo_to_s3_operator.py
+++ b/tests/contrib/operators/test_mongo_to_s3_operator.py
@@ -1,30 +1,28 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 import unittest
 
 from airflow import DAG
 from airflow.contrib.operators.mongo_to_s3 import MongoToS3Operator
 from airflow.models import TaskInstance
 from airflow.utils import timezone
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test_mongo_to_s3_operator'
 MONGO_CONN_ID = 'default_mongo'
diff --git a/tests/contrib/operators/test_mysql_to_gcs_operator.py b/tests/contrib/operators/test_mysql_to_gcs_operator.py
index 6e2e3f90dfb95..bb9685e51669e 100644
--- a/tests/contrib/operators/test_mysql_to_gcs_operator.py
+++ b/tests/contrib/operators/test_mysql_to_gcs_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,46 +17,219 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import sys
 import unittest
-import json
-from mock import MagicMock
 
 from airflow.contrib.operators.mysql_to_gcs import \
     MySqlToGoogleCloudStorageOperator
+from tests.compat import mock
+
+PY3 = sys.version_info[0] == 3
+
+TASK_ID = 'test-mysql-to-gcs'
+MYSQL_CONN_ID = 'mysql_conn_test'
+SQL = 'select 1'
+BUCKET = 'gs://test'
+JSON_FILENAME = 'test_{}.ndjson'
+CSV_FILENAME = 'test_{}.csv'
+
+ROWS = [
+    ('mock_row_content_1', 42),
+    ('mock_row_content_2', 43),
+    ('mock_row_content_3', 44)
+]
+CURSOR_DESCRIPTION = (
+    ('some_str', 0, 0, 0, 0, 0, False),
+    ('some_num', 1005, 0, 0, 0, 0, False)
+)
+NDJSON_LINES = [
+    b'{"some_num": 42, "some_str": "mock_row_content_1"}\n',
+    b'{"some_num": 43, "some_str": "mock_row_content_2"}\n',
+    b'{"some_num": 44, "some_str": "mock_row_content_3"}\n'
+]
+CSV_LINES = [
+    b'some_str,some_num\r\n'
+    b'mock_row_content_1,42\r\n',
+    b'mock_row_content_2,43\r\n',
+    b'mock_row_content_3,44\r\n'
+]
+CSV_LINES_PIPE_DELIMITED = [
+    b'some_str|some_num\r\n'
+    b'mock_row_content_1|42\r\n',
+    b'mock_row_content_2|43\r\n',
+    b'mock_row_content_3|44\r\n'
+]
+SCHEMA_FILENAME = 'schema_test.json'
+SCHEMA_JSON = [
+    b'[{"mode": "REQUIRED", "name": "some_str", "type": "FLOAT"}, ',
+    b'{"mode": "REQUIRED", "name": "some_num", "type": "STRING"}]'
+]
 
 
 class MySqlToGoogleCloudStorageOperatorTest(unittest.TestCase):
 
-    def test_write_local_data_files(self):
-
-        # Configure
-        task_id = "some_test_id"
-        sql = "some_sql"
-        bucket = "some_bucket"
-        filename = "some_filename"
-        row_iter = [[1, b'byte_str_1'], [2, b'byte_str_2']]
-        schema = []
-        schema.append({
-            'name': 'location',
-            'type': 'STRING',
-            'mode': 'nullable',
-        })
-        schema.append({
-            'name': 'uuid',
-            'type': 'BYTES',
-            'mode': 'nullable',
-        })
-        schema_str = json.dumps(schema)
+    def test_init(self):
+        """Test MySqlToGoogleCloudStorageOperator instance is properly initialized."""
+        op = MySqlToGoogleCloudStorageOperator(
+            task_id=TASK_ID, sql=SQL, bucket=BUCKET, filename=JSON_FILENAME,
+            export_format='CSV', field_delimiter='|')
+        self.assertEqual(op.task_id, TASK_ID)
+        self.assertEqual(op.sql, SQL)
+        self.assertEqual(op.bucket, BUCKET)
+        self.assertEqual(op.filename, JSON_FILENAME)
+        self.assertEqual(op.export_format, 'csv')
+        self.assertEqual(op.field_delimiter, '|')
+
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.MySqlHook')
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.GoogleCloudStorageHook')
+    def test_exec_success_json(self, gcs_hook_mock_class, mysql_hook_mock_class):
+        """Test successful run of execute function for JSON"""
+        op = MySqlToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            mysql_conn_id=MYSQL_CONN_ID,
+            sql=SQL,
+            bucket=BUCKET,
+            filename=JSON_FILENAME)
+
+        mysql_hook_mock = mysql_hook_mock_class.return_value
+        mysql_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
+        mysql_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
+
+        gcs_hook_mock = gcs_hook_mock_class.return_value
 
+        def _assert_upload(bucket, obj, tmp_filename, mime_type=None):
+            self.assertEqual(BUCKET, bucket)
+            self.assertEqual(JSON_FILENAME.format(0), obj)
+            self.assertEqual('application/json', mime_type)
+            with open(tmp_filename, 'rb') as f:
+                self.assertEqual(b''.join(NDJSON_LINES), f.read())
+
+        gcs_hook_mock.upload.side_effect = _assert_upload
+
+        op.execute(None)
+
+        mysql_hook_mock_class.assert_called_once_with(mysql_conn_id=MYSQL_CONN_ID)
+        mysql_hook_mock.get_conn().cursor().execute.assert_called_once_with(SQL)
+
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.MySqlHook')
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.GoogleCloudStorageHook')
+    def test_exec_success_csv(self, gcs_hook_mock_class, mysql_hook_mock_class):
+        """Test successful run of execute function for CSV"""
         op = MySqlToGoogleCloudStorageOperator(
-            task_id=task_id,
-            sql=sql,
-            bucket=bucket,
-            filename=filename,
-            schema=schema_str)
+            task_id=TASK_ID,
+            mysql_conn_id=MYSQL_CONN_ID,
+            sql=SQL,
+            export_format='CSV',
+            bucket=BUCKET,
+            filename=CSV_FILENAME)
+
+        mysql_hook_mock = mysql_hook_mock_class.return_value
+        mysql_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
+        mysql_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
+
+        gcs_hook_mock = gcs_hook_mock_class.return_value
+
+        def _assert_upload(bucket, obj, tmp_filename, mime_type=None):
+            self.assertEqual(BUCKET, bucket)
+            self.assertEqual(CSV_FILENAME.format(0), obj)
+            self.assertEqual('text/csv', mime_type)
+            with open(tmp_filename, 'rb') as f:
+                self.assertEqual(b''.join(CSV_LINES), f.read())
+
+        gcs_hook_mock.upload.side_effect = _assert_upload
+
+        op.execute(None)
+
+        mysql_hook_mock_class.assert_called_once_with(mysql_conn_id=MYSQL_CONN_ID)
+        mysql_hook_mock.get_conn().cursor().execute.assert_called_once_with(SQL)
 
-        cursor_mock = MagicMock()
-        cursor_mock.__iter__.return_value = row_iter
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.MySqlHook')
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.GoogleCloudStorageHook')
+    def test_exec_success_csv_with_delimiter(self, gcs_hook_mock_class, mysql_hook_mock_class):
+        """Test successful run of execute function for CSV with a field delimiter"""
+        op = MySqlToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            mysql_conn_id=MYSQL_CONN_ID,
+            sql=SQL,
+            export_format='csv',
+            field_delimiter='|',
+            bucket=BUCKET,
+            filename=CSV_FILENAME)
+
+        mysql_hook_mock = mysql_hook_mock_class.return_value
+        mysql_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
+        mysql_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
+
+        gcs_hook_mock = gcs_hook_mock_class.return_value
+
+        def _assert_upload(bucket, obj, tmp_filename, mime_type=None):
+            self.assertEqual(BUCKET, bucket)
+            self.assertEqual(CSV_FILENAME.format(0), obj)
+            self.assertEqual('text/csv', mime_type)
+            with open(tmp_filename, 'rb') as f:
+                self.assertEqual(b''.join(CSV_LINES_PIPE_DELIMITED), f.read())
+
+        gcs_hook_mock.upload.side_effect = _assert_upload
+
+        op.execute(None)
+
+        mysql_hook_mock_class.assert_called_once_with(mysql_conn_id=MYSQL_CONN_ID)
+        mysql_hook_mock.get_conn().cursor().execute.assert_called_once_with(SQL)
+
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.MySqlHook')
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.GoogleCloudStorageHook')
+    def test_file_splitting(self, gcs_hook_mock_class, mysql_hook_mock_class):
+        """Test that ndjson is split by approx_max_file_size_bytes param."""
+        mysql_hook_mock = mysql_hook_mock_class.return_value
+        mysql_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
+        mysql_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
+
+        gcs_hook_mock = gcs_hook_mock_class.return_value
+        expected_upload = {
+            JSON_FILENAME.format(0): b''.join(NDJSON_LINES[:2]),
+            JSON_FILENAME.format(1): NDJSON_LINES[2],
+        }
+
+        def _assert_upload(bucket, obj, tmp_filename, mime_type=None):
+            self.assertEqual(BUCKET, bucket)
+            self.assertEqual('application/json', mime_type)
+            with open(tmp_filename, 'rb') as f:
+                self.assertEqual(expected_upload[obj], f.read())
+
+        gcs_hook_mock.upload.side_effect = _assert_upload
+
+        op = MySqlToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            sql=SQL,
+            bucket=BUCKET,
+            filename=JSON_FILENAME,
+            approx_max_file_size_bytes=len(expected_upload[JSON_FILENAME.format(0)]))
+        op.execute(None)
+
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.MySqlHook')
+    @mock.patch('airflow.contrib.operators.mysql_to_gcs.GoogleCloudStorageHook')
+    def test_schema_file(self, gcs_hook_mock_class, mysql_hook_mock_class):
+        """Test writing schema files."""
+        mysql_hook_mock = mysql_hook_mock_class.return_value
+        mysql_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
+        mysql_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
+
+        gcs_hook_mock = gcs_hook_mock_class.return_value
+
+        def _assert_upload(bucket, obj, tmp_filename, mime_type):
+            if obj == SCHEMA_FILENAME:
+                with open(tmp_filename, 'rb') as f:
+                    self.assertEqual(b''.join(SCHEMA_JSON), f.read())
+
+        gcs_hook_mock.upload.side_effect = _assert_upload
+
+        op = MySqlToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            sql=SQL,
+            bucket=BUCKET,
+            filename=JSON_FILENAME,
+            schema_filename=SCHEMA_FILENAME)
+        op.execute(None)
 
-        # Run
-        op._write_local_data_files(cursor_mock)
+        # once for the file and once for the schema
+        self.assertEqual(2, gcs_hook_mock.upload.call_count)
diff --git a/tests/contrib/operators/test_opsgenie_alert_operator.py b/tests/contrib/operators/test_opsgenie_alert_operator.py
new file mode 100644
index 0000000000000..1b4467bc5a523
--- /dev/null
+++ b/tests/contrib/operators/test_opsgenie_alert_operator.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+
+from airflow import DAG, configuration
+
+from airflow.contrib.operators.opsgenie_alert_operator import OpsgenieAlertOperator
+from airflow.utils import timezone
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+
+class TestOpsgenieAlertOperator(unittest.TestCase):
+    _config = {
+        'message': 'An example alert message',
+        'alias': 'Life is too short for no alias',
+        'description': 'Every alert needs a description',
+        'responders': [
+            {'id': '4513b7ea-3b91-438f-b7e4-e3e54af9147c', 'type': 'team'},
+            {'name': 'NOC', 'type': 'team'},
+            {'id': 'bb4d9938-c3c2-455d-aaab-727aa701c0d8', 'type': 'user'},
+            {'username': 'trinity@opsgenie.com', 'type': 'user'},
+            {'id': 'aee8a0de-c80f-4515-a232-501c0bc9d715', 'type': 'escalation'},
+            {'name': 'Nightwatch Escalation', 'type': 'escalation'},
+            {'id': '80564037-1984-4f38-b98e-8a1f662df552', 'type': 'schedule'},
+            {'name': 'First Responders Schedule', 'type': 'schedule'}
+        ],
+        'visibleTo': [
+            {'id': '4513b7ea-3b91-438f-b7e4-e3e54af9147c', 'type': 'team'},
+            {'name': 'rocket_team', 'type': 'team'},
+            {'id': 'bb4d9938-c3c2-455d-aaab-727aa701c0d8', 'type': 'user'},
+            {'username': 'trinity@opsgenie.com', 'type': 'user'}
+        ],
+        'actions': ['Restart', 'AnExampleAction'],
+        'tags': ['OverwriteQuietHours', 'Critical'],
+        'details': {'key1': 'value1', 'key2': 'value2'},
+        'entity': 'An example entity',
+        'source': 'Airflow',
+        'priority': 'P1',
+        'user': 'Jesse',
+        'note': 'Write this down'
+    }
+
+    expected_payload_dict = {
+        'message': _config['message'],
+        'alias': _config['alias'],
+        'description': _config['description'],
+        'responders': _config['responders'],
+        'visibleTo': _config['visibleTo'],
+        'actions': _config['actions'],
+        'tags': _config['tags'],
+        'details': _config['details'],
+        'entity': _config['entity'],
+        'source': _config['source'],
+        'priority': _config['priority'],
+        'user': _config['user'],
+        'note': _config['note']
+    }
+
+    def setUp(self):
+        configuration.load_test_config()
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG('test_dag_id', default_args=args)
+
+    def test_build_opsgenie_payload(self):
+        # Given / When
+        operator = OpsgenieAlertOperator(
+            task_id='opsgenie_alert_job',
+            dag=self.dag,
+            **self._config
+        )
+
+        payload = operator._build_opsgenie_payload()
+
+        # Then
+        self.assertEqual(self.expected_payload_dict, payload)
+
+    def test_properties(self):
+        # Given / When
+        operator = OpsgenieAlertOperator(
+            task_id='opsgenie_alert_job',
+            dag=self.dag,
+            **self._config
+        )
+
+        self.assertEqual('opsgenie_default', operator.opsgenie_conn_id)
+        self.assertEqual(self._config['message'], operator.message)
+        self.assertEqual(self._config['alias'], operator.alias)
+        self.assertEqual(self._config['description'], operator.description)
+        self.assertEqual(self._config['responders'], operator.responders)
+        self.assertEqual(self._config['visibleTo'], operator.visibleTo)
+        self.assertEqual(self._config['actions'], operator.actions)
+        self.assertEqual(self._config['tags'], operator.tags)
+        self.assertEqual(self._config['details'], operator.details)
+        self.assertEqual(self._config['entity'], operator.entity)
+        self.assertEqual(self._config['source'], operator.source)
+        self.assertEqual(self._config['priority'], operator.priority)
+        self.assertEqual(self._config['user'], operator.user)
+        self.assertEqual(self._config['note'], operator.note)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_oracle_to_azure_data_lake_transfer.py b/tests/contrib/operators/test_oracle_to_azure_data_lake_transfer.py
new file mode 100644
index 0000000000000..7686afad21885
--- /dev/null
+++ b/tests/contrib/operators/test_oracle_to_azure_data_lake_transfer.py
@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from airflow.contrib.operators.oracle_to_azure_data_lake_transfer \
+    import OracleToAzureDataLakeTransfer
+from airflow.utils.file import TemporaryDirectory
+import unicodecsv as csv
+import os
+
+from tests.compat import mock, MagicMock
+
+
+class OracleToAzureDataLakeTransferTest(unittest.TestCase):
+
+    mock_module_path = 'airflow.contrib.operators.oracle_to_azure_data_lake_transfer'
+
+    def test_write_temp_file(self):
+        task_id = "some_test_id"
+        sql = "some_sql"
+        sql_params = {':p_data': "2018-01-01"}
+        oracle_conn_id = "oracle_conn_id"
+        filename = "some_filename"
+        azure_data_lake_conn_id = 'azure_data_lake_conn_id'
+        azure_data_lake_path = 'azure_data_lake_path'
+        delimiter = '|'
+        encoding = 'utf-8'
+        cursor_description = [
+            ('id', "<class 'cx_Oracle.NUMBER'>", 39, None, 38, 0, 0),
+            ('description', "<class 'cx_Oracle.STRING'>", 60, 240, None, None, 1)
+        ]
+        cursor_rows = [[1, 'description 1'], [2, 'description 2']]
+        mock_cursor = MagicMock()
+        mock_cursor.description = cursor_description
+        mock_cursor.__iter__.return_value = cursor_rows
+
+        op = OracleToAzureDataLakeTransfer(
+            task_id=task_id,
+            filename=filename,
+            oracle_conn_id=oracle_conn_id,
+            sql=sql,
+            sql_params=sql_params,
+            azure_data_lake_conn_id=azure_data_lake_conn_id,
+            azure_data_lake_path=azure_data_lake_path,
+            delimiter=delimiter,
+            encoding=encoding)
+
+        with TemporaryDirectory(prefix='airflow_oracle_to_azure_op_') as temp:
+            op._write_temp_file(mock_cursor, os.path.join(temp, filename))
+
+            assert os.path.exists(os.path.join(temp, filename)) == 1
+
+            with open(os.path.join(temp, filename), 'rb') as csvfile:
+                temp_file = csv.reader(csvfile, delimiter=delimiter, encoding=encoding)
+
+                rownum = 0
+                for row in temp_file:
+                    if rownum == 0:
+                        self.assertEqual(row[0], 'id')
+                        self.assertEqual(row[1], 'description')
+                    else:
+                        self.assertEqual(row[0], str(cursor_rows[rownum - 1][0]))
+                        self.assertEqual(row[1], cursor_rows[rownum - 1][1])
+                    rownum = rownum + 1
+
+    @mock.patch(mock_module_path + '.OracleHook',
+                autospec=True)
+    @mock.patch(mock_module_path + '.AzureDataLakeHook',
+                autospec=True)
+    def test_execute(self, mock_data_lake_hook, mock_oracle_hook):
+        task_id = "some_test_id"
+        sql = "some_sql"
+        sql_params = {':p_data': "2018-01-01"}
+        oracle_conn_id = "oracle_conn_id"
+        filename = "some_filename"
+        azure_data_lake_conn_id = 'azure_data_lake_conn_id'
+        azure_data_lake_path = 'azure_data_lake_path'
+        delimiter = '|'
+        encoding = 'latin-1'
+        cursor_description = [
+            ('id', "<class 'cx_Oracle.NUMBER'>", 39, None, 38, 0, 0),
+            ('description', "<class 'cx_Oracle.STRING'>", 60, 240, None, None, 1)
+        ]
+        cursor_rows = [[1, 'description 1'], [2, 'description 2']]
+        cursor_mock = MagicMock()
+        cursor_mock.description.return_value = cursor_description
+        cursor_mock.__iter__.return_value = cursor_rows
+        mock_oracle_conn = MagicMock()
+        mock_oracle_conn.cursor().return_value = cursor_mock
+        mock_oracle_hook.get_conn().return_value = mock_oracle_conn
+
+        op = OracleToAzureDataLakeTransfer(
+            task_id=task_id,
+            filename=filename,
+            oracle_conn_id=oracle_conn_id,
+            sql=sql,
+            sql_params=sql_params,
+            azure_data_lake_conn_id=azure_data_lake_conn_id,
+            azure_data_lake_path=azure_data_lake_path,
+            delimiter=delimiter,
+            encoding=encoding)
+
+        op.execute(None)
+
+        mock_oracle_hook.assert_called_once_with(oracle_conn_id=oracle_conn_id)
+        mock_data_lake_hook.assert_called_once_with(
+            azure_data_lake_conn_id=azure_data_lake_conn_id)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_oracle_to_oracle_transfer.py b/tests/contrib/operators/test_oracle_to_oracle_transfer.py
new file mode 100644
index 0000000000000..e814b83ea3a12
--- /dev/null
+++ b/tests/contrib/operators/test_oracle_to_oracle_transfer.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from airflow.contrib.operators.oracle_to_oracle_transfer \
+    import OracleToOracleTransfer
+from tests.compat import MagicMock
+
+
+class OracleToOracleTransferTest(unittest.TestCase):
+
+    @staticmethod
+    def test_execute():
+        oracle_destination_conn_id = 'oracle_destination_conn_id'
+        destination_table = 'destination_table'
+        oracle_source_conn_id = 'oracle_source_conn_id'
+        source_sql = "select sysdate from dual where trunc(sysdate) = :p_data"
+        source_sql_params = {':p_data': "2018-01-01"}
+        rows_chunk = 5000
+        cursor_description = [
+            ('id', "<class 'cx_Oracle.NUMBER'>", 39, None, 38, 0, 0),
+            ('description', "<class 'cx_Oracle.STRING'>", 60, 240, None, None, 1)
+        ]
+        cursor_rows = [[1, 'description 1'], [2, 'description 2']]
+
+        mock_dest_hook = MagicMock()
+        mock_src_hook = MagicMock()
+        mock_src_conn = mock_src_hook.get_conn.return_value.__enter__.return_value
+        mock_cursor = mock_src_conn.cursor.return_value
+        mock_cursor.description.__iter__.return_value = cursor_description
+        mock_cursor.fetchmany.side_effect = [cursor_rows, []]
+
+        op = OracleToOracleTransfer(
+            task_id='copy_data',
+            oracle_destination_conn_id=oracle_destination_conn_id,
+            destination_table=destination_table,
+            oracle_source_conn_id=oracle_source_conn_id,
+            source_sql=source_sql,
+            source_sql_params=source_sql_params,
+            rows_chunk=rows_chunk)
+
+        op._execute(mock_src_hook, mock_dest_hook, None)
+
+        assert mock_src_hook.get_conn.called
+        assert mock_src_conn.cursor.called
+        mock_cursor.execute.assert_called_with(source_sql, source_sql_params)
+        mock_cursor.fetchmany.assert_called_with(rows_chunk)
+        mock_dest_hook.bulk_insert_rows.assert_called_once_with(
+            destination_table,
+            cursor_rows,
+            commit_every=rows_chunk,
+            target_fields=['id', 'description'])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_postgres_to_gcs_operator.py b/tests/contrib/operators/test_postgres_to_gcs_operator.py
index ca72016974c0e..420c1d4195379 100644
--- a/tests/contrib/operators/test_postgres_to_gcs_operator.py
+++ b/tests/contrib/operators/test_postgres_to_gcs_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,43 +22,61 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import sys
 import unittest
 
-from airflow.contrib.operators.postgres_to_gcs_operator import PostgresToGoogleCloudStorageOperator
+from airflow.hooks.postgres_hook import PostgresHook
+from airflow.contrib.operators.postgres_to_gcs_operator import \
+    PostgresToGoogleCloudStorageOperator
+from tests.compat import patch
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
-PY3 = sys.version_info[0] == 3
+TABLES = {'postgres_to_gcs_operator', 'postgres_to_gcs_operator_empty'}
 
 TASK_ID = 'test-postgres-to-gcs'
-POSTGRES_CONN_ID = 'postgres_conn_test'
-SQL = 'select 1'
+POSTGRES_CONN_ID = 'postgres_default'
+SQL = 'SELECT * FROM postgres_to_gcs_operator'
 BUCKET = 'gs://test'
 FILENAME = 'test_{}.ndjson'
-# we expect the psycopg cursor to return encoded strs in py2 and decoded in py3
-if PY3:
-    ROWS = [('mock_row_content_1', 42), ('mock_row_content_2', 43), ('mock_row_content_3', 44)]
-    CURSOR_DESCRIPTION = (('some_str', 0), ('some_num', 1005))
-else:
-    ROWS = [(b'mock_row_content_1', 42), (b'mock_row_content_2', 43), (b'mock_row_content_3', 44)]
-    CURSOR_DESCRIPTION = ((b'some_str', 0), (b'some_num', 1005))
+
 NDJSON_LINES = [
     b'{"some_num": 42, "some_str": "mock_row_content_1"}\n',
     b'{"some_num": 43, "some_str": "mock_row_content_2"}\n',
     b'{"some_num": 44, "some_str": "mock_row_content_3"}\n'
 ]
 SCHEMA_FILENAME = 'schema_test.json'
-SCHEMA_JSON = b'[{"mode": "NULLABLE", "name": "some_str", "type": "STRING"}, {"mode": "REPEATED", "name": "some_num", "type": "INTEGER"}]'
+SCHEMA_JSON = b'[{"mode": "NULLABLE", "name": "some_str", "type": "STRING"}, ' \
+              b'{"mode": "NULLABLE", "name": "some_num", "type": "INTEGER"}]'
 
 
 class PostgresToGoogleCloudStorageOperatorTest(unittest.TestCase):
+    def setUp(self):
+        postgres = PostgresHook()
+        with postgres.get_conn() as conn:
+            with conn.cursor() as cur:
+                for table in TABLES:
+                    cur.execute("DROP TABLE IF EXISTS {} CASCADE;".format(table))
+                    cur.execute("CREATE TABLE {}(some_str varchar, some_num integer);"
+                                .format(table))
+
+                cur.execute(
+                    "INSERT INTO postgres_to_gcs_operator VALUES(%s, %s);",
+                    ('mock_row_content_1', 42)
+                )
+                cur.execute(
+                    "INSERT INTO postgres_to_gcs_operator VALUES(%s, %s);",
+                    ('mock_row_content_2', 43)
+                )
+                cur.execute(
+                    "INSERT INTO postgres_to_gcs_operator VALUES(%s, %s);",
+                    ('mock_row_content_3', 44)
+                )
+
+    def tearDown(self):
+        postgres = PostgresHook()
+        with postgres.get_conn() as conn:
+            with conn.cursor() as cur:
+                for table in TABLES:
+                    cur.execute("DROP TABLE IF EXISTS {} CASCADE;".format(table))
+
     def test_init(self):
         """Test PostgresToGoogleCloudStorageOperator instance is properly initialized."""
         op = PostgresToGoogleCloudStorageOperator(
@@ -68,9 +86,8 @@ def test_init(self):
         self.assertEqual(op.bucket, BUCKET)
         self.assertEqual(op.filename, FILENAME)
 
-    @mock.patch('airflow.contrib.operators.postgres_to_gcs_operator.PostgresHook')
-    @mock.patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
-    def test_exec_success(self, gcs_hook_mock_class, pg_hook_mock_class):
+    @patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
+    def test_exec_success(self, gcs_hook_mock_class):
         """Test the execute function in case where the run is successful."""
         op = PostgresToGoogleCloudStorageOperator(
             task_id=TASK_ID,
@@ -79,10 +96,6 @@ def test_exec_success(self, gcs_hook_mock_class, pg_hook_mock_class):
             bucket=BUCKET,
             filename=FILENAME)
 
-        pg_hook_mock = pg_hook_mock_class.return_value
-        pg_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
-        pg_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
-
         gcs_hook_mock = gcs_hook_mock_class.return_value
 
         def _assert_upload(bucket, obj, tmp_filename, content_type):
@@ -96,16 +109,9 @@ def _assert_upload(bucket, obj, tmp_filename, content_type):
 
         op.execute(None)
 
-        pg_hook_mock_class.assert_called_once_with(postgres_conn_id=POSTGRES_CONN_ID)
-        pg_hook_mock.get_conn().cursor().execute.assert_called_once_with(SQL, None)
-
-    @mock.patch('airflow.contrib.operators.postgres_to_gcs_operator.PostgresHook')
-    @mock.patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
-    def test_file_splitting(self, gcs_hook_mock_class, pg_hook_mock_class):
+    @patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
+    def test_file_splitting(self, gcs_hook_mock_class):
         """Test that ndjson is split by approx_max_file_size_bytes param."""
-        pg_hook_mock = pg_hook_mock_class.return_value
-        pg_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
-        pg_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
 
         gcs_hook_mock = gcs_hook_mock_class.return_value
         expected_upload = {
@@ -129,13 +135,23 @@ def _assert_upload(bucket, obj, tmp_filename, content_type):
             approx_max_file_size_bytes=len(expected_upload[FILENAME.format(0)]))
         op.execute(None)
 
-    @mock.patch('airflow.contrib.operators.postgres_to_gcs_operator.PostgresHook')
-    @mock.patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
-    def test_schema_file(self, gcs_hook_mock_class, pg_hook_mock_class):
+    @patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
+    def test_empty_query(self, gcs_hook_mock_class):
+        """If the sql returns no rows, we should not upload any files"""
+        gcs_hook_mock = gcs_hook_mock_class.return_value
+
+        op = PostgresToGoogleCloudStorageOperator(
+            task_id=TASK_ID,
+            sql='SELECT * FROM postgres_to_gcs_operator_empty',
+            bucket=BUCKET,
+            filename=FILENAME)
+        op.execute(None)
+
+        assert not gcs_hook_mock.upload.called, 'No data means no files in the bucket'
+
+    @patch('airflow.contrib.operators.postgres_to_gcs_operator.GoogleCloudStorageHook')
+    def test_schema_file(self, gcs_hook_mock_class):
         """Test writing schema files."""
-        pg_hook_mock = pg_hook_mock_class.return_value
-        pg_hook_mock.get_conn().cursor().__iter__.return_value = iter(ROWS)
-        pg_hook_mock.get_conn().cursor().description = CURSOR_DESCRIPTION
 
         gcs_hook_mock = gcs_hook_mock_class.return_value
 
diff --git a/tests/contrib/operators/test_pubsub_operator.py b/tests/contrib/operators/test_pubsub_operator.py
index 8f775da43a8be..ada3b8c9edc38 100644
--- a/tests/contrib/operators/test_pubsub_operator.py
+++ b/tests/contrib/operators/test_pubsub_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -26,15 +26,7 @@
     PubSubTopicCreateOperator, PubSubTopicDeleteOperator,
     PubSubSubscriptionCreateOperator, PubSubSubscriptionDeleteOperator,
     PubSubPublishOperator)
-
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-task-id'
 TEST_PROJECT = 'test-project'
@@ -101,7 +93,7 @@ def test_execute(self, mock_hook):
         mock_hook.return_value.create_subscription.assert_called_once_with(
             TEST_PROJECT, TEST_TOPIC, TEST_SUBSCRIPTION, None,
             10, False)
-        self.assertEquals(response, TEST_SUBSCRIPTION)
+        self.assertEqual(response, TEST_SUBSCRIPTION)
 
     @mock.patch('airflow.contrib.operators.pubsub_operator.PubSubHook')
     def test_execute_different_project_ids(self, mock_hook):
@@ -116,7 +108,7 @@ def test_execute_different_project_ids(self, mock_hook):
         mock_hook.return_value.create_subscription.assert_called_once_with(
             TEST_PROJECT, TEST_TOPIC, TEST_SUBSCRIPTION, another_project,
             10, False)
-        self.assertEquals(response, TEST_SUBSCRIPTION)
+        self.assertEqual(response, TEST_SUBSCRIPTION)
 
     @mock.patch('airflow.contrib.operators.pubsub_operator.PubSubHook')
     def test_execute_no_subscription(self, mock_hook):
@@ -127,7 +119,7 @@ def test_execute_no_subscription(self, mock_hook):
         response = operator.execute(None)
         mock_hook.return_value.create_subscription.assert_called_once_with(
             TEST_PROJECT, TEST_TOPIC, None, None, 10, False)
-        self.assertEquals(response, TEST_SUBSCRIPTION)
+        self.assertEqual(response, TEST_SUBSCRIPTION)
 
 
 class PubSubSubscriptionDeleteOperatorTest(unittest.TestCase):
diff --git a/tests/contrib/operators/test_qubole_check_operator.py b/tests/contrib/operators/test_qubole_check_operator.py
index 29044827ee6e5..b356f6091c90a 100644
--- a/tests/contrib/operators/test_qubole_check_operator.py
+++ b/tests/contrib/operators/test_qubole_check_operator.py
@@ -24,14 +24,8 @@
 from airflow.contrib.operators.qubole_check_operator import QuboleValueCheckOperator
 from airflow.contrib.hooks.qubole_check_hook import QuboleCheckHook
 from airflow.contrib.hooks.qubole_hook import QuboleHook
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from qds_sdk.commands import HiveCommand
+from tests.compat import mock
 
 
 class QuboleValueCheckOperatorTest(unittest.TestCase):
@@ -80,11 +74,13 @@ def test_execute_pass(self, mock_get_hook):
         mock_hook.get_first.assert_called_with(query)
 
     @mock.patch.object(QuboleValueCheckOperator, 'get_hook')
-    def test_execute_fail(self, mock_get_hook):
+    def test_execute_assertion_fail(self, mock_get_hook):
 
         mock_cmd = mock.Mock()
         mock_cmd.status = 'done'
         mock_cmd.id = 123
+        mock_cmd.is_success = mock.Mock(
+            return_value=HiveCommand.is_success(mock_cmd.status))
 
         mock_hook = mock.Mock()
         mock_hook.get_first.return_value = [11]
@@ -97,6 +93,30 @@ def test_execute_fail(self, mock_get_hook):
                                      'Qubole Command Id: ' + str(mock_cmd.id)):
             operator.execute()
 
+        mock_cmd.is_success.assert_called_with(mock_cmd.status)
+
+    @mock.patch.object(QuboleValueCheckOperator, 'get_hook')
+    def test_execute_assert_query_fail(self, mock_get_hook):
+
+        mock_cmd = mock.Mock()
+        mock_cmd.status = 'error'
+        mock_cmd.id = 123
+        mock_cmd.is_success = mock.Mock(
+            return_value=HiveCommand.is_success(mock_cmd.status))
+
+        mock_hook = mock.Mock()
+        mock_hook.get_first.return_value = [11]
+        mock_hook.cmd = mock_cmd
+        mock_get_hook.return_value = mock_hook
+
+        operator = self.__construct_operator('select value from tab1 limit 1;', 5, 1)
+
+        with self.assertRaises(AirflowException) as cm:
+            operator.execute()
+
+        self.assertNotIn('Qubole Command Id: ', str(cm.exception))
+        mock_cmd.is_success.assert_called_with(mock_cmd.status)
+
     @mock.patch.object(QuboleCheckHook, 'get_query_results')
     @mock.patch.object(QuboleHook, 'execute')
     def test_results_parser_callable(self, mock_execute, mock_get_query_results):
diff --git a/tests/contrib/operators/test_qubole_operator.py b/tests/contrib/operators/test_qubole_operator.py
index bf61262bb398d..de314ba110909 100644
--- a/tests/contrib/operators/test_qubole_operator.py
+++ b/tests/contrib/operators/test_qubole_operator.py
@@ -21,23 +21,16 @@
 import unittest
 from datetime import datetime
 
-from airflow.models import DAG, Connection, TaskInstance
+from airflow.models import DAG, TaskInstance
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 from airflow.contrib.hooks.qubole_hook import QuboleHook
 from airflow.contrib.operators.qubole_operator import QuboleOperator
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
-DAG_ID="qubole_test_dag"
-TASK_ID="test_task"
-DEFAULT_CONN="qubole_default"
+DAG_ID = "qubole_test_dag"
+TASK_ID = "test_task"
+DEFAULT_CONN = "qubole_default"
 TEMPLATE_CONN = "my_conn_id"
 DEFAULT_DATE = datetime(2017, 1, 1)
 
@@ -60,7 +53,7 @@ def test_init_with_template_connection(self):
                                   qubole_conn_id="{{ dag_run.conf['qubole_conn_id'] }}")
 
         result = task.render_template('qubole_conn_id', "{{ qubole_conn_id }}",
-                                      {'qubole_conn_id' : TEMPLATE_CONN})
+                                      {'qubole_conn_id': TEMPLATE_CONN})
         self.assertEqual(task.task_id, TASK_ID)
         self.assertEqual(result, TEMPLATE_CONN)
 
@@ -93,26 +86,22 @@ def test_hyphen_args_note_id(self):
         dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
 
         with dag:
-            task = QuboleOperator(task_id=TASK_ID, command_type='sparkcmd',
-                                  note_id="123", dag=dag)
-        self.assertEqual(task.get_hook().create_cmd_args({'run_id':'dummy'})[0],
-                         "--note-id=123")
+            task = QuboleOperator(task_id=TASK_ID, command_type='sparkcmd', note_id="123", dag=dag)
+
+        self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[0], "--note-id=123")
 
     def test_position_args_parameters(self):
         dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
 
         with dag:
             task = QuboleOperator(task_id=TASK_ID, command_type='pigcmd',
-                          parameters="key1=value1 key2=value2", dag=dag)
+                                  parameters="key1=value1 key2=value2", dag=dag)
 
-        self.assertEqual(task.get_hook().create_cmd_args({'run_id':'dummy'})[1],
-                         "key1=value1")
-        self.assertEqual(task.get_hook().create_cmd_args({'run_id':'dummy'})[2],
-                         "key2=value2")
+        self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[1], "key1=value1")
+        self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[2], "key2=value2")
 
-        task = QuboleOperator(task_id=TASK_ID, command_type='hadoopcmd',
-                          sub_command="s3distcp --src s3n://airflow/source_hadoopcmd " +
-                                      "--dest s3n://airflow/destination_hadoopcmd", dag=dag)
+        cmd = "s3distcp --src s3n://airflow/source_hadoopcmd --dest s3n://airflow/destination_hadoopcmd"
+        task = QuboleOperator(task_id=TASK_ID, command_type='hadoopcmd', dag=dag, sub_command=cmd)
 
         self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[1],
                          "s3distcp")
@@ -124,5 +113,3 @@ def test_position_args_parameters(self):
                          "--dest")
         self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[5],
                          "s3n://airflow/destination_hadoopcmd")
-
-
diff --git a/tests/contrib/operators/test_s3_copy_object_operator.py b/tests/contrib/operators/test_s3_copy_object_operator.py
new file mode 100644
index 0000000000000..25904ecc51476
--- /dev/null
+++ b/tests/contrib/operators/test_s3_copy_object_operator.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+import unittest
+
+import boto3
+from moto import mock_s3
+
+from airflow.contrib.operators.s3_copy_object_operator import S3CopyObjectOperator
+
+
+class TestS3CopyObjectOperator(unittest.TestCase):
+
+    def setUp(self):
+        self.source_bucket = "bucket1"
+        self.source_key = "path1/data.txt"
+        self.dest_bucket = "bucket2"
+        self.dest_key = "path2/data_copy.txt"
+
+    @mock_s3
+    def test_s3_copy_object_arg_combination_1(self):
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=self.source_bucket)
+        conn.create_bucket(Bucket=self.dest_bucket)
+        conn.upload_fileobj(Bucket=self.source_bucket,
+                            Key=self.source_key,
+                            Fileobj=io.BytesIO(b"input"))
+
+        # there should be nothing found before S3CopyObjectOperator is executed
+        self.assertFalse('Contents' in conn.list_objects(Bucket=self.dest_bucket,
+                                                         Prefix=self.dest_key))
+
+        t = S3CopyObjectOperator(task_id="test_task_s3_copy_object",
+                                 source_bucket_key=self.source_key,
+                                 source_bucket_name=self.source_bucket,
+                                 dest_bucket_key=self.dest_key,
+                                 dest_bucket_name=self.dest_bucket)
+        t.execute(None)
+
+        objects_in_dest_bucket = conn.list_objects(Bucket=self.dest_bucket,
+                                                   Prefix=self.dest_key)
+        # there should be object found, and there should only be one object found
+        self.assertEqual(len(objects_in_dest_bucket['Contents']), 1)
+        # the object found should be consistent with dest_key specified earlier
+        self.assertEqual(objects_in_dest_bucket['Contents'][0]['Key'], self.dest_key)
+
+    @mock_s3
+    def test_s3_copy_object_arg_combination_2(self):
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=self.source_bucket)
+        conn.create_bucket(Bucket=self.dest_bucket)
+        conn.upload_fileobj(Bucket=self.source_bucket,
+                            Key=self.source_key,
+                            Fileobj=io.BytesIO(b"input"))
+
+        # there should be nothing found before S3CopyObjectOperator is executed
+        self.assertFalse('Contents' in conn.list_objects(Bucket=self.dest_bucket,
+                                                         Prefix=self.dest_key))
+
+        source_key_s3_url = "s3://{}/{}".format(self.source_bucket, self.source_key)
+        dest_key_s3_url = "s3://{}/{}".format(self.dest_bucket, self.dest_key)
+        t = S3CopyObjectOperator(task_id="test_task_s3_copy_object",
+                                 source_bucket_key=source_key_s3_url,
+                                 dest_bucket_key=dest_key_s3_url)
+        t.execute(None)
+
+        objects_in_dest_bucket = conn.list_objects(Bucket=self.dest_bucket,
+                                                   Prefix=self.dest_key)
+        # there should be object found, and there should only be one object found
+        self.assertEqual(len(objects_in_dest_bucket['Contents']), 1)
+        # the object found should be consistent with dest_key specified earlier
+        self.assertEqual(objects_in_dest_bucket['Contents'][0]['Key'], self.dest_key)
diff --git a/tests/contrib/operators/test_s3_delete_objects_operator.py b/tests/contrib/operators/test_s3_delete_objects_operator.py
new file mode 100644
index 0000000000000..05ce1f57db758
--- /dev/null
+++ b/tests/contrib/operators/test_s3_delete_objects_operator.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import io
+import unittest
+
+import boto3
+from moto import mock_s3
+
+from airflow.contrib.operators.s3_delete_objects_operator import S3DeleteObjectsOperator
+
+
+class TestS3DeleteObjectsOperator(unittest.TestCase):
+
+    @mock_s3
+    def test_s3_delete_single_object(self):
+        bucket = "testbucket"
+        key = "path/data.txt"
+
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=bucket)
+        conn.upload_fileobj(Bucket=bucket,
+                            Key=key,
+                            Fileobj=io.BytesIO(b"input"))
+
+        # The object should be detected before the DELETE action is taken
+        objects_in_dest_bucket = conn.list_objects(Bucket=bucket,
+                                                   Prefix=key)
+        self.assertEqual(len(objects_in_dest_bucket['Contents']), 1)
+        self.assertEqual(objects_in_dest_bucket['Contents'][0]['Key'], key)
+
+        t = S3DeleteObjectsOperator(task_id="test_task_s3_delete_single_object",
+                                    bucket=bucket,
+                                    keys=key)
+        t.execute(None)
+
+        # There should be no object found in the bucket created earlier
+        self.assertFalse('Contents' in conn.list_objects(Bucket=bucket,
+                                                         Prefix=key))
+
+    @mock_s3
+    def test_s3_delete_multiple_objects(self):
+        bucket = "testbucket"
+        key_pattern = "path/data"
+        n_keys = 3
+        keys = [key_pattern + str(i) for i in range(n_keys)]
+
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=bucket)
+        for k in keys:
+            conn.upload_fileobj(Bucket=bucket,
+                                Key=k,
+                                Fileobj=io.BytesIO(b"input"))
+
+        # The objects should be detected before the DELETE action is taken
+        objects_in_dest_bucket = conn.list_objects(Bucket=bucket,
+                                                   Prefix=key_pattern)
+        self.assertEqual(len(objects_in_dest_bucket['Contents']), n_keys)
+        self.assertEqual(sorted([x['Key'] for x in objects_in_dest_bucket['Contents']]),
+                         sorted(keys))
+
+        t = S3DeleteObjectsOperator(task_id="test_task_s3_delete_multiple_objects",
+                                    bucket=bucket,
+                                    keys=keys)
+        t.execute(None)
+
+        # There should be no object found in the bucket created earlier
+        self.assertFalse('Contents' in conn.list_objects(Bucket=bucket,
+                                                         Prefix=key_pattern))
diff --git a/tests/contrib/operators/test_s3_list_operator.py b/tests/contrib/operators/test_s3_list_operator.py
index de95c7623b18a..e041fd07545f5 100644
--- a/tests/contrib/operators/test_s3_list_operator.py
+++ b/tests/contrib/operators/test_s3_list_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,14 +20,7 @@
 import unittest
 
 from airflow.contrib.operators.s3_list_operator import S3ListOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-s3-list-operator'
 BUCKET = 'test-bucket'
diff --git a/tests/contrib/operators/test_s3_to_gcs_operator.py b/tests/contrib/operators/test_s3_to_gcs_operator.py
index 807882c324936..28f71b3973d85 100644
--- a/tests/contrib/operators/test_s3_to_gcs_operator.py
+++ b/tests/contrib/operators/test_s3_to_gcs_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,14 +22,7 @@
 from airflow.contrib.hooks.gcs_hook import _parse_gcs_url
 from airflow.contrib.operators.s3_to_gcs_operator import \
     S3ToGoogleCloudStorageOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-s3-gcs-operator'
 S3_BUCKET = 'test-bucket'
@@ -88,8 +81,8 @@ def _assert_upload(bucket, object, tmp_filename):
 
         uploaded_files = operator.execute(None)
 
-        s3_one_mock_hook.assert_called_once_with(aws_conn_id=AWS_CONN_ID)
-        s3_two_mock_hook.assert_called_once_with(aws_conn_id=AWS_CONN_ID)
+        s3_one_mock_hook.assert_called_once_with(aws_conn_id=AWS_CONN_ID, verify=None)
+        s3_two_mock_hook.assert_called_once_with(aws_conn_id=AWS_CONN_ID, verify=None)
         gcs_mock_hook.assert_called_once_with(
             google_cloud_storage_conn_id=GCS_CONN_ID, delegate_to=None)
 
diff --git a/tests/contrib/operators/test_s3_to_sftp_operator.py b/tests/contrib/operators/test_s3_to_sftp_operator.py
new file mode 100644
index 0000000000000..359c5eb912172
--- /dev/null
+++ b/tests/contrib/operators/test_s3_to_sftp_operator.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow import models
+from airflow.contrib.operators.s3_to_sftp_operator import S3ToSFTPOperator
+from airflow.contrib.operators.ssh_operator import SSHOperator
+from airflow.models import DAG, TaskInstance
+from airflow.settings import Session
+from airflow.utils import timezone
+from airflow.utils.timezone import datetime
+import boto3
+from moto import mock_s3
+
+
+TASK_ID = 'test_s3_to_sftp'
+BUCKET = 'test-s3-bucket'
+S3_KEY = 'test/test_1_file.csv'
+SFTP_PATH = '/tmp/remote_path.txt'
+SFTP_CONN_ID = 'ssh_default'
+S3_CONN_ID = 'aws_default'
+LOCAL_FILE_PATH = '/tmp/test_s3_upload'
+
+SFTP_MOCK_FILE = 'test_sftp_file.csv'
+S3_MOCK_FILES = 'test_1_file.csv'
+
+TEST_DAG_ID = 'unit_tests'
+DEFAULT_DATE = datetime(2018, 1, 1)
+
+
+def reset(dag_id=TEST_DAG_ID):
+    session = Session()
+    tis = session.query(models.TaskInstance).filter_by(dag_id=dag_id)
+    tis.delete()
+    session.commit()
+    session.close()
+
+
+reset()
+
+
+class S3ToSFTPOperatorTest(unittest.TestCase):
+    @mock_s3
+    def setUp(self):
+        configuration.load_test_config()
+        from airflow.contrib.hooks.ssh_hook import SSHHook
+        from airflow.hooks.S3_hook import S3Hook
+
+        hook = SSHHook(ssh_conn_id='ssh_default')
+        s3_hook = S3Hook('aws_default')
+        hook.no_host_key_check = True
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE,
+            'provide_context': True
+        }
+        dag = DAG(TEST_DAG_ID + 'test_schedule_dag_once', default_args=args)
+        dag.schedule_interval = '@once'
+
+        self.hook = hook
+        self.s3_hook = s3_hook
+
+        self.ssh_client = self.hook.get_conn()
+        self.sftp_client = self.ssh_client.open_sftp()
+
+        self.dag = dag
+        self.s3_bucket = BUCKET
+        self.sftp_path = SFTP_PATH
+        self.s3_key = S3_KEY
+
+    @mock_s3
+    def test_s3_to_sftp_operation(self):
+        # Setting
+        configuration.conf.set("core", "enable_xcom_pickling", "True")
+        test_remote_file_content = \
+            "This is remote file content \n which is also multiline " \
+            "another line here \n this is last line. EOF"
+
+        # Test for creation of s3 bucket
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=self.s3_bucket)
+        self.assertTrue((self.s3_hook.check_for_bucket(self.s3_bucket)))
+
+        with open(LOCAL_FILE_PATH, 'w') as f:
+            f.write(test_remote_file_content)
+        self.s3_hook.load_file(LOCAL_FILE_PATH, self.s3_key, bucket_name=BUCKET)
+
+        # Check if object was created in s3
+        objects_in_dest_bucket = conn.list_objects(Bucket=self.s3_bucket,
+                                                   Prefix=self.s3_key)
+        # there should be object found, and there should only be one object found
+        self.assertEqual(len(objects_in_dest_bucket['Contents']), 1)
+
+        # the object found should be consistent with dest_key specified earlier
+        self.assertEqual(objects_in_dest_bucket['Contents'][0]['Key'], self.s3_key)
+
+        # get remote file to local
+        run_task = S3ToSFTPOperator(
+            s3_bucket=BUCKET,
+            s3_key=S3_KEY,
+            sftp_path=SFTP_PATH,
+            sftp_conn_id=SFTP_CONN_ID,
+            s3_conn_id=S3_CONN_ID,
+            task_id=TASK_ID,
+            dag=self.dag
+        )
+        self.assertIsNotNone(run_task)
+
+        run_task.execute(None)
+
+        # Check that the file is created remotely
+        check_file_task = SSHOperator(
+            task_id="test_check_file",
+            ssh_hook=self.hook,
+            command="cat {0}".format(self.sftp_path),
+            do_xcom_push=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(check_file_task)
+        ti3 = TaskInstance(task=check_file_task, execution_date=timezone.utcnow())
+        ti3.run()
+        self.assertEqual(
+            ti3.xcom_pull(task_ids='test_check_file', key='return_value').strip(),
+            test_remote_file_content.encode('utf-8'))
+
+        # Clean up after finishing with test
+        conn.delete_object(Bucket=self.s3_bucket, Key=self.s3_key)
+        conn.delete_bucket(Bucket=self.s3_bucket)
+        self.assertFalse((self.s3_hook.check_for_bucket(self.s3_bucket)))
+
+    def delete_remote_resource(self):
+        # check the remote file content
+        remove_file_task = SSHOperator(
+            task_id="test_check_file",
+            ssh_hook=self.hook,
+            command="rm {0}".format(self.sftp_path),
+            do_xcom_push=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(remove_file_task)
+        ti3 = TaskInstance(task=remove_file_task, execution_date=timezone.utcnow())
+        ti3.run()
+
+    def tearDown(self):
+        self.delete_remote_resource()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_base_operator.py b/tests/contrib/operators/test_sagemaker_base_operator.py
new file mode 100644
index 0000000000000..5ab7a614ae3b1
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_base_operator.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# 'License'); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator
+
+config = {
+    'key1': '1',
+    'key2': {
+        'key3': '3',
+        'key4': '4'
+    },
+    'key5': [
+        {
+            'key6': '6'
+        },
+        {
+            'key6': '7'
+        }
+    ]
+}
+
+parsed_config = {
+    'key1': 1,
+    'key2': {
+        'key3': 3,
+        'key4': 4
+    },
+    'key5': [
+        {
+            'key6': 6
+        },
+        {
+            'key6': 7
+        }
+    ]
+}
+
+
+class TestSageMakerBaseOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerBaseOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_id',
+            config=config
+        )
+
+    def test_parse_integer(self):
+        self.sagemaker.integer_fields = [
+            ['key1'], ['key2', 'key3'], ['key2', 'key4'], ['key5', 'key6']
+        ]
+        self.sagemaker.parse_config_integers()
+        self.assertEqual(self.sagemaker.config, parsed_config)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py b/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py
new file mode 100644
index 0000000000000..911a7e2334782
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.operators.sagemaker_endpoint_config_operator \
+    import SageMakerEndpointConfigOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+model_name = 'test-model-name'
+config_name = 'test-config-name'
+
+create_endpoint_config_params = {
+    'EndpointConfigName': config_name,
+    'ProductionVariants': [
+        {
+            'VariantName': 'AllTraffic',
+            'ModelName': model_name,
+            'InitialInstanceCount': '1',
+            'InstanceType': 'ml.c4.xlarge'
+        }
+    ]
+}
+
+
+class TestSageMakerEndpointConfigOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerEndpointConfigOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_id',
+            config=create_endpoint_config_params
+        )
+
+    def test_parse_config_integers(self):
+        self.sagemaker.parse_config_integers()
+        for variant in self.sagemaker.config['ProductionVariants']:
+            self.assertEqual(variant['InitialInstanceCount'],
+                             int(variant['InitialInstanceCount']))
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_endpoint_config')
+    def test_execute(self, mock_model, mock_client):
+        mock_model.return_value = {
+            'EndpointConfigArn': 'testarn',
+            'ResponseMetadata': {
+                'HTTPStatusCode': 200
+            }
+        }
+        self.sagemaker.execute(None)
+        mock_model.assert_called_once_with(create_endpoint_config_params)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    def test_execute_with_failure(self, mock_model, mock_client):
+        mock_model.return_value = {
+            'EndpointConfigArn': 'testarn',
+            'ResponseMetadata': {
+                'HTTPStatusCode': 200
+            }
+        }
+        self.assertRaises(AirflowException, self.sagemaker.execute, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_endpoint_operator.py b/tests/contrib/operators/test_sagemaker_endpoint_operator.py
new file mode 100644
index 0000000000000..cc6f118855d69
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_endpoint_operator.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.operators.sagemaker_endpoint_operator \
+    import SageMakerEndpointOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+role = 'arn:aws:iam:role/test-role'
+bucket = 'test-bucket'
+image = 'test-image'
+output_url = 's3://{}/test/output'.format(bucket)
+model_name = 'test-model-name'
+config_name = 'test-endpoint-config-name'
+endpoint_name = 'test-endpoint-name'
+
+create_model_params = {
+    'ModelName': model_name,
+    'PrimaryContainer': {
+        'Image': image,
+        'ModelDataUrl': output_url,
+    },
+    'ExecutionRoleArn': role
+}
+
+create_endpoint_config_params = {
+    'EndpointConfigName': config_name,
+    'ProductionVariants': [
+        {
+            'VariantName': 'AllTraffic',
+            'ModelName': model_name,
+            'InitialInstanceCount': '1',
+            'InstanceType': 'ml.c4.xlarge'
+        }
+    ]
+}
+
+create_endpoint_params = {
+    'EndpointName': endpoint_name,
+    'EndpointConfigName': config_name
+}
+
+config = {
+    'Model': create_model_params,
+    'EndpointConfig': create_endpoint_config_params,
+    'Endpoint': create_endpoint_params
+}
+
+
+class TestSageMakerEndpointOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerEndpointOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_id',
+            config=config,
+            wait_for_completion=False,
+            check_interval=5,
+            operation='create'
+        )
+
+    def test_parse_config_integers(self):
+        self.sagemaker.parse_config_integers()
+        for variant in self.sagemaker.config['EndpointConfig']['ProductionVariants']:
+            self.assertEqual(variant['InitialInstanceCount'],
+                             int(variant['InitialInstanceCount']))
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    @mock.patch.object(SageMakerHook, 'create_endpoint_config')
+    @mock.patch.object(SageMakerHook, 'create_endpoint')
+    def test_execute(self, mock_endpoint, mock_endpoint_config,
+                     mock_model, mock_client):
+        mock_endpoint.return_value = {'EndpointArn': 'testarn',
+                                      'ResponseMetadata':
+                                      {'HTTPStatusCode': 200}}
+        self.sagemaker.execute(None)
+        mock_model.assert_called_once_with(create_model_params)
+        mock_endpoint_config.assert_called_once_with(create_endpoint_config_params)
+        mock_endpoint.assert_called_once_with(create_endpoint_params,
+                                              wait_for_completion=False,
+                                              check_interval=5,
+                                              max_ingestion_time=None
+                                              )
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    @mock.patch.object(SageMakerHook, 'create_endpoint_config')
+    @mock.patch.object(SageMakerHook, 'create_endpoint')
+    def test_execute_with_failure(self, mock_endpoint, mock_endpoint_config,
+                                  mock_model, mock_client):
+        mock_endpoint.return_value = {'EndpointArn': 'testarn',
+                                      'ResponseMetadata':
+                                      {'HTTPStatusCode': 404}}
+        self.assertRaises(AirflowException, self.sagemaker.execute, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_model_operator.py b/tests/contrib/operators/test_sagemaker_model_operator.py
new file mode 100644
index 0000000000000..c01e466e161d4
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_model_operator.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.operators.sagemaker_model_operator \
+    import SageMakerModelOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+role = 'arn:aws:iam:role/test-role'
+
+bucket = 'test-bucket'
+
+model_name = 'test-model-name'
+
+image = 'test-image'
+
+output_url = 's3://{}/test/output'.format(bucket)
+create_model_params = {
+    'ModelName': model_name,
+    'PrimaryContainer': {
+        'Image': image,
+        'ModelDataUrl': output_url,
+    },
+    'ExecutionRoleArn': role
+}
+
+
+class TestSageMakerModelOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerModelOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_id',
+            config=create_model_params
+        )
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    def test_execute(self, mock_model, mock_client):
+        mock_model.return_value = {'ModelArn': 'testarn',
+                                   'ResponseMetadata':
+                                       {'HTTPStatusCode': 200}}
+        self.sagemaker.execute(None)
+        mock_model.assert_called_once_with(create_model_params)
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    def test_execute_with_failure(self, mock_model, mock_client):
+        mock_model.return_value = {'ModelArn': 'testarn',
+                                   'ResponseMetadata':
+                                       {'HTTPStatusCode': 404}}
+        self.assertRaises(AirflowException, self.sagemaker.execute, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_training_operator.py b/tests/contrib/operators/test_sagemaker_training_operator.py
new file mode 100644
index 0000000000000..8d81c6147b486
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_training_operator.py
@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# 'License'); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.operators.sagemaker_training_operator \
+    import SageMakerTrainingOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+role = 'arn:aws:iam:role/test-role'
+
+bucket = 'test-bucket'
+
+key = 'test/data'
+data_url = 's3://{}/{}'.format(bucket, key)
+
+job_name = 'test-job-name'
+
+image = 'test-image'
+
+output_url = 's3://{}/test/output'.format(bucket)
+create_training_params = \
+    {
+        'AlgorithmSpecification': {
+            'TrainingImage': image,
+            'TrainingInputMode': 'File'
+        },
+        'RoleArn': role,
+        'OutputDataConfig': {
+            'S3OutputPath': output_url
+        },
+        'ResourceConfig': {
+            'InstanceCount': '2',
+            'InstanceType': 'ml.c4.8xlarge',
+            'VolumeSizeInGB': '50'
+        },
+        'TrainingJobName': job_name,
+        'HyperParameters': {
+            'k': '10',
+            'feature_dim': '784',
+            'mini_batch_size': '500',
+            'force_dense': 'True'
+        },
+        'StoppingCondition': {
+            'MaxRuntimeInSeconds': '3600'
+        },
+        'InputDataConfig': [
+            {
+                'ChannelName': 'train',
+                'DataSource': {
+                    'S3DataSource': {
+                        'S3DataType': 'S3Prefix',
+                        'S3Uri': data_url,
+                        'S3DataDistributionType': 'FullyReplicated'
+                    }
+                },
+                'CompressionType': 'None',
+                'RecordWrapperType': 'None'
+            }
+        ]
+    }
+
+
+class TestSageMakerTrainingOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerTrainingOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_id',
+            config=create_training_params,
+            wait_for_completion=False,
+            check_interval=5
+        )
+
+    def test_parse_config_integers(self):
+        self.sagemaker.parse_config_integers()
+        self.assertEqual(self.sagemaker.config['ResourceConfig']['InstanceCount'],
+                         int(self.sagemaker.config['ResourceConfig']['InstanceCount']))
+        self.assertEqual(self.sagemaker.config['ResourceConfig']['VolumeSizeInGB'],
+                         int(self.sagemaker.config['ResourceConfig']['VolumeSizeInGB']))
+        self.assertEqual(self.sagemaker.config['StoppingCondition']['MaxRuntimeInSeconds'],
+                         int(self.sagemaker.config['StoppingCondition']['MaxRuntimeInSeconds']))
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_training_job')
+    def test_execute(self, mock_training, mock_client):
+        mock_training.return_value = {'TrainingJobArn': 'testarn',
+                                      'ResponseMetadata':
+                                          {'HTTPStatusCode': 200}}
+        self.sagemaker.execute(None)
+        mock_training.assert_called_once_with(create_training_params,
+                                              wait_for_completion=False,
+                                              print_log=True,
+                                              check_interval=5,
+                                              max_ingestion_time=None
+                                              )
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_training_job')
+    def test_execute_with_failure(self, mock_training, mock_client):
+        mock_training.return_value = {'TrainingJobArn': 'testarn',
+                                      'ResponseMetadata':
+                                          {'HTTPStatusCode': 404}}
+        self.assertRaises(AirflowException, self.sagemaker.execute, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_transform_operator.py b/tests/contrib/operators/test_sagemaker_transform_operator.py
new file mode 100644
index 0000000000000..74825bbdb2ba7
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_transform_operator.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.operators.sagemaker_transform_operator \
+    import SageMakerTransformOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+role = 'arn:aws:iam:role/test-role'
+
+bucket = 'test-bucket'
+
+key = 'test/data'
+data_url = 's3://{}/{}'.format(bucket, key)
+
+job_name = 'test-job-name'
+
+model_name = 'test-model-name'
+
+image = 'test-image'
+
+output_url = 's3://{}/test/output'.format(bucket)
+
+create_transform_params = {
+    'TransformJobName': job_name,
+    'ModelName': model_name,
+    'MaxConcurrentTransforms': '12',
+    'MaxPayloadInMB': '6',
+    'BatchStrategy': 'MultiRecord',
+    'TransformInput': {
+        'DataSource': {
+            'S3DataSource': {
+                'S3DataType': 'S3Prefix',
+                'S3Uri': data_url
+            }
+        }
+    },
+    'TransformOutput': {
+        'S3OutputPath': output_url,
+    },
+    'TransformResources': {
+        'InstanceType': 'ml.m4.xlarge',
+        'InstanceCount': '3'
+    }
+}
+
+create_model_params = {
+    'ModelName': model_name,
+    'PrimaryContainer': {
+        'Image': image,
+        'ModelDataUrl': output_url,
+    },
+    'ExecutionRoleArn': role
+}
+
+config = {
+    'Model': create_model_params,
+    'Transform': create_transform_params
+}
+
+
+class TestSageMakerTransformOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerTransformOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_id',
+            config=config,
+            wait_for_completion=False,
+            check_interval=5
+        )
+
+    def test_parse_config_integers(self):
+        self.sagemaker.parse_config_integers()
+        test_config = self.sagemaker.config['Transform']
+        self.assertEqual(test_config['TransformResources']['InstanceCount'],
+                         int(test_config['TransformResources']['InstanceCount']))
+        self.assertEqual(test_config['MaxConcurrentTransforms'],
+                         int(test_config['MaxConcurrentTransforms']))
+        self.assertEqual(test_config['MaxPayloadInMB'],
+                         int(test_config['MaxPayloadInMB']))
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    @mock.patch.object(SageMakerHook, 'create_transform_job')
+    def test_execute(self, mock_transform, mock_model, mock_client):
+        mock_transform.return_value = {'TransformJobArn': 'testarn',
+                                       'ResponseMetadata':
+                                       {'HTTPStatusCode': 200}}
+        self.sagemaker.execute(None)
+        mock_model.assert_called_once_with(create_model_params)
+        mock_transform.assert_called_once_with(create_transform_params,
+                                               wait_for_completion=False,
+                                               check_interval=5,
+                                               max_ingestion_time=None
+                                               )
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_model')
+    @mock.patch.object(SageMakerHook, 'create_transform_job')
+    def test_execute_with_failure(self, mock_transform, mock_model, mock_client):
+        mock_transform.return_value = {'TransformJobArn': 'testarn',
+                                       'ResponseMetadata':
+                                       {'HTTPStatusCode': 404}}
+        self.assertRaises(AirflowException, self.sagemaker.execute, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sagemaker_tuning_operator.py b/tests/contrib/operators/test_sagemaker_tuning_operator.py
new file mode 100644
index 0000000000000..088a055296c9a
--- /dev/null
+++ b/tests/contrib/operators/test_sagemaker_tuning_operator.py
@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.contrib.operators.sagemaker_tuning_operator \
+    import SageMakerTuningOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+role = 'arn:aws:iam:role/test-role'
+
+bucket = 'test-bucket'
+
+key = 'test/data'
+data_url = 's3://{}/{}'.format(bucket, key)
+
+job_name = 'test-job-name'
+
+image = 'test-image'
+
+output_url = 's3://{}/test/output'.format(bucket)
+
+create_tuning_params = {'HyperParameterTuningJobName': job_name,
+                        'HyperParameterTuningJobConfig': {
+                            'Strategy': 'Bayesian',
+                            'HyperParameterTuningJobObjective': {
+                                'Type': 'Maximize',
+                                'MetricName': 'test_metric'
+                            },
+                            'ResourceLimits': {
+                                'MaxNumberOfTrainingJobs': '123',
+                                'MaxParallelTrainingJobs': '123'
+                            },
+                            'ParameterRanges': {
+                                'IntegerParameterRanges': [
+                                    {
+                                        'Name': 'k',
+                                        'MinValue': '2',
+                                        'MaxValue': '10'
+                                    },
+                                ]
+                            }
+                        },
+                        'TrainingJobDefinition': {
+                            'StaticHyperParameters':
+                                {
+                                    'k': '10',
+                                    'feature_dim': '784',
+                                    'mini_batch_size': '500',
+                                    'force_dense': 'True'
+                                },
+                            'AlgorithmSpecification':
+                                {
+                                    'TrainingImage': image,
+                                    'TrainingInputMode': 'File'
+                                },
+                            'RoleArn': role,
+                            'InputDataConfig':
+                                [
+                                    {
+                                        'ChannelName': 'train',
+                                        'DataSource': {
+                                            'S3DataSource': {
+                                                'S3DataType': 'S3Prefix',
+                                                'S3Uri': data_url,
+                                                'S3DataDistributionType':
+                                                    'FullyReplicated'
+                                            }
+                                        },
+                                        'CompressionType': 'None',
+                                        'RecordWrapperType': 'None'
+                                    }
+                                ],
+                            'OutputDataConfig':
+                                {
+                                    'S3OutputPath': output_url
+                                },
+                            'ResourceConfig':
+                                {
+                                    'InstanceCount': '2',
+                                    'InstanceType': 'ml.c4.8xlarge',
+                                    'VolumeSizeInGB': '50'
+                                },
+                            'StoppingCondition': dict(MaxRuntimeInSeconds=60 * 60)
+                        }
+                        }
+
+
+class TestSageMakerTuningOperator(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.sagemaker = SageMakerTuningOperator(
+            task_id='test_sagemaker_operator',
+            aws_conn_id='sagemaker_test_conn',
+            config=create_tuning_params,
+            wait_for_completion=False,
+            check_interval=5
+        )
+
+    def test_parse_config_integers(self):
+        self.sagemaker.parse_config_integers()
+        self.assertEqual(self.sagemaker.config['TrainingJobDefinition']['ResourceConfig']
+                         ['InstanceCount'],
+                         int(self.sagemaker.config['TrainingJobDefinition']['ResourceConfig']
+                             ['InstanceCount']))
+        self.assertEqual(self.sagemaker.config['TrainingJobDefinition']['ResourceConfig']
+                         ['VolumeSizeInGB'],
+                         int(self.sagemaker.config['TrainingJobDefinition']['ResourceConfig']
+                             ['VolumeSizeInGB']))
+        self.assertEqual(self.sagemaker.config['HyperParameterTuningJobConfig']['ResourceLimits']
+                         ['MaxNumberOfTrainingJobs'],
+                         int(self.sagemaker.config['HyperParameterTuningJobConfig']['ResourceLimits']
+                             ['MaxNumberOfTrainingJobs']))
+        self.assertEqual(self.sagemaker.config['HyperParameterTuningJobConfig']['ResourceLimits']
+                         ['MaxParallelTrainingJobs'],
+                         int(self.sagemaker.config['HyperParameterTuningJobConfig']['ResourceLimits']
+                             ['MaxParallelTrainingJobs']))
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_tuning_job')
+    def test_execute(self, mock_tuning, mock_client):
+        mock_tuning.return_value = {'TrainingJobArn': 'testarn',
+                                    'ResponseMetadata':
+                                    {'HTTPStatusCode': 200}}
+        self.sagemaker.execute(None)
+        mock_tuning.assert_called_once_with(create_tuning_params,
+                                            wait_for_completion=False,
+                                            check_interval=5,
+                                            max_ingestion_time=None
+                                            )
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'create_tuning_job')
+    def test_execute_with_failure(self, mock_tuning, mock_client):
+        mock_tuning.return_value = {'TrainingJobArn': 'testarn',
+                                    'ResponseMetadata':
+                                    {'HTTPStatusCode': 404}}
+        self.assertRaises(AirflowException, self.sagemaker.execute, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_sftp_operator.py b/tests/contrib/operators/test_sftp_operator.py
index 01446a6fddd49..66d793991de7a 100644
--- a/tests/contrib/operators/test_sftp_operator.py
+++ b/tests/contrib/operators/test_sftp_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,6 +20,7 @@
 import os
 import unittest
 from base64 import b64encode
+import six
 
 from airflow import configuration
 from airflow import models
@@ -41,6 +42,7 @@ def reset(dag_id=TEST_DAG_ID):
     session.commit()
     session.close()
 
+
 reset()
 
 
@@ -60,12 +62,20 @@ def setUp(self):
         self.hook = hook
         self.dag = dag
         self.test_dir = "/tmp"
+        self.test_local_dir = "/tmp/tmp2"
+        self.test_remote_dir = "/tmp/tmp1"
         self.test_local_filename = 'test_local_file'
         self.test_remote_filename = 'test_remote_file'
         self.test_local_filepath = '{0}/{1}'.format(self.test_dir,
                                                     self.test_local_filename)
+        # Local Filepath with Intermediate Directory
+        self.test_local_filepath_int_dir = '{0}/{1}'.format(self.test_local_dir,
+                                                            self.test_local_filename)
         self.test_remote_filepath = '{0}/{1}'.format(self.test_dir,
                                                      self.test_remote_filename)
+        # Remote Filepath with Intermediate Directory
+        self.test_remote_filepath_int_dir = '{0}/{1}'.format(self.test_remote_dir,
+                                                             self.test_remote_filename)
 
     def test_pickle_file_transfer_put(self):
         configuration.conf.set("core", "enable_xcom_pickling", "True")
@@ -78,12 +88,78 @@ def test_pickle_file_transfer_put(self):
 
         # put test file to remote
         put_test_task = SFTPOperator(
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.PUT,
+            create_intermediate_dirs=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(put_test_task)
+        ti2 = TaskInstance(task=put_test_task, execution_date=timezone.utcnow())
+        ti2.run()
+
+        # check the remote file content
+        check_file_task = SSHOperator(
+            task_id="test_check_file",
+            ssh_hook=self.hook,
+            command="cat {0}".format(self.test_remote_filepath),
+            do_xcom_push=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(check_file_task)
+        ti3 = TaskInstance(task=check_file_task, execution_date=timezone.utcnow())
+        ti3.run()
+        self.assertEqual(
+            ti3.xcom_pull(task_ids='test_check_file', key='return_value').strip(),
+            test_local_file_content)
+
+    def test_file_transfer_no_intermediate_dir_error_put(self):
+        configuration.conf.set("core", "enable_xcom_pickling", "True")
+        test_local_file_content = \
+            b"This is local file content \n which is multiline " \
+            b"continuing....with other character\nanother line here \n this is last line"
+        # create a test file locally
+        with open(self.test_local_filepath, 'wb') as f:
+            f.write(test_local_file_content)
+
+        # Try to put test file to remote
+        # This should raise an error with "No such file" as the directory
+        # does not exist
+        with self.assertRaises(Exception) as error:
+            put_test_task = SFTPOperator(
                 task_id="test_sftp",
                 ssh_hook=self.hook,
                 local_filepath=self.test_local_filepath,
-                remote_filepath=self.test_remote_filepath,
+                remote_filepath=self.test_remote_filepath_int_dir,
                 operation=SFTPOperation.PUT,
+                create_intermediate_dirs=False,
                 dag=self.dag
+            )
+            self.assertIsNotNone(put_test_task)
+            ti2 = TaskInstance(task=put_test_task, execution_date=timezone.utcnow())
+            ti2.run()
+        self.assertIn('No such file', str(error.exception))
+
+    def test_file_transfer_with_intermediate_dir_put(self):
+        configuration.conf.set("core", "enable_xcom_pickling", "True")
+        test_local_file_content = \
+            b"This is local file content \n which is multiline " \
+            b"continuing....with other character\nanother line here \n this is last line"
+        # create a test file locally
+        with open(self.test_local_filepath, 'wb') as f:
+            f.write(test_local_file_content)
+
+        # put test file to remote
+        put_test_task = SFTPOperator(
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath_int_dir,
+            operation=SFTPOperation.PUT,
+            create_intermediate_dirs=True,
+            dag=self.dag
         )
         self.assertIsNotNone(put_test_task)
         ti2 = TaskInstance(task=put_test_task, execution_date=timezone.utcnow())
@@ -91,18 +167,18 @@ def test_pickle_file_transfer_put(self):
 
         # check the remote file content
         check_file_task = SSHOperator(
-                task_id="test_check_file",
-                ssh_hook=self.hook,
-                command="cat {0}".format(self.test_remote_filepath),
-                do_xcom_push=True,
-                dag=self.dag
+            task_id="test_check_file",
+            ssh_hook=self.hook,
+            command="cat {0}".format(self.test_remote_filepath_int_dir),
+            do_xcom_push=True,
+            dag=self.dag
         )
         self.assertIsNotNone(check_file_task)
         ti3 = TaskInstance(task=check_file_task, execution_date=timezone.utcnow())
         ti3.run()
         self.assertEqual(
-                ti3.xcom_pull(task_ids='test_check_file', key='return_value').strip(),
-                test_local_file_content)
+            ti3.xcom_pull(task_ids='test_check_file', key='return_value').strip(),
+            test_local_file_content)
 
     def test_json_file_transfer_put(self):
         configuration.conf.set("core", "enable_xcom_pickling", "False")
@@ -115,12 +191,12 @@ def test_json_file_transfer_put(self):
 
         # put test file to remote
         put_test_task = SFTPOperator(
-                task_id="test_sftp",
-                ssh_hook=self.hook,
-                local_filepath=self.test_local_filepath,
-                remote_filepath=self.test_remote_filepath,
-                operation=SFTPOperation.PUT,
-                dag=self.dag
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.PUT,
+            dag=self.dag
         )
         self.assertIsNotNone(put_test_task)
         ti2 = TaskInstance(task=put_test_task, execution_date=timezone.utcnow())
@@ -128,19 +204,18 @@ def test_json_file_transfer_put(self):
 
         # check the remote file content
         check_file_task = SSHOperator(
-                task_id="test_check_file",
-                ssh_hook=self.hook,
-                command="cat {0}".format(self.test_remote_filepath),
-                do_xcom_push=True,
-                dag=self.dag
+            task_id="test_check_file",
+            ssh_hook=self.hook,
+            command="cat {0}".format(self.test_remote_filepath),
+            do_xcom_push=True,
+            dag=self.dag
         )
         self.assertIsNotNone(check_file_task)
         ti3 = TaskInstance(task=check_file_task, execution_date=timezone.utcnow())
         ti3.run()
         self.assertEqual(
-                ti3.xcom_pull(task_ids='test_check_file', key='return_value').strip(),
-                b64encode(test_local_file_content).decode('utf-8'))
-
+            ti3.xcom_pull(task_ids='test_check_file', key='return_value').strip(),
+            b64encode(test_local_file_content).decode('utf-8'))
 
     def test_pickle_file_transfer_get(self):
         configuration.conf.set("core", "enable_xcom_pickling", "True")
@@ -150,12 +225,12 @@ def test_pickle_file_transfer_get(self):
 
         # create a test file remotely
         create_file_task = SSHOperator(
-                task_id="test_create_file",
-                ssh_hook=self.hook,
-                command="echo '{0}' > {1}".format(test_remote_file_content,
-                                                  self.test_remote_filepath),
-                do_xcom_push=True,
-                dag=self.dag
+            task_id="test_create_file",
+            ssh_hook=self.hook,
+            command="echo '{0}' > {1}".format(test_remote_file_content,
+                                              self.test_remote_filepath),
+            do_xcom_push=True,
+            dag=self.dag
         )
         self.assertIsNotNone(create_file_task)
         ti1 = TaskInstance(task=create_file_task, execution_date=timezone.utcnow())
@@ -163,12 +238,12 @@ def test_pickle_file_transfer_get(self):
 
         # get remote file to local
         get_test_task = SFTPOperator(
-                task_id="test_sftp",
-                ssh_hook=self.hook,
-                local_filepath=self.test_local_filepath,
-                remote_filepath=self.test_remote_filepath,
-                operation=SFTPOperation.GET,
-                dag=self.dag
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.GET,
+            dag=self.dag
         )
         self.assertIsNotNone(get_test_task)
         ti2 = TaskInstance(task=get_test_task, execution_date=timezone.utcnow())
@@ -188,12 +263,12 @@ def test_json_file_transfer_get(self):
 
         # create a test file remotely
         create_file_task = SSHOperator(
-                task_id="test_create_file",
-                ssh_hook=self.hook,
-                command="echo '{0}' > {1}".format(test_remote_file_content,
-                                                  self.test_remote_filepath),
-                do_xcom_push=True,
-                dag=self.dag
+            task_id="test_create_file",
+            ssh_hook=self.hook,
+            command="echo '{0}' > {1}".format(test_remote_file_content,
+                                              self.test_remote_filepath),
+            do_xcom_push=True,
+            dag=self.dag
         )
         self.assertIsNotNone(create_file_task)
         ti1 = TaskInstance(task=create_file_task, execution_date=timezone.utcnow())
@@ -201,12 +276,88 @@ def test_json_file_transfer_get(self):
 
         # get remote file to local
         get_test_task = SFTPOperator(
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.GET,
+            dag=self.dag
+        )
+        self.assertIsNotNone(get_test_task)
+        ti2 = TaskInstance(task=get_test_task, execution_date=timezone.utcnow())
+        ti2.run()
+
+        # test the received content
+        content_received = None
+        with open(self.test_local_filepath, 'r') as f:
+            content_received = f.read()
+        self.assertEqual(content_received.strip(),
+                         test_remote_file_content.encode('utf-8').decode('utf-8'))
+
+    def test_file_transfer_no_intermediate_dir_error_get(self):
+        configuration.conf.set("core", "enable_xcom_pickling", "True")
+        test_remote_file_content = \
+            "This is remote file content \n which is also multiline " \
+            "another line here \n this is last line. EOF"
+
+        # create a test file remotely
+        create_file_task = SSHOperator(
+            task_id="test_create_file",
+            ssh_hook=self.hook,
+            command="echo '{0}' > {1}".format(test_remote_file_content,
+                                              self.test_remote_filepath),
+            do_xcom_push=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(create_file_task)
+        ti1 = TaskInstance(task=create_file_task, execution_date=timezone.utcnow())
+        ti1.run()
+
+        # Try to GET test file from remote
+        # This should raise an error with "No such file" as the directory
+        # does not exist
+        with self.assertRaises(Exception) as error:
+            get_test_task = SFTPOperator(
                 task_id="test_sftp",
                 ssh_hook=self.hook,
-                local_filepath=self.test_local_filepath,
+                local_filepath=self.test_local_filepath_int_dir,
                 remote_filepath=self.test_remote_filepath,
                 operation=SFTPOperation.GET,
                 dag=self.dag
+            )
+            self.assertIsNotNone(get_test_task)
+            ti2 = TaskInstance(task=get_test_task, execution_date=timezone.utcnow())
+            ti2.run()
+        self.assertIn('No such file', str(error.exception))
+
+    def test_file_transfer_with_intermediate_dir_error_get(self):
+        configuration.conf.set("core", "enable_xcom_pickling", "True")
+        test_remote_file_content = \
+            "This is remote file content \n which is also multiline " \
+            "another line here \n this is last line. EOF"
+
+        # create a test file remotely
+        create_file_task = SSHOperator(
+            task_id="test_create_file",
+            ssh_hook=self.hook,
+            command="echo '{0}' > {1}".format(test_remote_file_content,
+                                              self.test_remote_filepath),
+            do_xcom_push=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(create_file_task)
+        ti1 = TaskInstance(task=create_file_task, execution_date=timezone.utcnow())
+        ti1.run()
+
+        # get remote file to local
+        get_test_task = SFTPOperator(
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            local_filepath=self.test_local_filepath_int_dir,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.GET,
+            create_intermediate_dirs=True,
+            dag=self.dag
         )
         self.assertIsNotNone(get_test_task)
         ti2 = TaskInstance(task=get_test_task, execution_date=timezone.utcnow())
@@ -214,30 +365,104 @@ def test_json_file_transfer_get(self):
 
         # test the received content
         content_received = None
-        with open(self.test_local_filepath, 'r') as f:
+        with open(self.test_local_filepath_int_dir, 'r') as f:
             content_received = f.read()
-        self.assertEqual(content_received.strip(),
-            test_remote_file_content.encode('utf-8').decode('utf-8'))
+        self.assertEqual(content_received.strip(), test_remote_file_content)
+
+    def test_arg_checking(self):
+        from airflow.exceptions import AirflowException
+        conn_id = "conn_id_for_testing"
+        os.environ['AIRFLOW_CONN_' + conn_id.upper()] = "ssh://test_id@localhost"
+
+        # Exception should be raised if neither ssh_hook nor ssh_conn_id is provided
+        if six.PY2:
+            self.assertRaisesRegex = self.assertRaisesRegexp
+        with self.assertRaisesRegex(AirflowException,
+                                    "Cannot operate without ssh_hook or ssh_conn_id."):
+            task_0 = SFTPOperator(
+                task_id="test_sftp",
+                local_filepath=self.test_local_filepath,
+                remote_filepath=self.test_remote_filepath,
+                operation=SFTPOperation.PUT,
+                dag=self.dag
+            )
+            task_0.execute(None)
+
+        # if ssh_hook is invalid/not provided, use ssh_conn_id to create SSHHook
+        task_1 = SFTPOperator(
+            task_id="test_sftp",
+            ssh_hook="string_rather_than_SSHHook",  # invalid ssh_hook
+            ssh_conn_id=conn_id,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.PUT,
+            dag=self.dag
+        )
+        try:
+            task_1.execute(None)
+        except Exception:
+            pass
+        self.assertEqual(task_1.ssh_hook.ssh_conn_id, conn_id)
+
+        task_2 = SFTPOperator(
+            task_id="test_sftp",
+            ssh_conn_id=conn_id,  # no ssh_hook provided
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.PUT,
+            dag=self.dag
+        )
+        try:
+            task_2.execute(None)
+        except Exception:
+            pass
+        self.assertEqual(task_2.ssh_hook.ssh_conn_id, conn_id)
+
+        # if both valid ssh_hook and ssh_conn_id are provided, ignore ssh_conn_id
+        task_3 = SFTPOperator(
+            task_id="test_sftp",
+            ssh_hook=self.hook,
+            ssh_conn_id=conn_id,
+            local_filepath=self.test_local_filepath,
+            remote_filepath=self.test_remote_filepath,
+            operation=SFTPOperation.PUT,
+            dag=self.dag
+        )
+        try:
+            task_3.execute(None)
+        except Exception:
+            pass
+        self.assertEqual(task_3.ssh_hook.ssh_conn_id, self.hook.ssh_conn_id)
 
     def delete_local_resource(self):
         if os.path.exists(self.test_local_filepath):
             os.remove(self.test_local_filepath)
+        if os.path.exists(self.test_local_filepath_int_dir):
+            os.remove(self.test_local_filepath_int_dir)
+        if os.path.exists(self.test_local_dir):
+            os.rmdir(self.test_local_dir)
 
     def delete_remote_resource(self):
-        # check the remote file content
-        remove_file_task = SSHOperator(
+        if os.path.exists(self.test_remote_filepath):
+            # check the remote file content
+            remove_file_task = SSHOperator(
                 task_id="test_check_file",
                 ssh_hook=self.hook,
                 command="rm {0}".format(self.test_remote_filepath),
                 do_xcom_push=True,
                 dag=self.dag
-        )
-        self.assertIsNotNone(remove_file_task)
-        ti3 = TaskInstance(task=remove_file_task, execution_date=timezone.utcnow())
-        ti3.run()
+            )
+            self.assertIsNotNone(remove_file_task)
+            ti3 = TaskInstance(task=remove_file_task, execution_date=timezone.utcnow())
+            ti3.run()
+        if os.path.exists(self.test_remote_filepath_int_dir):
+            os.remove(self.test_remote_filepath_int_dir)
+        if os.path.exists(self.test_remote_dir):
+            os.rmdir(self.test_remote_dir)
 
     def tearDown(self):
-        self.delete_local_resource() and self.delete_remote_resource()
+        self.delete_local_resource()
+        self.delete_remote_resource()
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/operators/test_sftp_to_s3_operator.py b/tests/contrib/operators/test_sftp_to_s3_operator.py
new file mode 100644
index 0000000000000..4be3a71d208f6
--- /dev/null
+++ b/tests/contrib/operators/test_sftp_to_s3_operator.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow import models
+from airflow.contrib.operators.sftp_to_s3_operator import SFTPToS3Operator
+from airflow.contrib.operators.ssh_operator import SSHOperator
+from airflow.models import DAG, TaskInstance
+from airflow.settings import Session
+from airflow.utils import timezone
+from airflow.utils.timezone import datetime
+from airflow.contrib.hooks.ssh_hook import SSHHook
+from airflow.hooks.S3_hook import S3Hook
+
+import boto3
+from moto import mock_s3
+
+BUCKET = 'test-bucket'
+S3_KEY = 'test/test_1_file.csv'
+SFTP_PATH = '/tmp/remote_path.txt'
+SFTP_CONN_ID = 'ssh_default'
+S3_CONN_ID = 'aws_default'
+
+SFTP_MOCK_FILE = 'test_sftp_file.csv'
+S3_MOCK_FILES = 'test_1_file.csv'
+
+TEST_DAG_ID = 'unit_tests'
+DEFAULT_DATE = datetime(2018, 1, 1)
+
+
+def reset(dag_id=TEST_DAG_ID):
+    session = Session()
+    tis = session.query(models.TaskInstance).filter_by(dag_id=dag_id)
+    tis.delete()
+    session.commit()
+    session.close()
+
+
+reset()
+
+
+class SFTPToS3OperatorTest(unittest.TestCase):
+
+    @mock_s3
+    def setUp(self):
+        configuration.load_test_config()
+
+        hook = SSHHook(ssh_conn_id='ssh_default')
+        s3_hook = S3Hook('aws_default')
+        hook.no_host_key_check = True
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE,
+            'provide_context': True
+        }
+        dag = DAG(TEST_DAG_ID + 'test_schedule_dag_once', default_args=args)
+        dag.schedule_interval = '@once'
+
+        self.hook = hook
+        self.s3_hook = s3_hook
+
+        self.ssh_client = self.hook.get_conn()
+        self.sftp_client = self.ssh_client.open_sftp()
+
+        self.dag = dag
+        self.s3_bucket = BUCKET
+        self.sftp_path = SFTP_PATH
+        self.s3_key = S3_KEY
+
+    @mock_s3
+    def test_sftp_to_s3_operation(self):
+        # Setting
+        configuration.conf.set("core", "enable_xcom_pickling", "True")
+        test_remote_file_content = \
+            "This is remote file content \n which is also multiline " \
+            "another line here \n this is last line. EOF"
+
+        # create a test file remotely
+        create_file_task = SSHOperator(
+            task_id="test_create_file",
+            ssh_hook=self.hook,
+            command="echo '{0}' > {1}".format(test_remote_file_content,
+                                              self.sftp_path),
+            do_xcom_push=True,
+            dag=self.dag
+        )
+        self.assertIsNotNone(create_file_task)
+        ti1 = TaskInstance(task=create_file_task, execution_date=timezone.utcnow())
+        ti1.run()
+
+        # Test for creation of s3 bucket
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=self.s3_bucket)
+        self.assertTrue((self.s3_hook.check_for_bucket(self.s3_bucket)))
+
+        # get remote file to local
+        run_task = SFTPToS3Operator(
+            s3_bucket=BUCKET,
+            s3_key=S3_KEY,
+            sftp_path=SFTP_PATH,
+            sftp_conn_id=SFTP_CONN_ID,
+            s3_conn_id=S3_CONN_ID,
+            task_id='test_sftp_to_s3',
+            dag=self.dag
+        )
+        self.assertIsNotNone(run_task)
+
+        run_task.execute(None)
+
+        # Check if object was created in s3
+        objects_in_dest_bucket = conn.list_objects(Bucket=self.s3_bucket,
+                                                   Prefix=self.s3_key)
+        # there should be object found, and there should only be one object found
+        self.assertEqual(len(objects_in_dest_bucket['Contents']), 1)
+
+        # the object found should be consistent with dest_key specified earlier
+        self.assertEqual(objects_in_dest_bucket['Contents'][0]['Key'], self.s3_key)
+
+        # Clean up after finishing with test
+        conn.delete_object(Bucket=self.s3_bucket, Key=self.s3_key)
+        conn.delete_bucket(Bucket=self.s3_bucket)
+        self.assertFalse((self.s3_hook.check_for_bucket(self.s3_bucket)))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_slack_webhook_operator.py b/tests/contrib/operators/test_slack_webhook_operator.py
index 544b790eecda0..68078bb8018a9 100644
--- a/tests/contrib/operators/test_slack_webhook_operator.py
+++ b/tests/contrib/operators/test_slack_webhook_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -33,6 +33,7 @@ class TestSlackWebhookOperator(unittest.TestCase):
         'http_conn_id': 'slack-webhook-default',
         'webhook_token': 'manual_token',
         'message': 'your message here',
+        'attachments': [{'fallback': 'Required plain-text summary'}],
         'channel': '#general',
         'username': 'SlackMcSlackFace',
         'icon_emoji': ':hankey',
@@ -59,6 +60,7 @@ def test_execute(self):
         self.assertEqual(self._config['http_conn_id'], operator.http_conn_id)
         self.assertEqual(self._config['webhook_token'], operator.webhook_token)
         self.assertEqual(self._config['message'], operator.message)
+        self.assertEqual(self._config['attachments'], operator.attachments)
         self.assertEqual(self._config['channel'], operator.channel)
         self.assertEqual(self._config['username'], operator.username)
         self.assertEqual(self._config['icon_emoji'], operator.icon_emoji)
diff --git a/tests/contrib/operators/test_snowflake_operator.py b/tests/contrib/operators/test_snowflake_operator.py
index 669bd65563021..8b394a7d5e0e3 100644
--- a/tests/contrib/operators/test_snowflake_operator.py
+++ b/tests/contrib/operators/test_snowflake_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,14 +25,7 @@
 from airflow.utils import timezone
 
 from airflow.contrib.operators.snowflake_operator import SnowflakeOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 DEFAULT_DATE = timezone.datetime(2015, 1, 1)
diff --git a/tests/contrib/operators/test_sns_publish_operator.py b/tests/contrib/operators/test_sns_publish_operator.py
new file mode 100644
index 0000000000000..d30bfa57f8dad
--- /dev/null
+++ b/tests/contrib/operators/test_sns_publish_operator.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import mock
+import unittest
+
+from airflow.contrib.operators.sns_publish_operator import SnsPublishOperator
+
+TASK_ID = "sns_publish_job"
+AWS_CONN_ID = "custom_aws_conn"
+TARGET_ARN = "arn:aws:sns:eu-central-1:1234567890:test-topic"
+MESSAGE = "Message to send"
+
+
+class TestSnsPublishOperator(unittest.TestCase):
+
+    def test_init(self):
+        # Given / When
+        operator = SnsPublishOperator(
+            task_id=TASK_ID,
+            aws_conn_id=AWS_CONN_ID,
+            target_arn=TARGET_ARN,
+            message=MESSAGE
+        )
+
+        # Then
+        self.assertEqual(TASK_ID, operator.task_id)
+        self.assertEqual(AWS_CONN_ID, operator.aws_conn_id)
+        self.assertEqual(TARGET_ARN, operator.target_arn)
+        self.assertEqual(MESSAGE, operator.message)
+
+    @mock.patch('airflow.contrib.operators.sns_publish_operator.AwsSnsHook')
+    def test_execute(self, mock_hook):
+        # Given
+        hook_response = {'MessageId': 'foobar'}
+
+        hook_instance = mock_hook.return_value
+        hook_instance.publish_to_target.return_value = hook_response
+
+        operator = SnsPublishOperator(
+            task_id=TASK_ID,
+            aws_conn_id=AWS_CONN_ID,
+            target_arn=TARGET_ARN,
+            message=MESSAGE
+        )
+
+        # When
+        result = operator.execute(None)
+
+        # Then
+        self.assertEqual(hook_response, result)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/operators/test_spark_jdbc_operator.py b/tests/contrib/operators/test_spark_jdbc_operator.py
index 9f2cd23f8f247..652c2bc6306bb 100644
--- a/tests/contrib/operators/test_spark_jdbc_operator.py
+++ b/tests/contrib/operators/test_spark_jdbc_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/operators/test_spark_sql_operator.py b/tests/contrib/operators/test_spark_sql_operator.py
index 8fa1428e5a5cb..bbe6868dc71f9 100644
--- a/tests/contrib/operators/test_spark_sql_operator.py
+++ b/tests/contrib/operators/test_spark_sql_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,7 +28,6 @@
 
 
 class TestSparkSqlOperator(unittest.TestCase):
-
     _config = {
         'sql': 'SELECT 22',
         'conn_id': 'spark_special_conn_id',
@@ -74,5 +73,6 @@ def test_execute(self):
         self.assertEqual(self._config['num_executors'], operator._num_executors)
         self.assertEqual(self._config['yarn_queue'], operator._yarn_queue)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/operators/test_spark_submit_operator.py b/tests/contrib/operators/test_spark_submit_operator.py
index 0a528d9b1904c..be596ca6ea2eb 100644
--- a/tests/contrib/operators/test_spark_submit_operator.py
+++ b/tests/contrib/operators/test_spark_submit_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,6 +28,7 @@
 
 from datetime import timedelta
 
+
 DEFAULT_DATE = timezone.datetime(2017, 1, 1)
 
 
@@ -39,7 +40,8 @@ class TestSparkSubmitOperator(unittest.TestCase):
         },
         'files': 'hive-site.xml',
         'py_files': 'sample_library.py',
-        'driver_classpath': 'parquet.jar',
+        'archives': 'sample_archive.zip#SAMPLE',
+        'driver_class_path': 'parquet.jar',
         'jars': 'parquet.jar',
         'packages': 'com.databricks:spark-avro_2.11:3.2.0',
         'exclude_packages': 'org.bad.dependency:1.0.0',
@@ -73,22 +75,25 @@ def setUp(self):
         self.dag = DAG('test_dag_id', default_args=args)
 
     def test_execute(self):
+
         # Given / When
         conn_id = 'spark_default'
         operator = SparkSubmitOperator(
             task_id='spark_submit_job',
+            spark_binary="sparky",
             dag=self.dag,
             **self._config
         )
 
-        # Then
+        # Then expected results
         expected_dict = {
             'conf': {
                 'parquet.compression': 'SNAPPY'
             },
             'files': 'hive-site.xml',
             'py_files': 'sample_library.py',
-            'driver_classpath': 'parquet.jar',
+            'archives': 'sample_archive.zip#SAMPLE',
+            'driver_class_path': 'parquet.jar',
             'jars': 'parquet.jar',
             'packages': 'com.databricks:spark-avro_2.11:3.2.0',
             'exclude_packages': 'org.bad.dependency:1.0.0',
@@ -110,7 +115,8 @@ def test_execute(self):
                 '--start', '{{ macros.ds_add(ds, -1)}}',
                 '--end', '{{ ds }}',
                 '--with-spaces', 'args should keep embdedded spaces',
-            ]
+            ],
+            "spark_binary": "sparky"
         }
 
         self.assertEqual(conn_id, operator._conn_id)
@@ -118,7 +124,8 @@ def test_execute(self):
         self.assertEqual(expected_dict['conf'], operator._conf)
         self.assertEqual(expected_dict['files'], operator._files)
         self.assertEqual(expected_dict['py_files'], operator._py_files)
-        self.assertEqual(expected_dict['driver_classpath'], operator._driver_classpath)
+        self.assertEqual(expected_dict['archives'], operator._archives)
+        self.assertEqual(expected_dict['driver_class_path'], operator._driver_class_path)
         self.assertEqual(expected_dict['jars'], operator._jars)
         self.assertEqual(expected_dict['packages'], operator._packages)
         self.assertEqual(expected_dict['exclude_packages'], operator._exclude_packages)
@@ -135,6 +142,7 @@ def test_execute(self):
         self.assertEqual(expected_dict['java_class'], operator._java_class)
         self.assertEqual(expected_dict['driver_memory'], operator._driver_memory)
         self.assertEqual(expected_dict['application_args'], operator._application_args)
+        self.assertEqual(expected_dict['spark_binary'], operator._spark_binary)
 
     def test_render_template(self):
         # Given
diff --git a/tests/contrib/operators/test_sqoop_operator.py b/tests/contrib/operators/test_sqoop_operator.py
index 83ffa6fb796d6..484d74b801748 100644
--- a/tests/contrib/operators/test_sqoop_operator.py
+++ b/tests/contrib/operators/test_sqoop_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -64,7 +64,8 @@ class TestSqoopOperator(unittest.TestCase):
         },
         'extra_export_options': {
             'update-key': 'id',
-            'update-mode': 'allowinsert'
+            'update-mode': 'allowinsert',
+            'fetch-size': 1
         }
     }
 
@@ -111,7 +112,7 @@ def test_execute(self):
         self.assertEqual(self._config['extra_export_options'], operator.extra_export_options)
 
         # the following are meant to be more of examples
-        sqoop_import_op = SqoopOperator(
+        SqoopOperator(
             task_id='sqoop_import_using_table',
             cmd_type='import',
             conn_id='sqoop_default',
@@ -125,12 +126,13 @@ def test_execute(self):
             dag=self.dag
         )
 
-        sqoop_import_op_qry = SqoopOperator(
+        SqoopOperator(
             task_id='sqoop_import_using_query',
             cmd_type='import',
             conn_id='sqoop_default',
             query='select name, age from company where $CONDITIONS',
-            split_by='age', # the mappers will pass in values to the $CONDITIONS based on the field you select to split by
+            split_by='age',
+            # the mappers will pass in values to the $CONDITIONS based on the field you select to split by
             verbose=True,
             num_mappers=None,
             hcatalog_database='default',
@@ -140,7 +142,7 @@ def test_execute(self):
             dag=self.dag
         )
 
-        sqoop_import_op_with_partition = SqoopOperator(
+        SqoopOperator(
             task_id='sqoop_import_with_partition',
             cmd_type='import',
             conn_id='sqoop_default',
@@ -153,11 +155,13 @@ def test_execute(self):
             extra_import_options={
                 'hcatalog-storage-stanza': "\"stored as orcfile\"",
                 'hive-partition-key': 'day',
-                'hive-partition-value': '2017-10-18'},
+                'hive-partition-value': '2017-10-18',
+                'fetch-size': 1
+            },
             dag=self.dag
         )
 
-        sqoop_export_op_name = SqoopOperator(
+        SqoopOperator(
             task_id='sqoop_export_tablename',
             cmd_type='export',
             conn_id='sqoop_default',
@@ -170,7 +174,7 @@ def test_execute(self):
             dag=self.dag
         )
 
-        sqoop_export_op_path = SqoopOperator(
+        SqoopOperator(
             task_id='sqoop_export_tablepath',
             cmd_type='export',
             conn_id='sqoop_default',
diff --git a/tests/contrib/operators/test_ssh_operator.py b/tests/contrib/operators/test_ssh_operator.py
index b97ba84a012a2..a8b6841f11282 100644
--- a/tests/contrib/operators/test_ssh_operator.py
+++ b/tests/contrib/operators/test_ssh_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,6 +19,7 @@
 
 import unittest
 from base64 import b64encode
+import six
 
 from airflow import configuration
 from airflow import models
@@ -39,6 +40,7 @@ def reset(dag_id=TEST_DAG_ID):
     session.commit()
     session.close()
 
+
 reset()
 
 
@@ -58,20 +60,37 @@ def setUp(self):
         self.hook = hook
         self.dag = dag
 
+    def test_hook_created_correctly(self):
+        TIMEOUT = 20
+        SSH_ID = "ssh_default"
+        task = SSHOperator(
+            task_id="test",
+            command="echo -n airflow",
+            dag=self.dag,
+            timeout=TIMEOUT,
+            ssh_conn_id="ssh_default"
+        )
+        self.assertIsNotNone(task)
+
+        task.execute(None)
+
+        self.assertEqual(TIMEOUT, task.ssh_hook.timeout)
+        self.assertEqual(SSH_ID, task.ssh_hook.ssh_conn_id)
+
     def test_json_command_execution(self):
         configuration.conf.set("core", "enable_xcom_pickling", "False")
         task = SSHOperator(
-                task_id="test",
-                ssh_hook=self.hook,
-                command="echo -n airflow",
-                do_xcom_push=True,
-                dag=self.dag,
+            task_id="test",
+            ssh_hook=self.hook,
+            command="echo -n airflow",
+            do_xcom_push=True,
+            dag=self.dag,
         )
 
         self.assertIsNotNone(task)
 
         ti = TaskInstance(
-                task=task, execution_date=timezone.utcnow())
+            task=task, execution_date=timezone.utcnow())
         ti.run()
         self.assertIsNotNone(ti.duration)
         self.assertEqual(ti.xcom_pull(task_ids='test', key='return_value'),
@@ -80,17 +99,17 @@ def test_json_command_execution(self):
     def test_pickle_command_execution(self):
         configuration.conf.set("core", "enable_xcom_pickling", "True")
         task = SSHOperator(
-                task_id="test",
-                ssh_hook=self.hook,
-                command="echo -n airflow",
-                do_xcom_push=True,
-                dag=self.dag,
+            task_id="test",
+            ssh_hook=self.hook,
+            command="echo -n airflow",
+            do_xcom_push=True,
+            dag=self.dag,
         )
 
         self.assertIsNotNone(task)
 
         ti = TaskInstance(
-                task=task, execution_date=timezone.utcnow())
+            task=task, execution_date=timezone.utcnow())
         ti.run()
         self.assertIsNotNone(ti.duration)
         self.assertEqual(ti.xcom_pull(task_ids='test', key='return_value'), b'airflow')
@@ -131,6 +150,65 @@ def test_no_output_command(self):
         self.assertIsNotNone(ti.duration)
         self.assertEqual(ti.xcom_pull(task_ids='test', key='return_value'), b'')
 
+    def test_arg_checking(self):
+        import os
+        from airflow.exceptions import AirflowException
+        conn_id = "conn_id_for_testing"
+        TIMEOUT = 5
+        os.environ['AIRFLOW_CONN_' + conn_id.upper()] = "ssh://test_id@localhost"
+
+        # Exception should be raised if neither ssh_hook nor ssh_conn_id is provided
+        if six.PY2:
+            self.assertRaisesRegex = self.assertRaisesRegexp
+        with self.assertRaisesRegex(AirflowException,
+                                    "Cannot operate without ssh_hook or ssh_conn_id."):
+            task_0 = SSHOperator(task_id="test", command="echo -n airflow",
+                                 timeout=TIMEOUT, dag=self.dag)
+            task_0.execute(None)
+
+        # if ssh_hook is invalid/not provided, use ssh_conn_id to create SSHHook
+        task_1 = SSHOperator(
+            task_id="test_1",
+            ssh_hook="string_rather_than_SSHHook",  # invalid ssh_hook
+            ssh_conn_id=conn_id,
+            command="echo -n airflow",
+            timeout=TIMEOUT,
+            dag=self.dag
+        )
+        try:
+            task_1.execute(None)
+        except Exception:
+            pass
+        self.assertEqual(task_1.ssh_hook.ssh_conn_id, conn_id)
+
+        task_2 = SSHOperator(
+            task_id="test_2",
+            ssh_conn_id=conn_id,  # no ssh_hook provided
+            command="echo -n airflow",
+            timeout=TIMEOUT,
+            dag=self.dag
+        )
+        try:
+            task_2.execute(None)
+        except Exception:
+            pass
+        self.assertEqual(task_2.ssh_hook.ssh_conn_id, conn_id)
+
+        # if both valid ssh_hook and ssh_conn_id are provided, ignore ssh_conn_id
+        task_3 = SSHOperator(
+            task_id="test_3",
+            ssh_hook=self.hook,
+            ssh_conn_id=conn_id,
+            command="echo -n airflow",
+            timeout=TIMEOUT,
+            dag=self.dag
+        )
+        try:
+            task_3.execute(None)
+        except Exception:
+            pass
+        self.assertEqual(task_3.ssh_hook.ssh_conn_id, self.hook.ssh_conn_id)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/operators/test_vertica_to_mysql.py b/tests/contrib/operators/test_vertica_to_mysql.py
index 615f111f840f4..c76c4d5dccc26 100644
--- a/tests/contrib/operators/test_vertica_to_mysql.py
+++ b/tests/contrib/operators/test_vertica_to_mysql.py
@@ -30,14 +30,14 @@ def mock_get_conn():
     commit_mock = mock.MagicMock(
     )
     cursor_mock = mock.MagicMock(
-        execute     = [],
-        fetchall    = [['1', '2', '3']],
-        description =  ['a', 'b', 'c'],
-        iterate     = [['1', '2', '3']],
+        execute=[],
+        fetchall=[['1', '2', '3']],
+        description=['a', 'b', 'c'],
+        iterate=[['1', '2', '3']],
     )
     conn_mock = mock.MagicMock(
-        commit      = commit_mock,
-        cursor      = cursor_mock,
+        commit=commit_mock,
+        cursor=cursor_mock,
     )
     return conn_mock
 
diff --git a/tests/contrib/operators/test_wasb_delete_blob_operator.py b/tests/contrib/operators/test_wasb_delete_blob_operator.py
new file mode 100644
index 0000000000000..04c3f7efdb9a3
--- /dev/null
+++ b/tests/contrib/operators/test_wasb_delete_blob_operator.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import datetime
+import unittest
+
+from airflow import DAG, configuration
+from airflow.contrib.operators.wasb_delete_blob_operator import WasbDeleteBlobOperator
+from tests.compat import mock
+
+
+class TestWasbDeleteBlobOperator(unittest.TestCase):
+
+    _config = {
+        'container_name': 'container',
+        'blob_name': 'blob',
+    }
+
+    def setUp(self):
+        configuration.load_test_config()
+        args = {
+            'owner': 'airflow',
+            'start_date': datetime.datetime(2017, 1, 1)
+        }
+        self.dag = DAG('test_dag_id', default_args=args)
+
+    def test_init(self):
+        operator = WasbDeleteBlobOperator(
+            task_id='wasb_operator',
+            dag=self.dag,
+            **self._config
+        )
+        self.assertEqual(operator.container_name,
+                         self._config['container_name'])
+        self.assertEqual(operator.blob_name, self._config['blob_name'])
+        self.assertEqual(operator.is_prefix, False)
+        self.assertEqual(operator.ignore_if_missing, False)
+
+        operator = WasbDeleteBlobOperator(
+            task_id='wasb_operator',
+            dag=self.dag,
+            is_prefix=True,
+            ignore_if_missing=True,
+            **self._config
+        )
+        self.assertEqual(operator.is_prefix, True)
+        self.assertEqual(operator.ignore_if_missing, True)
+
+    @mock.patch('airflow.contrib.operators.wasb_delete_blob_operator.WasbHook',
+                autospec=True)
+    def test_execute(self, mock_hook):
+        mock_instance = mock_hook.return_value
+        operator = WasbDeleteBlobOperator(
+            task_id='wasb_operator',
+            dag=self.dag,
+            is_prefix=True,
+            ignore_if_missing=True,
+            **self._config
+        )
+        operator.execute(None)
+        mock_instance.delete_file.assert_called_once_with(
+            'container', 'blob', True, True
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/__init__.py b/tests/contrib/sensors/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/contrib/sensors/__init__.py
+++ b/tests/contrib/sensors/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/sensors/test_athena_sensor.py b/tests/contrib/sensors/test_athena_sensor.py
new file mode 100644
index 0000000000000..48da102fd2b27
--- /dev/null
+++ b/tests/contrib/sensors/test_athena_sensor.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from airflow import configuration, AirflowException
+from airflow.contrib.sensors.aws_athena_sensor import AthenaSensor
+from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook
+from tests.compat import mock
+
+
+class TestAthenaSensor(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+
+        self.sensor = AthenaSensor(task_id='test_athena_sensor',
+                                   query_execution_id='abc',
+                                   sleep_time=5,
+                                   max_retires=1,
+                                   aws_conn_id='aws_default')
+
+    @mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("SUCCEEDED",))
+    def test_poke_success(self, mock_poll_query_status):
+        self.assertTrue(self.sensor.poke(None))
+
+    @mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("RUNNING",))
+    def test_poke_running(self, mock_poll_query_status):
+        self.assertFalse(self.sensor.poke(None))
+
+    @mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("QUEUED",))
+    def test_poke_queued(self, mock_poll_query_status):
+        self.assertFalse(self.sensor.poke(None))
+
+    @mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("FAILED",))
+    def test_poke_failed(self, mock_poll_query_status):
+        with self.assertRaises(AirflowException) as context:
+            self.sensor.poke(None)
+        self.assertIn('Athena sensor failed', str(context.exception))
+
+    @mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("CANCELLED",))
+    def test_poke_cancelled(self, mock_poll_query_status):
+        with self.assertRaises(AirflowException) as context:
+            self.sensor.poke(None)
+        self.assertIn('Athena sensor failed', str(context.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_aws_glue_catalog_partition_sensor.py b/tests/contrib/sensors/test_aws_glue_catalog_partition_sensor.py
new file mode 100644
index 0000000000000..a925f0e29685c
--- /dev/null
+++ b/tests/contrib/sensors/test_aws_glue_catalog_partition_sensor.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.hooks.aws_glue_catalog_hook import AwsGlueCatalogHook
+from airflow.contrib.sensors.aws_glue_catalog_partition_sensor import AwsGlueCatalogPartitionSensor
+from tests.compat import mock
+
+try:
+    from moto import mock_glue
+except ImportError:
+    mock_glue = None
+
+
+@unittest.skipIf(mock_glue is None,
+                 "Skipping test because moto.mock_glue is not available")
+class TestAwsGlueCatalogPartitionSensor(unittest.TestCase):
+
+    task_id = 'test_glue_catalog_partition_sensor'
+
+    def setUp(self):
+        configuration.load_test_config()
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'check_for_partition')
+    def test_poke(self, mock_check_for_partition):
+        mock_check_for_partition.return_value = True
+        op = AwsGlueCatalogPartitionSensor(task_id=self.task_id,
+                                           table_name='tbl')
+        self.assertTrue(op.poke(None))
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'check_for_partition')
+    def test_poke_false(self, mock_check_for_partition):
+        mock_check_for_partition.return_value = False
+        op = AwsGlueCatalogPartitionSensor(task_id=self.task_id,
+                                           table_name='tbl')
+        self.assertFalse(op.poke(None))
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'check_for_partition')
+    def test_poke_default_args(self, mock_check_for_partition):
+        table_name = 'test_glue_catalog_partition_sensor_tbl'
+        op = AwsGlueCatalogPartitionSensor(task_id=self.task_id,
+                                           table_name=table_name)
+        op.poke(None)
+
+        self.assertEqual(op.hook.region_name, None)
+        self.assertEqual(op.hook.aws_conn_id, 'aws_default')
+        mock_check_for_partition.assert_called_once_with('default',
+                                                         table_name,
+                                                         "ds='{{ ds }}'")
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'check_for_partition')
+    def test_poke_nondefault_args(self, mock_check_for_partition):
+        table_name = 'my_table'
+        expression = 'col=val'
+        aws_conn_id = 'my_aws_conn_id'
+        region_name = 'us-west-2'
+        database_name = 'my_db'
+        poke_interval = 2
+        timeout = 3
+        op = AwsGlueCatalogPartitionSensor(task_id=self.task_id,
+                                           table_name=table_name,
+                                           expression=expression,
+                                           aws_conn_id=aws_conn_id,
+                                           region_name=region_name,
+                                           database_name=database_name,
+                                           poke_interval=poke_interval,
+                                           timeout=timeout)
+        op.poke(None)
+
+        self.assertEqual(op.hook.region_name, region_name)
+        self.assertEqual(op.hook.aws_conn_id, aws_conn_id)
+        self.assertEqual(op.poke_interval, poke_interval)
+        self.assertEqual(op.timeout, timeout)
+        mock_check_for_partition.assert_called_once_with(database_name,
+                                                         table_name,
+                                                         expression)
+
+    @mock_glue
+    @mock.patch.object(AwsGlueCatalogHook, 'check_for_partition')
+    def test_dot_notation(self, mock_check_for_partition):
+        db_table = 'my_db.my_tbl'
+        op = AwsGlueCatalogPartitionSensor(task_id=self.task_id,
+                                           table_name=db_table)
+        op.poke(None)
+
+        mock_check_for_partition.assert_called_once_with('my_db',
+                                                         'my_tbl',
+                                                         "ds='{{ ds }}'")
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_aws_redshift_cluster_sensor.py b/tests/contrib/sensors/test_aws_redshift_cluster_sensor.py
index 5bee24aeda548..6753cefa428a3 100644
--- a/tests/contrib/sensors/test_aws_redshift_cluster_sensor.py
+++ b/tests/contrib/sensors/test_aws_redshift_cluster_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -31,10 +31,12 @@
     mock_redshift = None
 
 
-@mock_redshift
 class TestAwsRedshiftClusterSensor(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
+
+    @staticmethod
+    def _create_cluster():
         client = boto3.client('redshift', region_name='us-east-1')
         client.create_cluster(
             ClusterIdentifier='test_cluster',
@@ -46,7 +48,9 @@ def setUp(self):
             raise ValueError('AWS not properly mocked')
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_poke(self):
+        self._create_cluster()
         op = AwsRedshiftClusterSensor(task_id='test_cluster_sensor',
                                       poke_interval=1,
                                       timeout=5,
@@ -56,7 +60,9 @@ def test_poke(self):
         self.assertTrue(op.poke(None))
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_poke_false(self):
+        self._create_cluster()
         op = AwsRedshiftClusterSensor(task_id='test_cluster_sensor',
                                       poke_interval=1,
                                       timeout=5,
@@ -67,7 +73,9 @@ def test_poke_false(self):
         self.assertFalse(op.poke(None))
 
     @unittest.skipIf(mock_redshift is None, 'mock_redshift package not present')
+    @mock_redshift
     def test_poke_cluster_not_found(self):
+        self._create_cluster()
         op = AwsRedshiftClusterSensor(task_id='test_cluster_sensor',
                                       poke_interval=1,
                                       timeout=5,
diff --git a/tests/contrib/sensors/test_bash_sensor.py b/tests/contrib/sensors/test_bash_sensor.py
index 346be5180eaa9..e678425997790 100644
--- a/tests/contrib/sensors/test_bash_sensor.py
+++ b/tests/contrib/sensors/test_bash_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/sensors/test_cassandra_sensor.py b/tests/contrib/sensors/test_cassandra_sensor.py
index 0f0e7f5eb3b56..c07bc0be2a44d 100644
--- a/tests/contrib/sensors/test_cassandra_sensor.py
+++ b/tests/contrib/sensors/test_cassandra_sensor.py
@@ -24,7 +24,8 @@
 
 from airflow import DAG
 from airflow import configuration
-from airflow.contrib.sensors.cassandra_sensor import CassandraRecordSensor
+from airflow.contrib.sensors.cassandra_record_sensor import CassandraRecordSensor
+from airflow.contrib.sensors.cassandra_table_sensor import CassandraTableSensor
 from airflow.utils import timezone
 
 
@@ -54,5 +55,27 @@ def test_poke(self, mock_record_exists):
         mock_record_exists.assert_called_once_with('t', {'foo': 'bar'})
 
 
+class TestCassandraTableSensor(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG('test_dag_id', default_args=args)
+        self.sensor = CassandraTableSensor(
+            task_id='test_task',
+            cassandra_conn_id='cassandra_default',
+            dag=self.dag,
+            table='t',
+        )
+
+    @patch("airflow.contrib.hooks.cassandra_hook.CassandraHook.table_exists")
+    def test_poke(self, mock_table_exists):
+        self.sensor.poke(None)
+        mock_table_exists.assert_called_once_with('t')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/sensors/test_datadog_sensor.py b/tests/contrib/sensors/test_datadog_sensor.py
index 2f93bb1d4500e..2c59d665d2e69 100644
--- a/tests/contrib/sensors/test_datadog_sensor.py
+++ b/tests/contrib/sensors/test_datadog_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,13 +19,13 @@
 
 import json
 import unittest
-
-from mock import patch
+from typing import List
 
 from airflow import configuration
-from airflow import models
 from airflow.contrib.sensors.datadog_sensor import DatadogSensor
+from airflow.models.connection import Connection
 from airflow.utils import db
+from tests.compat import patch
 
 at_least_one_event = [{'alert_type': 'info',
                        'comments': [],
@@ -56,7 +56,7 @@
                        'title': 'Something big happened!',
                        'url': '/event/jump_to?event_id=2603387619536318141'}]
 
-zero_events = []
+zero_events = []  # type: List
 
 
 class TestDatadogSensor(unittest.TestCase):
@@ -64,7 +64,7 @@ class TestDatadogSensor(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='datadog_default', conn_type='datadog',
                 login='login', password='password',
                 extra=json.dumps({'api_key': 'api_key', 'app_key': 'app_key'})
diff --git a/tests/contrib/sensors/test_emr_base_sensor.py b/tests/contrib/sensors/test_emr_base_sensor.py
index 8d00db713fb2d..6dd6fc2b87917 100644
--- a/tests/contrib/sensors/test_emr_base_sensor.py
+++ b/tests/contrib/sensors/test_emr_base_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -33,13 +33,15 @@ class EmrBaseSensorSubclass(EmrBaseSensor):
             NON_TERMINAL_STATES = ['PENDING', 'RUNNING', 'CONTINUE']
             FAILED_STATE = ['FAILED']
 
-            def get_emr_response(self):
+            @staticmethod
+            def get_emr_response():
                 return {
                     'SomeKey': {'State': 'COMPLETED'},
                     'ResponseMetadata': {'HTTPStatusCode': 200}
                 }
 
-            def state_from_response(self, response):
+            @staticmethod
+            def state_from_response(response):
                 return response['SomeKey']['State']
 
         operator = EmrBaseSensorSubclass(
@@ -56,13 +58,15 @@ class EmrBaseSensorSubclass(EmrBaseSensor):
             NON_TERMINAL_STATES = ['PENDING', 'RUNNING', 'CONTINUE']
             FAILED_STATE = ['FAILED']
 
-            def get_emr_response(self):
+            @staticmethod
+            def get_emr_response():
                 return {
                     'SomeKey': {'State': 'PENDING'},
                     'ResponseMetadata': {'HTTPStatusCode': 200}
                 }
 
-            def state_from_response(self, response):
+            @staticmethod
+            def state_from_response(response):
                 return response['SomeKey']['State']
 
         operator = EmrBaseSensorSubclass(
@@ -79,13 +83,15 @@ class EmrBaseSensorSubclass(EmrBaseSensor):
             NON_TERMINAL_STATES = ['PENDING', 'RUNNING', 'CONTINUE']
             FAILED_STATE = ['FAILED']
 
-            def get_emr_response(self):
+            @staticmethod
+            def get_emr_response():
                 return {
                     'SomeKey': {'State': 'COMPLETED'},
                     'ResponseMetadata': {'HTTPStatusCode': 400}
                 }
 
-            def state_from_response(self, response):
+            @staticmethod
+            def state_from_response(response):
                 return response['SomeKey']['State']
 
         operator = EmrBaseSensorSubclass(
@@ -102,13 +108,15 @@ class EmrBaseSensorSubclass(EmrBaseSensor):
             NON_TERMINAL_STATES = ['PENDING', 'RUNNING', 'CONTINUE']
             FAILED_STATE = ['FAILED']
 
-            def get_emr_response(self):
+            @staticmethod
+            def get_emr_response():
                 return {
                     'SomeKey': {'State': 'FAILED'},
                     'ResponseMetadata': {'HTTPStatusCode': 200}
                 }
 
-            def state_from_response(self, response):
+            @staticmethod
+            def state_from_response(response):
                 return response['SomeKey']['State']
 
         operator = EmrBaseSensorSubclass(
diff --git a/tests/contrib/sensors/test_emr_job_flow_sensor.py b/tests/contrib/sensors/test_emr_job_flow_sensor.py
index a66e52ffdf5f1..5b33cb5bb418d 100644
--- a/tests/contrib/sensors/test_emr_job_flow_sensor.py
+++ b/tests/contrib/sensors/test_emr_job_flow_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -42,7 +42,8 @@
         'Status': {
             'State': 'STARTING',
             'StateChangeReason': {},
-            'Timeline': {'CreationDateTime': datetime.datetime(2016, 6, 27, 21, 5, 2, 348000, tzinfo=tzlocal())}
+            'Timeline': {
+                'CreationDateTime': datetime.datetime(2016, 6, 27, 21, 5, 2, 348000, tzinfo=tzlocal())}
         },
         'Tags': [
             {'Key': 'app', 'Value': 'analytics'},
@@ -74,7 +75,8 @@
         'Status': {
             'State': 'TERMINATED',
             'StateChangeReason': {},
-            'Timeline': {'CreationDateTime': datetime.datetime(2016, 6, 27, 21, 5, 2, 348000, tzinfo=tzlocal())}
+            'Timeline': {
+                'CreationDateTime': datetime.datetime(2016, 6, 27, 21, 5, 2, 348000, tzinfo=tzlocal())}
         },
         'Tags': [
             {'Key': 'app', 'Value': 'analytics'},
@@ -107,10 +109,8 @@ def setUp(self):
         # Mock out the emr_client creator
         self.boto3_session_mock = MagicMock(return_value=mock_emr_session)
 
-
     def test_execute_calls_with_the_job_flow_id_until_it_reaches_a_terminal_state(self):
         with patch('boto3.session.Session', self.boto3_session_mock):
-
             operator = EmrJobFlowSensor(
                 task_id='test_task',
                 poke_interval=2,
diff --git a/tests/contrib/sensors/test_emr_step_sensor.py b/tests/contrib/sensors/test_emr_step_sensor.py
index 7f72c73befd03..08cb8d1aeac52 100644
--- a/tests/contrib/sensors/test_emr_step_sensor.py
+++ b/tests/contrib/sensors/test_emr_step_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -83,6 +83,35 @@
     }
 }
 
+DESCRIBE_JOB_STEP_INTERRUPTED_RETURN = {
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+        'RequestId': '8dee8db2-3719-11e6-9e20-35b2f861a2a6'
+    },
+    'Step': {
+        'ActionOnFailure': 'CONTINUE',
+        'Config': {
+            'Args': [
+                '/usr/lib/spark/bin/run-example',
+                'SparkPi',
+                '10'
+            ],
+            'Jar': 'command-runner.jar',
+            'Properties': {}
+        },
+        'Id': 's-VK57YR1Z9Z5N',
+        'Name': 'calculate_pi',
+        'Status': {
+            'State': 'INTERRUPTED',
+            'StateChangeReason': {},
+            'Timeline': {
+                'CreationDateTime': datetime(2016, 6, 20, 19, 0, 18, tzinfo=tzlocal()),
+                'StartDateTime': datetime(2016, 6, 20, 19, 2, 34, tzinfo=tzlocal())
+            }
+        }
+    }
+}
+
 DESCRIBE_JOB_STEP_COMPLETED_RETURN = {
     'ResponseMetadata': {
         'HTTPStatusCode': 200,
@@ -158,6 +187,17 @@ def test_step_cancelled(self):
         with patch('boto3.session.Session', self.boto3_session_mock):
             self.assertRaises(AirflowException, self.sensor.execute, None)
 
+    def test_step_interrupted(self):
+        self.emr_client_mock.describe_step.side_effect = [
+            DESCRIBE_JOB_STEP_RUNNING_RETURN,
+            DESCRIBE_JOB_STEP_INTERRUPTED_RETURN
+        ]
+
+        self.boto3_client_mock = MagicMock(return_value=self.emr_client_mock)
+
+        with patch('boto3.session.Session', self.boto3_session_mock):
+            self.assertRaises(AirflowException, self.sensor.execute, None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/sensors/test_file_sensor.py b/tests/contrib/sensors/test_file_sensor.py
index d78400e317fcd..0f17e2de5ae0e 100644
--- a/tests/contrib/sensors/test_file_sensor.py
+++ b/tests/contrib/sensors/test_file_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/sensors/test_ftp_sensor.py b/tests/contrib/sensors/test_ftp_sensor.py
index cf0fdb4918503..599e3c73d28b0 100644
--- a/tests/contrib/sensors/test_ftp_sensor.py
+++ b/tests/contrib/sensors/test_ftp_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -49,8 +49,13 @@ def test_poke(self):
                        task_id="test_task")
 
         self.hook_mock.get_mod_time.side_effect = \
-            [error_perm("550: Can't check for file existence"), None]
+            [error_perm("550: Can't check for file existence"),
+                error_perm("550: Directory or file does not exist"),
+                error_perm("550 - Directory or file does not exist"),
+                None]
 
+        self.assertFalse(op.poke(None))
+        self.assertFalse(op.poke(None))
         self.assertFalse(op.poke(None))
         self.assertTrue(op.poke(None))
 
@@ -66,6 +71,28 @@ def test_poke_fails_due_error(self):
 
         self.assertTrue("530" in str(context.exception))
 
+    def test_poke_fail_on_transient_error(self):
+        op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp",
+                       task_id="test_task")
+
+        self.hook_mock.get_mod_time.side_effect = \
+            error_perm("434: Host unavailable")
+
+        with self.assertRaises(error_perm) as context:
+            op.execute(None)
+
+        self.assertTrue("434" in str(context.exception))
+
+    def test_poke_ignore_transient_error(self):
+        op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp",
+                       task_id="test_task", fail_on_transient_errors=False)
+
+        self.hook_mock.get_mod_time.side_effect = \
+            [error_perm("434: Host unavailable"), None]
+
+        self.assertFalse(op.poke(None))
+        self.assertTrue(op.poke(None))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/contrib/sensors/test_gcp_transfer_sensor.py b/tests/contrib/sensors/test_gcp_transfer_sensor.py
new file mode 100644
index 0000000000000..d15aac30e87d2
--- /dev/null
+++ b/tests/contrib/sensors/test_gcp_transfer_sensor.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from parameterized import parameterized
+
+from airflow.contrib.hooks.gcp_transfer_hook import GcpTransferOperationStatus
+from airflow.contrib.sensors.gcp_transfer_sensor import GCPTransferServiceWaitForJobStatusSensor
+from tests.compat import mock
+
+
+class TestGcpStorageTransferOperationWaitForJobStatusSensor(unittest.TestCase):
+    @mock.patch('airflow.contrib.sensors.gcp_transfer_sensor.GCPTransferServiceHook')
+    def test_wait_for_status_success(self, mock_tool):
+        operations = [{'metadata': {'status': GcpTransferOperationStatus.SUCCESS}}]
+        mock_tool.return_value.list_transfer_operations.return_value = operations
+        mock_tool.operations_contain_expected_statuses.return_value = True
+
+        op = GCPTransferServiceWaitForJobStatusSensor(
+            task_id='task-id',
+            operation_name='operation-name',
+            job_name='job-name',
+            project_id='project-id',
+            expected_statuses=GcpTransferOperationStatus.SUCCESS,
+        )
+
+        context = {'ti': (mock.Mock(**{'xcom_push.return_value': None}))}
+        result = op.poke(context)
+
+        mock_tool.return_value.list_transfer_operations.assert_called_with(
+            filter={'project_id': 'project-id', 'job_names': ['job-name']}
+        )
+        mock_tool.operations_contain_expected_statuses.assert_called_with(
+            operations=operations, expected_statuses={GcpTransferOperationStatus.SUCCESS}
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.sensors.gcp_transfer_sensor.GCPTransferServiceHook')
+    def test_wait_for_status_success_default_expected_status(self, mock_tool):
+
+        op = GCPTransferServiceWaitForJobStatusSensor(
+            task_id='task-id',
+            operation_name='operation-name',
+            job_name='job-name',
+            project_id='project-id',
+            expected_statuses=GcpTransferOperationStatus.SUCCESS,
+        )
+
+        context = {'ti': (mock.Mock(**{'xcom_push.return_value': None}))}
+
+        result = op.poke(context)
+
+        mock_tool.operations_contain_expected_statuses.assert_called_with(
+            operations=mock.ANY, expected_statuses={GcpTransferOperationStatus.SUCCESS}
+        )
+        self.assertTrue(result)
+
+    @mock.patch('airflow.contrib.sensors.gcp_transfer_sensor.GCPTransferServiceHook')
+    def test_wait_for_status_after_retry(self, mock_tool):
+        operations_set = [
+            [{'metadata': {'status': GcpTransferOperationStatus.SUCCESS}}],
+            [{'metadata': {'status': GcpTransferOperationStatus.SUCCESS}}],
+        ]
+
+        mock_tool.return_value.list_transfer_operations.side_effect = operations_set
+        mock_tool.operations_contain_expected_statuses.side_effect = [False, True]
+
+        op = GCPTransferServiceWaitForJobStatusSensor(
+            task_id='task-id',
+            operation_name='operation-name',
+            job_name='job-name',
+            project_id='project-id',
+            expected_statuses=GcpTransferOperationStatus.SUCCESS,
+        )
+
+        context = {'ti': (mock.Mock(**{'xcom_push.return_value': None}))}
+
+        result = op.poke(context)
+        self.assertFalse(result)
+
+        mock_tool.operations_contain_expected_statuses.assert_called_with(
+            operations=operations_set[0], expected_statuses={GcpTransferOperationStatus.SUCCESS}
+        )
+        mock_tool.operations_contain_expected_statuses.reset_mock()
+
+        result = op.poke(context)
+        self.assertTrue(result)
+
+        mock_tool.operations_contain_expected_statuses.assert_called_with(
+            operations=operations_set[1], expected_statuses={GcpTransferOperationStatus.SUCCESS}
+        )
+
+    @parameterized.expand(
+        [
+            (GcpTransferOperationStatus.SUCCESS, {GcpTransferOperationStatus.SUCCESS}),
+            ({GcpTransferOperationStatus.SUCCESS}, {GcpTransferOperationStatus.SUCCESS}),
+            (
+                {GcpTransferOperationStatus.SUCCESS, GcpTransferOperationStatus.SUCCESS},
+                {GcpTransferOperationStatus.SUCCESS, GcpTransferOperationStatus.SUCCESS},
+            ),
+        ]
+    )
+    @mock.patch('airflow.contrib.sensors.gcp_transfer_sensor.GCPTransferServiceHook')
+    def test_wait_for_status_normalize_status(self, expected_status, received_status, mock_tool):
+        operations = [{'metadata': {'status': GcpTransferOperationStatus.SUCCESS}}]
+
+        mock_tool.return_value.list_transfer_operations.return_value = operations
+        mock_tool.operations_contain_expected_statuses.side_effect = [False, True]
+
+        op = GCPTransferServiceWaitForJobStatusSensor(
+            task_id='task-id',
+            operation_name='operation-name',
+            job_name='job-name',
+            project_id='project-id',
+            expected_statuses=expected_status,
+        )
+
+        context = {'ti': (mock.Mock(**{'xcom_push.return_value': None}))}
+
+        result = op.poke(context)
+        self.assertFalse(result)
+
+        mock_tool.operations_contain_expected_statuses.assert_called_with(
+            operations=operations, expected_statuses=received_status
+        )
diff --git a/tests/contrib/sensors/test_hdfs_sensor.py b/tests/contrib/sensors/test_hdfs_sensor.py
index 26ce5570a0e44..b03b738686ed8 100644
--- a/tests/contrib/sensors/test_hdfs_sensor.py
+++ b/tests/contrib/sensors/test_hdfs_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -236,7 +236,7 @@ def test_should_match_regex_but_copyingext(self):
         self.log.debug('#' * 10)
         self.log.debug('Running %s', self._testMethodName)
         self.log.debug('#' * 10)
-        compiled_regex = re.compile("copying_file_\d+.txt")
+        compiled_regex = re.compile(r"copying_file_\d+.txt")
         task = HdfsSensorRegex(task_id='Should_match_the_regex_but_filesize',
                                filepath='/datadirectory/regex_dir',
                                regex=compiled_regex,
diff --git a/tests/contrib/sensors/test_imap_attachment_sensor.py b/tests/contrib/sensors/test_imap_attachment_sensor.py
new file mode 100644
index 0000000000000..66e3cfc7883b9
--- /dev/null
+++ b/tests/contrib/sensors/test_imap_attachment_sensor.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from mock import patch, Mock
+
+from airflow import configuration
+from airflow.contrib.sensors.imap_attachment_sensor import ImapAttachmentSensor
+from airflow.models.connection import Connection
+from airflow.utils import db
+
+imap_hook_string = 'airflow.contrib.sensors.imap_attachment_sensor.ImapHook'
+
+
+class TestImapAttachmentSensor(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='imap_test',
+                host='base_url',
+                login='user',
+                password='password'
+            )
+        )
+
+    @patch(imap_hook_string)
+    def test_poke_with_attachment_found(self, mock_imap_hook):
+        mock_imap_hook.return_value.__enter__ = Mock(return_value=mock_imap_hook)
+        mock_imap_hook.has_mail_attachment.return_value = True
+
+        imap_attachment_sensor = ImapAttachmentSensor(
+            conn_id='imap_test',
+            attachment_name='test_attachment',
+            task_id='check_for_attachment_on_mail_server_test',
+            dag=None
+        )
+
+        self.assertTrue(imap_attachment_sensor.poke(context={}))
+        mock_imap_hook.has_mail_attachment.assert_called_once_with(
+            name='test_attachment',
+            mail_folder='INBOX',
+            check_regex=False
+        )
+
+    @patch(imap_hook_string)
+    def test_poke_with_attachment_not_found(self, mock_imap_hook):
+        mock_imap_hook.return_value.__enter__ = Mock(return_value=mock_imap_hook)
+        mock_imap_hook.has_mail_attachment.return_value = False
+
+        imap_attachment_sensor = ImapAttachmentSensor(
+            conn_id='imap_test',
+            attachment_name='test_attachment',
+            task_id='check_for_attachment_on_mail_server_test',
+            dag=None
+        )
+
+        self.assertFalse(imap_attachment_sensor.poke(context={}))
+        mock_imap_hook.has_mail_attachment.assert_called_once_with(
+            name='test_attachment',
+            mail_folder='INBOX',
+            check_regex=False
+        )
+
+    @patch(imap_hook_string)
+    def test_poke_with_check_regex_true(self, mock_imap_hook):
+        mock_imap_hook.return_value.__enter__ = Mock(return_value=mock_imap_hook)
+        mock_imap_hook.has_mail_attachment.return_value = True
+
+        imap_attachment_sensor = ImapAttachmentSensor(
+            conn_id='imap_test',
+            attachment_name='.*_test_attachment',
+            task_id='check_for_attachment_on_mail_server_test',
+            check_regex=True,
+            dag=None
+        )
+
+        self.assertTrue(imap_attachment_sensor.poke(context={}))
+        mock_imap_hook.has_mail_attachment.assert_called_once_with(
+            name='.*_test_attachment',
+            mail_folder='INBOX',
+            check_regex=True
+        )
+
+    @patch(imap_hook_string)
+    def test_poke_with_different_mail_folder(self, mock_imap_hook):
+        mock_imap_hook.return_value.__enter__ = Mock(return_value=mock_imap_hook)
+        mock_imap_hook.has_mail_attachment.return_value = True
+
+        imap_attachment_sensor = ImapAttachmentSensor(
+            conn_id='imap_test',
+            attachment_name='test_attachment',
+            task_id='check_for_attachment_on_mail_server_test',
+            mail_folder='test',
+            dag=None
+        )
+
+        self.assertTrue(imap_attachment_sensor.poke(context={}))
+        mock_imap_hook.has_mail_attachment.assert_called_once_with(
+            name='test_attachment',
+            mail_folder='test',
+            check_regex=False
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_jira_sensor_test.py b/tests/contrib/sensors/test_jira_sensor_test.py
index 06a7713f35d48..edbcb4b57b2dc 100644
--- a/tests/contrib/sensors/test_jira_sensor_test.py
+++ b/tests/contrib/sensors/test_jira_sensor_test.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,8 +24,8 @@
 from mock import patch
 
 from airflow import DAG, configuration
-from airflow import models
 from airflow.contrib.sensors.jira_sensor import JiraTicketSensor
+from airflow.models.connection import Connection
 from airflow.utils import db, timezone
 
 DEFAULT_DATE = timezone.datetime(2017, 1, 1)
@@ -57,7 +57,7 @@ def setUp(self):
         dag = DAG('test_dag_id', default_args=args)
         self.dag = dag
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='jira_default', conn_type='jira',
                 host='https://localhost/jira/', port=443,
                 extra='{"verify": "False", "project": "AIRFLOW"}'))
@@ -67,13 +67,13 @@ def setUp(self):
     def test_issue_label_set(self, jira_mock):
         jira_mock.return_value.issue.return_value = minimal_test_ticket
 
-        ticket_label_sensor = JiraTicketSensor(task_id='search-ticket-test',
-                                               ticket_id='TEST-1226',
-                                               field_checker_func=
-                                               TestJiraSensor.field_checker_func,
-                                               timeout=518400,
-                                               poke_interval=10,
-                                               dag=self.dag)
+        ticket_label_sensor = JiraTicketSensor(
+            task_id='search-ticket-test',
+            ticket_id='TEST-1226',
+            field_checker_func=TestJiraSensor.field_checker_func,
+            timeout=518400,
+            poke_interval=10,
+            dag=self.dag)
 
         ticket_label_sensor.run(start_date=DEFAULT_DATE,
                                 end_date=DEFAULT_DATE, ignore_ti_state=True)
diff --git a/tests/contrib/sensors/test_mongo_sensor.py b/tests/contrib/sensors/test_mongo_sensor.py
new file mode 100644
index 0000000000000..e2110002ba456
--- /dev/null
+++ b/tests/contrib/sensors/test_mongo_sensor.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import unittest
+
+from airflow import DAG
+from airflow import configuration
+from airflow.contrib.hooks.mongo_hook import MongoHook
+from airflow.contrib.sensors.mongo_sensor import MongoSensor
+from airflow.models.connection import Connection
+from airflow.utils import db, timezone
+
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+
+class TestMongoSensor(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        db.merge_conn(
+            Connection(
+                conn_id='mongo_test', conn_type='mongo',
+                host='mongo', port='27017', schema='test'))
+
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+        self.dag = DAG('test_dag_id', default_args=args)
+
+        hook = MongoHook('mongo_test')
+        hook.insert_one('foo', {'bar': 'baz'})
+
+        self.sensor = MongoSensor(
+            task_id='test_task',
+            mongo_conn_id='mongo_test',
+            dag=self.dag,
+            collection='foo',
+            query={'bar': 'baz'}
+        )
+
+    def test_poke(self):
+        self.assertTrue(self.sensor.poke(None))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_pubsub_sensor.py b/tests/contrib/sensors/test_pubsub_sensor.py
index f1f8fade62c7b..317de92dcaec2 100644
--- a/tests/contrib/sensors/test_pubsub_sensor.py
+++ b/tests/contrib/sensors/test_pubsub_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,14 +25,7 @@
 
 from airflow.contrib.sensors.pubsub_sensor import PubSubPullSensor
 from airflow.exceptions import AirflowSensorTimeout
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 TASK_ID = 'test-task-id'
 TEST_PROJECT = 'test-project'
@@ -66,7 +59,7 @@ def test_poke_no_messages(self, mock_hook):
         operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
                                     subscription=TEST_SUBSCRIPTION)
         mock_hook.return_value.pull.return_value = []
-        self.assertEquals([], operator.poke(None))
+        self.assertEqual([], operator.poke(None))
 
     @mock.patch('airflow.contrib.sensors.pubsub_sensor.PubSubHook')
     def test_poke_with_ack_messages(self, mock_hook):
@@ -75,7 +68,7 @@ def test_poke_with_ack_messages(self, mock_hook):
                                     ack_messages=True)
         generated_messages = self._generate_messages(5)
         mock_hook.return_value.pull.return_value = generated_messages
-        self.assertEquals(generated_messages, operator.poke(None))
+        self.assertEqual(generated_messages, operator.poke(None))
         mock_hook.return_value.acknowledge.assert_called_with(
             TEST_PROJECT, TEST_SUBSCRIPTION, ['1', '2', '3', '4', '5']
         )
@@ -90,7 +83,7 @@ def test_execute(self, mock_hook):
         response = operator.execute(None)
         mock_hook.return_value.pull.assert_called_with(
             TEST_PROJECT, TEST_SUBSCRIPTION, 5, False)
-        self.assertEquals(response, generated_messages)
+        self.assertEqual(response, generated_messages)
 
     @mock.patch('airflow.contrib.sensors.pubsub_sensor.PubSubHook')
     def test_execute_timeout(self, mock_hook):
diff --git a/tests/contrib/sensors/test_python_sensor.py b/tests/contrib/sensors/test_python_sensor.py
new file mode 100644
index 0000000000000..6c852e748a8d1
--- /dev/null
+++ b/tests/contrib/sensors/test_python_sensor.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import unittest
+
+from airflow import DAG, configuration
+from airflow.contrib.sensors.python_sensor import PythonSensor
+from airflow.exceptions import AirflowSensorTimeout
+from airflow.models import DagBag
+from airflow.utils.timezone import datetime
+
+
+DEFAULT_DATE = datetime(2015, 1, 1)
+TEST_DAG_ID = 'python_sensor_dag'
+DEV_NULL = '/dev/null'
+
+
+class PythonSensorTests(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.dagbag = DagBag(
+            dag_folder=DEV_NULL,
+            include_examples=True
+        )
+        self.args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+        dag = DAG(TEST_DAG_ID, default_args=self.args)
+        self.dag = dag
+
+    def test_python_sensor_true(self):
+        t = PythonSensor(
+            task_id='python_sensor_check_true',
+            python_callable=lambda: True,
+            dag=self.dag)
+        t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+    def test_python_sensor_false(self):
+        t = PythonSensor(
+            task_id='python_sensor_check_false',
+            timeout=1,
+            python_callable=lambda: False,
+            dag=self.dag)
+        with self.assertRaises(AirflowSensorTimeout):
+            t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+    def test_python_sensor_raise(self):
+        t = PythonSensor(
+            task_id='python_sensor_check_raise',
+            python_callable=lambda: 1 / 0,
+            dag=self.dag)
+        with self.assertRaises(ZeroDivisionError):
+            t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
diff --git a/tests/contrib/sensors/test_qubole_sensor.py b/tests/contrib/sensors/test_qubole_sensor.py
index 1f28c3ddcd87c..bc9ad8f742f5e 100644
--- a/tests/contrib/sensors/test_qubole_sensor.py
+++ b/tests/contrib/sensors/test_qubole_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,7 +25,8 @@
 
 from airflow.contrib.sensors.qubole_sensor import QuboleFileSensor, QubolePartitionSensor
 from airflow.exceptions import AirflowException
-from airflow.models import DAG, Connection
+from airflow.models import DAG
+from airflow.models.connection import Connection
 from airflow.utils import db
 
 DAG_ID = "qubole_test_dag"
diff --git a/tests/contrib/sensors/test_redis_pub_sub_sensor.py b/tests/contrib/sensors/test_redis_pub_sub_sensor.py
new file mode 100644
index 0000000000000..42b47b6f57996
--- /dev/null
+++ b/tests/contrib/sensors/test_redis_pub_sub_sensor.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import unittest
+from airflow import DAG, configuration
+from airflow.contrib.sensors.redis_pub_sub_sensor import RedisPubSubSensor
+from airflow.utils import timezone
+from airflow.contrib.hooks.redis_hook import RedisHook
+from mock import patch, call, MagicMock
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+
+class TestRedisPubSubSensor(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+
+        args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+
+        self.dag = DAG('test_dag_id', default_args=args)
+
+        self.mock_context = MagicMock()
+
+    @patch('airflow.contrib.hooks.redis_hook.RedisHook.get_conn')
+    def test_poke_mock_true(self, mock_redis_conn):
+        sensor = RedisPubSubSensor(
+            task_id='test_task',
+            dag=self.dag,
+            channels='test',
+            redis_conn_id='redis_default'
+        )
+
+        self.mock_redis_conn = mock_redis_conn
+        self.mock_redis_conn().pubsub().get_message.return_value = \
+            {'type': 'message', 'channel': b'test', 'data': b'd1'}
+
+        result = sensor.poke(self.mock_context)
+        self.assertTrue(result)
+
+        context_calls = [call.xcom_push(key='message',
+                                        value={'type': 'message', 'channel': b'test', 'data': b'd1'})]
+
+        self.assertTrue(self.mock_context['ti'].method_calls == context_calls, "context call  should be same")
+
+    @patch('airflow.contrib.hooks.redis_hook.RedisHook.get_conn')
+    def test_poke_mock_false(self, mock_redis_conn):
+        sensor = RedisPubSubSensor(
+            task_id='test_task',
+            dag=self.dag,
+            channels='test',
+            redis_conn_id='redis_default'
+        )
+        self.mock_redis_conn = mock_redis_conn
+        self.mock_redis_conn().pubsub().get_message.return_value = \
+            {'type': 'subscribe', 'channel': b'test', 'data': b'd1'}
+
+        result = sensor.poke(self.mock_context)
+        self.assertFalse(result)
+
+        context_calls = []
+        self.assertTrue(self.mock_context['ti'].method_calls == context_calls, "context calls should be same")
+
+    def test_poke_true(self):
+        sensor = RedisPubSubSensor(
+            task_id='test_task',
+            dag=self.dag,
+            channels='test',
+            redis_conn_id='redis_default'
+        )
+
+        hook = RedisHook(redis_conn_id='redis_default')
+        redis = hook.get_conn()
+        redis.publish('test', 'message')
+
+        result = sensor.poke(self.mock_context)
+        self.assertFalse(result)
+        result = sensor.poke(self.mock_context)
+        self.assertTrue(result)
+        context_calls = [
+            call.xcom_push(
+                key='message',
+                value={'type': 'message', 'pattern': None, 'channel': b'test', 'data': b'message'})]
+        self.assertTrue(self.mock_context['ti'].method_calls == context_calls, "context calls should be same")
+        result = sensor.poke(self.mock_context)
+        self.assertFalse(result)
+
+    def test_poke_false(self):
+        sensor = RedisPubSubSensor(
+            task_id='test_task',
+            dag=self.dag,
+            channels='test',
+            redis_conn_id='redis_default'
+        )
+
+        result = sensor.poke(self.mock_context)
+        self.assertFalse(result)
+        self.assertTrue(self.mock_context['ti'].method_calls == [], "context calls should be same")
+        result = sensor.poke(self.mock_context)
+        self.assertFalse(result)
+        self.assertTrue(self.mock_context['ti'].method_calls == [], "context calls should be same")
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_redis_sensor.py b/tests/contrib/sensors/test_redis_sensor.py
index b931b49e9bd0c..ec2868c6f7158 100644
--- a/tests/contrib/sensors/test_redis_sensor.py
+++ b/tests/contrib/sensors/test_redis_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,11 +19,9 @@
 
 
 import unittest
-
-from mock import patch
-
 from airflow import DAG
 from airflow import configuration
+from airflow.contrib.hooks.redis_hook import RedisHook
 from airflow.contrib.sensors.redis_key_sensor import RedisKeySensor
 from airflow.utils import timezone
 
@@ -47,22 +45,13 @@ def setUp(self):
             key='test_key'
         )
 
-    @patch("airflow.contrib.hooks.redis_hook.RedisHook.key_exists")
-    def test_poke(self, key_exists):
-        key_exists.return_value = True
-        self.assertTrue(self.sensor.poke(None))
-
-        key_exists.return_value = False
-        self.assertFalse(self.sensor.poke(None))
-
-    @patch("airflow.contrib.hooks.redis_hook.StrictRedis.exists")
-    def test_existing_key_called(self, redis_client_exists):
-        self.sensor.run(
-            start_date=DEFAULT_DATE,
-            end_date=DEFAULT_DATE, ignore_ti_state=True
-        )
-
-        self.assertTrue(redis_client_exists.called_with('test_key'))
+    def test_poke(self):
+        hook = RedisHook(redis_conn_id='redis_default')
+        redis = hook.get_conn()
+        redis.set('test_key', 'test_value')
+        self.assertTrue(self.sensor.poke(None), "Key exists on first call.")
+        redis.delete('test_key')
+        self.assertFalse(self.sensor.poke(None), "Key does NOT exists on second call.")
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/sensors/test_sagemaker_base_sensor.py b/tests/contrib/sensors/test_sagemaker_base_sensor.py
new file mode 100644
index 0000000000000..5870544838033
--- /dev/null
+++ b/tests/contrib/sensors/test_sagemaker_base_sensor.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.sensors.sagemaker_base_sensor import SageMakerBaseSensor
+from airflow.exceptions import AirflowException
+
+
+class TestSagemakerBaseSensor(unittest.TestCase):
+    def setUp(self):
+        configuration.load_test_config()
+
+    def test_execute(self):
+        class SageMakerBaseSensorSubclass(SageMakerBaseSensor):
+            def non_terminal_states(self):
+                return ['PENDING', 'RUNNING', 'CONTINUE']
+
+            def failed_states(self):
+                return ['FAILED']
+
+            def get_sagemaker_response(self):
+                return {
+                    'SomeKey': {'State': 'COMPLETED'},
+                    'ResponseMetadata': {'HTTPStatusCode': 200}
+                }
+
+            def state_from_response(self, response):
+                return response['SomeKey']['State']
+
+        sensor = SageMakerBaseSensorSubclass(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test'
+        )
+
+        sensor.execute(None)
+
+    def test_poke_with_unfinished_job(self):
+        class SageMakerBaseSensorSubclass(SageMakerBaseSensor):
+            def non_terminal_states(self):
+                return ['PENDING', 'RUNNING', 'CONTINUE']
+
+            def failed_states(self):
+                return ['FAILED']
+
+            def get_sagemaker_response(self):
+                return {
+                    'SomeKey': {'State': 'PENDING'},
+                    'ResponseMetadata': {'HTTPStatusCode': 200}
+                }
+
+            def state_from_response(self, response):
+                return response['SomeKey']['State']
+
+        sensor = SageMakerBaseSensorSubclass(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test'
+        )
+
+        self.assertEqual(sensor.poke(None), False)
+
+    def test_poke_with_not_implemented_method(self):
+        class SageMakerBaseSensorSubclass(SageMakerBaseSensor):
+            def non_terminal_states(self):
+                return ['PENDING', 'RUNNING', 'CONTINUE']
+
+            def failed_states(self):
+                return ['FAILED']
+
+        sensor = SageMakerBaseSensorSubclass(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test'
+        )
+
+        self.assertRaises(NotImplementedError, sensor.poke, None)
+
+    def test_poke_with_bad_response(self):
+        class SageMakerBaseSensorSubclass(SageMakerBaseSensor):
+            def non_terminal_states(self):
+                return ['PENDING', 'RUNNING', 'CONTINUE']
+
+            def failed_states(self):
+                return ['FAILED']
+
+            def get_sagemaker_response(self):
+                return {
+                    'SomeKey': {'State': 'COMPLETED'},
+                    'ResponseMetadata': {'HTTPStatusCode': 400}
+                }
+
+            def state_from_response(self, response):
+                return response['SomeKey']['State']
+
+        sensor = SageMakerBaseSensorSubclass(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test'
+        )
+
+        self.assertEqual(sensor.poke(None), False)
+
+    def test_poke_with_job_failure(self):
+        class SageMakerBaseSensorSubclass(SageMakerBaseSensor):
+            def non_terminal_states(self):
+                return ['PENDING', 'RUNNING', 'CONTINUE']
+
+            def failed_states(self):
+                return ['FAILED']
+
+            def get_sagemaker_response(self):
+                return {
+                    'SomeKey': {'State': 'FAILED'},
+                    'ResponseMetadata': {'HTTPStatusCode': 200}
+                }
+
+            def state_from_response(self, response):
+                return response['SomeKey']['State']
+
+        sensor = SageMakerBaseSensorSubclass(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test'
+        )
+
+        self.assertRaises(AirflowException, sensor.poke, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py b/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py
new file mode 100644
index 0000000000000..0c2bcab70be0c
--- /dev/null
+++ b/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.sensors.sagemaker_endpoint_sensor \
+    import SageMakerEndpointSensor
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+DESCRIBE_ENDPOINT_CREATING_RESPONSE = {
+    'EndpointStatus': 'Creating',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+DESCRIBE_ENDPOINT_INSERVICE_RESPONSE = {
+    'EndpointStatus': 'InService',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+DESCRIBE_ENDPOINT_FAILED_RESPONSE = {
+    'EndpointStatus': 'Failed',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    },
+    'FailureReason': 'Unknown'
+}
+
+DESCRIBE_ENDPOINT_UPDATING_RESPONSE = {
+    'EndpointStatus': 'Updating',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+
+class TestSageMakerEndpointSensor(unittest.TestCase):
+    def setUp(self):
+        configuration.load_test_config()
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'describe_endpoint')
+    def test_sensor_with_failure(self, mock_describe, mock_client):
+        mock_describe.side_effect = [DESCRIBE_ENDPOINT_FAILED_RESPONSE]
+        sensor = SageMakerEndpointSensor(
+            task_id='test_task',
+            poke_interval=1,
+            aws_conn_id='aws_test',
+            endpoint_name='test_job_name'
+        )
+        self.assertRaises(AirflowException, sensor.execute, None)
+        mock_describe.assert_called_once_with('test_job_name')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, '__init__')
+    @mock.patch.object(SageMakerHook, 'describe_endpoint')
+    def test_sensor(self, mock_describe, hook_init, mock_client):
+        hook_init.return_value = None
+
+        mock_describe.side_effect = [
+            DESCRIBE_ENDPOINT_CREATING_RESPONSE,
+            DESCRIBE_ENDPOINT_UPDATING_RESPONSE,
+            DESCRIBE_ENDPOINT_INSERVICE_RESPONSE
+        ]
+        sensor = SageMakerEndpointSensor(
+            task_id='test_task',
+            poke_interval=1,
+            aws_conn_id='aws_test',
+            endpoint_name='test_job_name'
+        )
+
+        sensor.execute(None)
+
+        # make sure we called 3 times(terminated when its completed)
+        self.assertEqual(mock_describe.call_count, 3)
+
+        # make sure the hook was initialized with the specific params
+        hook_init.assert_called_with(aws_conn_id='aws_test')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_sagemaker_training_sensor.py b/tests/contrib/sensors/test_sagemaker_training_sensor.py
new file mode 100644
index 0000000000000..fe7944e33971d
--- /dev/null
+++ b/tests/contrib/sensors/test_sagemaker_training_sensor.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from datetime import datetime
+
+from airflow import configuration
+from airflow.contrib.sensors.sagemaker_training_sensor \
+    import SageMakerTrainingSensor
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook, LogState
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+DESCRIBE_TRAINING_COMPELETED_RESPONSE = {
+    'TrainingJobStatus': 'Completed',
+    'ResourceConfig': {
+        'InstanceCount': 1,
+        'InstanceType': 'ml.c4.xlarge',
+        'VolumeSizeInGB': 10
+    },
+    'TrainingStartTime': datetime(2018, 2, 17, 7, 15, 0, 103000),
+    'TrainingEndTime': datetime(2018, 2, 17, 7, 19, 34, 953000),
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+DESCRIBE_TRAINING_INPROGRESS_RESPONSE = dict(DESCRIBE_TRAINING_COMPELETED_RESPONSE)
+DESCRIBE_TRAINING_INPROGRESS_RESPONSE.update({'TrainingJobStatus': 'InProgress'})
+
+DESCRIBE_TRAINING_FAILED_RESPONSE = dict(DESCRIBE_TRAINING_COMPELETED_RESPONSE)
+DESCRIBE_TRAINING_FAILED_RESPONSE.update({'TrainingJobStatus': 'Failed',
+                                          'FailureReason': 'Unknown'})
+
+DESCRIBE_TRAINING_STOPPING_RESPONSE = dict(DESCRIBE_TRAINING_COMPELETED_RESPONSE)
+DESCRIBE_TRAINING_STOPPING_RESPONSE.update({'TrainingJobStatus': 'Stopping'})
+
+
+class TestSageMakerTrainingSensor(unittest.TestCase):
+    def setUp(self):
+        configuration.load_test_config()
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, '__init__')
+    @mock.patch.object(SageMakerHook, 'describe_training_job')
+    def test_sensor_with_failure(self, mock_describe_job, hook_init, mock_client):
+        hook_init.return_value = None
+
+        mock_describe_job.side_effect = [DESCRIBE_TRAINING_FAILED_RESPONSE]
+        sensor = SageMakerTrainingSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name',
+            print_log=False
+        )
+        self.assertRaises(AirflowException, sensor.execute, None)
+        mock_describe_job.assert_called_once_with('test_job_name')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, '__init__')
+    @mock.patch.object(SageMakerHook, 'describe_training_job')
+    def test_sensor(self, mock_describe_job, hook_init, mock_client):
+        hook_init.return_value = None
+
+        mock_describe_job.side_effect = [
+            DESCRIBE_TRAINING_INPROGRESS_RESPONSE,
+            DESCRIBE_TRAINING_STOPPING_RESPONSE,
+            DESCRIBE_TRAINING_COMPELETED_RESPONSE
+        ]
+        sensor = SageMakerTrainingSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name',
+            print_log=False
+        )
+
+        sensor.execute(None)
+
+        # make sure we called 3 times(terminated when its compeleted)
+        self.assertEqual(mock_describe_job.call_count, 3)
+
+        # make sure the hook was initialized with the specific params
+        hook_init.assert_called_with(aws_conn_id='aws_test')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'get_log_conn')
+    @mock.patch.object(SageMakerHook, '__init__')
+    @mock.patch.object(SageMakerHook, 'describe_training_job_with_log')
+    @mock.patch.object(SageMakerHook, 'describe_training_job')
+    def test_sensor_with_log(self, mock_describe_job, mock_describe_job_with_log,
+                             hook_init, mock_log_client, mock_client):
+        hook_init.return_value = None
+
+        mock_describe_job.return_value = DESCRIBE_TRAINING_COMPELETED_RESPONSE
+        mock_describe_job_with_log.side_effect = [
+            (LogState.WAIT_IN_PROGRESS, DESCRIBE_TRAINING_INPROGRESS_RESPONSE, 0),
+            (LogState.JOB_COMPLETE, DESCRIBE_TRAINING_STOPPING_RESPONSE, 0),
+            (LogState.COMPLETE, DESCRIBE_TRAINING_COMPELETED_RESPONSE, 0)
+        ]
+        sensor = SageMakerTrainingSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name',
+            print_log=True
+        )
+
+        sensor.execute(None)
+
+        self.assertEqual(mock_describe_job_with_log.call_count, 3)
+        self.assertEqual(mock_describe_job.call_count, 1)
+
+        hook_init.assert_called_with(aws_conn_id='aws_test')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_sagemaker_transform_sensor.py b/tests/contrib/sensors/test_sagemaker_transform_sensor.py
new file mode 100644
index 0000000000000..be84b96d88f6a
--- /dev/null
+++ b/tests/contrib/sensors/test_sagemaker_transform_sensor.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.sensors.sagemaker_transform_sensor \
+    import SageMakerTransformSensor
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+DESCRIBE_TRANSFORM_INPROGRESS_RESPONSE = {
+    'TransformJobStatus': 'InProgress',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+DESCRIBE_TRANSFORM_COMPELETED_RESPONSE = {
+    'TransformJobStatus': 'Compeleted',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+DESCRIBE_TRANSFORM_FAILED_RESPONSE = {
+    'TransformJobStatus': 'Failed',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    },
+    'FailureReason': 'Unknown'
+}
+DESCRIBE_TRANSFORM_STOPPING_RESPONSE = {
+    'TransformJobStatus': 'Stopping',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+
+class TestSageMakerTransformSensor(unittest.TestCase):
+    def setUp(self):
+        configuration.load_test_config()
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'describe_transform_job')
+    def test_sensor_with_failure(self, mock_describe_job, mock_client):
+        mock_describe_job.side_effect = [DESCRIBE_TRANSFORM_FAILED_RESPONSE]
+        sensor = SageMakerTransformSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name'
+        )
+        self.assertRaises(AirflowException, sensor.execute, None)
+        mock_describe_job.assert_called_once_with('test_job_name')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, '__init__')
+    @mock.patch.object(SageMakerHook, 'describe_transform_job')
+    def test_sensor(self, mock_describe_job, hook_init, mock_client):
+        hook_init.return_value = None
+
+        mock_describe_job.side_effect = [
+            DESCRIBE_TRANSFORM_INPROGRESS_RESPONSE,
+            DESCRIBE_TRANSFORM_STOPPING_RESPONSE,
+            DESCRIBE_TRANSFORM_COMPELETED_RESPONSE
+        ]
+        sensor = SageMakerTransformSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name'
+        )
+
+        sensor.execute(None)
+
+        # make sure we called 3 times(terminated when its compeleted)
+        self.assertEqual(mock_describe_job.call_count, 3)
+
+        # make sure the hook was initialized with the specific params
+        hook_init.assert_called_with(aws_conn_id='aws_test')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_sagemaker_tuning_sensor.py b/tests/contrib/sensors/test_sagemaker_tuning_sensor.py
new file mode 100644
index 0000000000000..b904e8a2e3aea
--- /dev/null
+++ b/tests/contrib/sensors/test_sagemaker_tuning_sensor.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow import configuration
+from airflow.contrib.sensors.sagemaker_tuning_sensor \
+    import SageMakerTuningSensor
+from airflow.contrib.hooks.sagemaker_hook import SageMakerHook
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+DESCRIBE_TUNING_INPROGRESS_RESPONSE = {
+    'HyperParameterTuningJobStatus': 'InProgress',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+DESCRIBE_TUNING_COMPELETED_RESPONSE = {
+    'HyperParameterTuningJobStatus': 'Compeleted',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+DESCRIBE_TUNING_FAILED_RESPONSE = {
+    'HyperParameterTuningJobStatus': 'Failed',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    },
+    'FailureReason': 'Unknown'
+}
+
+DESCRIBE_TUNING_STOPPING_RESPONSE = {
+    'HyperParameterTuningJobStatus': 'Stopping',
+    'ResponseMetadata': {
+        'HTTPStatusCode': 200,
+    }
+}
+
+
+class TestSageMakerTuningSensor(unittest.TestCase):
+    def setUp(self):
+        configuration.load_test_config()
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, 'describe_tuning_job')
+    def test_sensor_with_failure(self, mock_describe_job, mock_client):
+        mock_describe_job.side_effect = [DESCRIBE_TUNING_FAILED_RESPONSE]
+        sensor = SageMakerTuningSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name'
+        )
+        self.assertRaises(AirflowException, sensor.execute, None)
+        mock_describe_job.assert_called_once_with('test_job_name')
+
+    @mock.patch.object(SageMakerHook, 'get_conn')
+    @mock.patch.object(SageMakerHook, '__init__')
+    @mock.patch.object(SageMakerHook, 'describe_tuning_job')
+    def test_sensor(self, mock_describe_job, hook_init, mock_client):
+        hook_init.return_value = None
+
+        mock_describe_job.side_effect = [
+            DESCRIBE_TUNING_INPROGRESS_RESPONSE,
+            DESCRIBE_TUNING_STOPPING_RESPONSE,
+            DESCRIBE_TUNING_COMPELETED_RESPONSE
+        ]
+        sensor = SageMakerTuningSensor(
+            task_id='test_task',
+            poke_interval=2,
+            aws_conn_id='aws_test',
+            job_name='test_job_name'
+        )
+
+        sensor.execute(None)
+
+        # make sure we called 3 times(terminated when its compeleted)
+        self.assertEqual(mock_describe_job.call_count, 3)
+
+        # make sure the hook was initialized with the specific params
+        hook_init.assert_called_with(aws_conn_id='aws_test')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/contrib/sensors/test_sftp_sensor.py b/tests/contrib/sensors/test_sftp_sensor.py
index 1c13f3fb879ae..0fb9d6c610ca9 100644
--- a/tests/contrib/sensors/test_sftp_sensor.py
+++ b/tests/contrib/sensors/test_sftp_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/contrib/sensors/test_wasb_sensor.py b/tests/contrib/sensors/test_wasb_sensor.py
index f715b0166e705..6edc58da9fb2d 100644
--- a/tests/contrib/sensors/test_wasb_sensor.py
+++ b/tests/contrib/sensors/test_wasb_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,14 +25,7 @@
 from airflow import DAG, configuration
 from airflow.contrib.sensors.wasb_sensor import WasbBlobSensor
 from airflow.contrib.sensors.wasb_sensor import WasbPrefixSensor
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class TestWasbBlobSensor(unittest.TestCase):
diff --git a/tests/contrib/sensors/test_weekday_sensor.py b/tests/contrib/sensors/test_weekday_sensor.py
new file mode 100644
index 0000000000000..0255aa3a162fe
--- /dev/null
+++ b/tests/contrib/sensors/test_weekday_sensor.py
@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import unittest
+from airflow import DAG, configuration, models
+from airflow.contrib.sensors.weekday_sensor import DayOfWeekSensor
+from airflow.contrib.utils.weekday import WeekDay
+from airflow.exceptions import AirflowSensorTimeout
+from airflow.models import DagBag
+from airflow.models.taskfail import TaskFail
+from airflow.settings import Session
+from airflow.utils.timezone import datetime
+
+DEFAULT_DATE = datetime(2018, 12, 10)
+WEEKDAY_DATE = datetime(2018, 12, 20)
+WEEKEND_DATE = datetime(2018, 12, 22)
+TEST_DAG_ID = 'weekday_sensor_dag'
+DEV_NULL = '/dev/null'
+
+
+class DayOfWeekSensorTests(unittest.TestCase):
+
+    def setUp(self):
+        configuration.load_test_config()
+        self.dagbag = DagBag(
+            dag_folder=DEV_NULL,
+            include_examples=True
+        )
+        self.args = {
+            'owner': 'airflow',
+            'start_date': DEFAULT_DATE
+        }
+        dag = DAG(TEST_DAG_ID, default_args=self.args)
+        self.dag = dag
+
+    def tearDown(self):
+        session = Session()
+        session.query(models.TaskInstance).filter_by(
+            dag_id=TEST_DAG_ID).delete()
+        session.query(TaskFail).filter_by(
+            dag_id=TEST_DAG_ID).delete()
+        session.commit()
+        session.close()
+
+    def test_weekday_sensor_true(self):
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_true',
+            week_day='Thursday',
+            use_task_execution_day=True,
+            dag=self.dag)
+        t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+
+    def test_weekday_sensor_false(self):
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_false',
+            poke_interval=1,
+            timeout=2,
+            week_day='Tuesday',
+            use_task_execution_day=True,
+            dag=self.dag)
+        with self.assertRaises(AirflowSensorTimeout):
+            t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+
+    def test_invalid_weekday_number(self):
+        invalid_week_day = 'Thsday'
+        with self.assertRaisesRegexp(AttributeError,
+                                     'Invalid Week Day passed: "{}"'.format(
+                                         invalid_week_day)):
+            DayOfWeekSensor(
+                task_id='weekday_sensor_invalid_weekday_num',
+                week_day=invalid_week_day,
+                use_task_execution_day=True,
+                dag=self.dag)
+
+    def test_weekday_sensor_with_enum(self):
+        week_day = WeekDay.THURSDAY
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_true',
+            week_day=WeekDay.THURSDAY,
+            use_task_execution_day=True,
+            dag=self.dag)
+        t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+        self.assertEqual(t.week_day, week_day)
+
+    def test_weekday_sensor_with_enum_set(self):
+        week_day = {WeekDay.THURSDAY}
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_true',
+            week_day=week_day,
+            use_task_execution_day=True,
+            dag=self.dag)
+        t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+        self.assertEqual(t.week_day, week_day)
+
+    def test_weekday_sensor_with_enum_set_2_items(self):
+        week_day = {WeekDay.THURSDAY, WeekDay.FRIDAY}
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_true',
+            week_day=week_day,
+            use_task_execution_day=True,
+            dag=self.dag)
+        t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+        self.assertEqual(t.week_day, week_day)
+
+    def test_weekday_sensor_with_string_set(self):
+        week_day = {'Thursday'}
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_true',
+            week_day=week_day,
+            use_task_execution_day=True,
+            dag=self.dag)
+        t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+        self.assertEqual(t.week_day, week_day)
+
+    def test_weekday_sensor_with_string_set_2_items(self):
+        week_day = {'Thursday', 'Friday'}
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_true',
+            week_day=week_day,
+            use_task_execution_day=True,
+            dag=self.dag)
+        t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
+        self.assertEqual(t.week_day, week_day)
+
+    def test_weekday_sensor_with_invalid_type(self):
+        invalid_week_day = ['Thsday']
+        with self.assertRaisesRegexp(TypeError,
+                                     'Unsupported Type for week_day parameter:'
+                                     ' {}. It should be one of str, set or '
+                                     'Weekday enum type'.format(type(invalid_week_day))
+                                     ):
+            DayOfWeekSensor(
+                task_id='weekday_sensor_check_true',
+                week_day=invalid_week_day,
+                use_task_execution_day=True,
+                dag=self.dag)
+
+    def test_weekday_sensor_timeout_with_set(self):
+        t = DayOfWeekSensor(
+            task_id='weekday_sensor_check_false',
+            poke_interval=1,
+            timeout=2,
+            week_day={WeekDay.MONDAY, WeekDay.TUESDAY},
+            use_task_execution_day=True,
+            dag=self.dag)
+        with self.assertRaises(AirflowSensorTimeout):
+            t.run(start_date=WEEKDAY_DATE, end_date=WEEKDAY_DATE, ignore_ti_state=True)
diff --git a/tests/contrib/utils/__init__.py b/tests/contrib/utils/__init__.py
index fc532e61c6c64..b7f8352944d3f 100644
--- a/tests/contrib/utils/__init__.py
+++ b/tests/contrib/utils/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,4 +17,3 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
diff --git a/tests/contrib/utils/base_gcp_mock.py b/tests/contrib/utils/base_gcp_mock.py
new file mode 100644
index 0000000000000..444a3245cad71
--- /dev/null
+++ b/tests/contrib/utils/base_gcp_mock.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+
+import mock
+
+GCP_PROJECT_ID_HOOK_UNIT_TEST = 'example-project'
+
+
+def mock_base_gcp_hook_default_project_id(self, gcp_conn_id, delegate_to=None):
+    self.extras = {
+        'extra__google_cloud_platform__project': GCP_PROJECT_ID_HOOK_UNIT_TEST
+    }
+    self._conn = gcp_conn_id
+    self.delegate_to = delegate_to
+
+
+def mock_base_gcp_hook_no_default_project_id(self, gcp_conn_id, delegate_to=None):
+    self.extras = {
+    }
+    self._conn = gcp_conn_id
+    self.delegate_to = delegate_to
+
+
+def get_open_mock():
+    m = mock.mock_open()
+    if sys.version_info[0] == 2:
+        open_module = '__builtin__'
+    else:
+        open_module = 'builtins'
+    return m, open_module
diff --git a/tests/contrib/utils/base_gcp_system_test_case.py b/tests/contrib/utils/base_gcp_system_test_case.py
new file mode 100644
index 0000000000000..70b47497c8680
--- /dev/null
+++ b/tests/contrib/utils/base_gcp_system_test_case.py
@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+import subprocess
+import unittest
+from glob import glob
+from shutil import move
+from tempfile import mkdtemp
+
+from airflow.utils import db as db_utils
+from airflow import models, settings, AirflowException, LoggingMixin
+from airflow.utils.timezone import datetime
+from tests.contrib.utils.gcp_authenticator import GcpAuthenticator
+from tests.contrib.utils.run_once_decorator import run_once
+
+AIRFLOW_MAIN_FOLDER = os.path.realpath(os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    os.pardir, os.pardir, os.pardir))
+
+AIRFLOW_PARENT_FOLDER = os.path.realpath(os.path.join(AIRFLOW_MAIN_FOLDER,
+                                                      os.pardir, os.pardir, os.pardir))
+ENV_FILE_RETRIEVER = os.path.join(AIRFLOW_PARENT_FOLDER,
+                                  "get_system_test_environment_variables.py")
+
+
+# Retrieve environment variables from parent directory retriever - it should be
+# in the path ${AIRFLOW_SOURCE_DIR}/../../get_system_test_environment_variables.py
+# and it should print all the variables in form of key=value to the stdout
+class RetrieveVariables:
+    @staticmethod
+    @run_once
+    def retrieve_variables():
+        if os.path.isfile(ENV_FILE_RETRIEVER):
+            if os.environ.get('AIRFLOW__CORE__UNIT_TEST_MODE'):
+                raise Exception("Please unset the AIRFLOW__CORE__UNIT_TEST_MODE")
+            variables = subprocess.check_output([ENV_FILE_RETRIEVER]).decode("utf-8")
+            print("Applying variables retrieved")
+            for line in variables.split("\n"):
+                try:
+                    variable, key = line.split("=")
+                except ValueError:
+                    continue
+                print("{}={}".format(variable, key))
+                os.environ[variable] = key
+
+
+RetrieveVariables.retrieve_variables()
+
+DEFAULT_DATE = datetime(2015, 1, 1)
+
+CONTRIB_OPERATORS_EXAMPLES_DAG_FOLDER = os.path.join(
+    AIRFLOW_MAIN_FOLDER, "airflow", "contrib", "example_dags")
+
+OPERATORS_EXAMPLES_DAG_FOLDER = os.path.join(
+    AIRFLOW_MAIN_FOLDER, "airflow", "example_dags")
+
+AIRFLOW_HOME = os.environ.get('AIRFLOW_HOME',
+                              os.path.join(os.path.expanduser('~'), 'airflow'))
+
+DAG_FOLDER = os.path.join(AIRFLOW_HOME, "dags")
+
+
+SKIP_TEST_WARNING = """
+The test is only run when the test is run in with GCP-system-tests enabled
+environment. You can enable it in one of two ways:
+
+* Set GCP_CONFIG_DIR environment variable to point to the GCP configuration
+  directory which keeps variables.env file with environment variables to set
+  and keys directory which keeps service account keys in .json format
+* Run this test within automated environment variable workspace where
+  config directory is checked out next to the airflow one.
+
+""".format(__file__)
+
+
+class BaseGcpSystemTestCase(unittest.TestCase, LoggingMixin):
+    def __init__(self,
+                 method_name,
+                 gcp_key,
+                 project_extra=None):
+        super(BaseGcpSystemTestCase, self).__init__(methodName=method_name)
+        self.gcp_authenticator = GcpAuthenticator(gcp_key=gcp_key,
+                                                  project_extra=project_extra)
+        self.setup_called = False
+
+    @staticmethod
+    def skip_check(key_name):
+        return GcpAuthenticator(key_name).full_key_path is None
+
+    def setUp(self):
+        self.gcp_authenticator.gcp_store_authentication()
+        self.gcp_authenticator.gcp_authenticate()
+        # We checked that authentication works. Ne we revoke it to make
+        # sure we are not relying on the default authentication
+        self.gcp_authenticator.gcp_revoke_authentication()
+        self.setup_called = True
+
+    # noinspection PyPep8Naming
+    def tearDown(self):
+        self.gcp_authenticator.gcp_restore_authentication()
+
+
+class DagGcpSystemTestCase(BaseGcpSystemTestCase):
+    def __init__(self,
+                 method_name,
+                 gcp_key,
+                 dag_id=None,
+                 dag_name=None,
+                 require_local_executor=False,
+                 example_dags_folder=CONTRIB_OPERATORS_EXAMPLES_DAG_FOLDER,
+                 project_extra=None):
+        super(DagGcpSystemTestCase, self).__init__(method_name=method_name,
+                                                   gcp_key=gcp_key,
+                                                   project_extra=project_extra)
+        self.dag_id = dag_id
+        self.dag_name = self.dag_id + '.py' if not dag_name else dag_name
+        self.example_dags_folder = example_dags_folder
+        self.require_local_executor = require_local_executor
+        self.temp_dir = None
+
+    @staticmethod
+    def _get_dag_folder():
+        return DAG_FOLDER
+
+    @staticmethod
+    def _get_files_to_link(path):
+        """
+        Returns all file names (note - file names not paths)
+        that have the same base name as the .py dag file (for example dag_name.sql etc.)
+        :param path: path to the dag file.
+        :return: list of files matching the base name
+        """
+        prefix, ext = os.path.splitext(path)
+        assert ext == '.py', "Dag name should be a .py file and is {} file".format(ext)
+        files_to_link = []
+        for file in glob(prefix + ".*"):
+            files_to_link.append(os.path.basename(file))
+        return files_to_link
+
+    def _symlink_dag_and_associated_files(self, remove=False):
+        target_folder = self._get_dag_folder()
+        source_path = os.path.join(self.example_dags_folder, self.dag_name)
+        for file_name in self._get_files_to_link(source_path):
+            source_path = os.path.join(self.example_dags_folder, file_name)
+            target_path = os.path.join(target_folder, file_name)
+            if remove:
+                try:
+                    self.log.info("Remove symlink: %s -> %s", target_path, source_path)
+                    os.remove(target_path)
+                except OSError:
+                    pass
+            else:
+                if not os.path.exists(target_path):
+                    self.log.info("Symlink: %s -> %s ", target_path, source_path)
+                    os.symlink(source_path, target_path)
+                else:
+                    self.log.info("Symlink %s already exists. Not symlinking it.", target_path)
+
+    def _store_dags_to_temporary_directory(self):
+        dag_folder = self._get_dag_folder()
+        self.temp_dir = mkdtemp()
+        self.log.info("Storing DAGS from %s to temporary directory %s", dag_folder, self.temp_dir)
+        try:
+            os.mkdir(dag_folder)
+        except OSError:
+            pass
+        for file in os.listdir(dag_folder):
+            move(os.path.join(dag_folder, file), os.path.join(self.temp_dir, file))
+
+    def _restore_dags_from_temporary_directory(self):
+        dag_folder = self._get_dag_folder()
+        self.log.info("Restoring DAGS to %s from temporary directory %s", dag_folder, self.temp_dir)
+        for file in os.listdir(self.temp_dir):
+            move(os.path.join(self.temp_dir, file), os.path.join(dag_folder, file))
+
+    def _run_dag(self, dag_id=None):
+        self.log.info("Attempting to run DAG: %s", self.dag_id)
+        if not self.setup_called:
+            raise AirflowException("Please make sure to call super.setUp() in your "
+                                   "test class!")
+        dag_folder = self._get_dag_folder()
+        dag_bag = models.DagBag(dag_folder=dag_folder, include_examples=False)
+        self.args = {'owner': 'airflow', 'start_date': DEFAULT_DATE}
+        dag = dag_bag.get_dag(self.dag_id or dag_id)
+        if dag is None:
+            raise AirflowException(
+                "The Dag {} could not be found. It's either an import problem or "
+                "the dag {} was not symlinked to the DAGs folder. "
+                "The content of the {} folder is {}".
+                format(self.dag_id,
+                       self.dag_id + ".py",
+                       dag_folder,
+                       os.listdir(dag_folder)))
+        dag.clear(reset_dag_runs=True)
+        dag.run(ignore_first_depends_on_past=True, verbose=True)
+
+    @staticmethod
+    def _check_local_executor_setup():
+        postgres_path = os.path.realpath(os.path.join(
+            AIRFLOW_MAIN_FOLDER,
+            "tests", "contrib", "operators", "postgres_local_executor.cfg"))
+        if postgres_path != os.environ.get('AIRFLOW_CONFIG'):
+            raise AirflowException(
+                """
+Please set AIRFLOW_CONFIG variable to '{}'
+and make sure you have a Postgres server running locally and
+airflow/airflow.db database created.
+
+You can create the database via these commands:
+'createuser root'
+'createdb airflow/airflow.db`
+
+""".format(postgres_path))
+
+    # noinspection PyPep8Naming
+    def setUp(self):
+        if self.require_local_executor:
+            self._check_local_executor_setup()
+        try:
+            # We want to avoid random errors while database got reset - those
+            # Are apparently triggered by parser trying to parse DAGs while
+            # The tables are dropped. We move the dags temporarily out of the dags folder
+            # and move them back after reset
+            self._store_dags_to_temporary_directory()
+            try:
+                db_utils.upgradedb()
+                db_utils.resetdb(settings.RBAC)
+            finally:
+                self._restore_dags_from_temporary_directory()
+            self._symlink_dag_and_associated_files()
+            super(DagGcpSystemTestCase, self).setUp()
+
+        except Exception as e:
+            # In case of any error during setup - restore the authentication
+            self.gcp_authenticator.gcp_restore_authentication()
+            raise e
+
+    def tearDown(self):
+        self._symlink_dag_and_associated_files(remove=True)
+        super(DagGcpSystemTestCase, self).tearDown()
diff --git a/tests/contrib/utils/gcp_authenticator.py b/tests/contrib/utils/gcp_authenticator.py
new file mode 100644
index 0000000000000..cec7ade2da46e
--- /dev/null
+++ b/tests/contrib/utils/gcp_authenticator.py
@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import os
+import subprocess
+
+from airflow import settings, AirflowException
+from tests.contrib.utils.logging_command_executor import LoggingCommandExecutor
+
+from airflow.models import Connection
+
+GCP_COMPUTE_KEY = 'gcp_compute.json'
+GCP_FUNCTION_KEY = 'gcp_function.json'
+GCP_CLOUDSQL_KEY = 'gcp_cloudsql.json'
+GCP_BIGTABLE_KEY = 'gcp_bigtable.json'
+GCP_SPANNER_KEY = 'gcp_spanner.json'
+GCP_GCS_KEY = 'gcp_gcs.json'
+GCP_AI_KEY = 'gcp_ai.json'
+GCP_GCS_TRANSFER_KEY = 'gcp_gcs_transfer.json'
+
+KEYPATH_EXTRA = 'extra__google_cloud_platform__key_path'
+KEYFILE_DICT_EXTRA = 'extra__google_cloud_platform__keyfile_dict'
+SCOPE_EXTRA = 'extra__google_cloud_platform__scope'
+PROJECT_EXTRA = 'extra__google_cloud_platform__project'
+
+AIRFLOW_MAIN_FOLDER = os.path.realpath(
+    os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)
+)
+
+
+class GcpAuthenticator(LoggingCommandExecutor):
+    """
+    Manages authentication to Google Cloud Platform. It helps to manage
+    connection - it can authenticate with the gcp key name specified
+    """
+
+    original_account = None
+
+    def __init__(self, gcp_key, project_extra=None):
+        """
+        Initialises the authenticator.
+
+        :param gcp_key: name of the key to use for authentication (see GCP_*_KEY values)
+        :param project_extra: optional extra project parameter passed to google cloud
+               connection
+        """
+        super(GcpAuthenticator, self).__init__()
+        self.gcp_key = gcp_key
+        self.project_extra = project_extra
+        self.project_id = self.get_project_id()
+        self.full_key_path = None
+        self._set_key_path()
+
+    @staticmethod
+    def get_project_id():
+        return os.environ.get('GCP_PROJECT_ID')
+
+    def set_key_path_in_airflow_connection(self):
+        """
+        Set key path in 'google_cloud_default' connection to point to the full
+        key path
+        :return: None
+        """
+        session = settings.Session()
+        try:
+            conn = session.query(Connection).filter(Connection.conn_id == 'google_cloud_default')[0]
+            extras = conn.extra_dejson
+            extras[KEYPATH_EXTRA] = self.full_key_path
+            if extras.get(KEYFILE_DICT_EXTRA):
+                del extras[KEYFILE_DICT_EXTRA]
+            extras[SCOPE_EXTRA] = 'https://www.googleapis.com/auth/cloud-platform'
+            extras[PROJECT_EXTRA] = self.project_extra if self.project_extra else self.project_id
+            conn.extra = json.dumps(extras)
+            session.commit()
+        except BaseException as ex:
+            self.log.info('Airflow DB Session error:' + str(ex))
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+    def set_dictionary_in_airflow_connection(self):
+        """
+        Set dictionary in 'google_cloud_default' connection to contain content
+        of the json service account file.
+        :return: None
+        """
+        session = settings.Session()
+        try:
+            conn = session.query(Connection).filter(Connection.conn_id == 'google_cloud_default')[0]
+            extras = conn.extra_dejson
+            with open(self.full_key_path, "r") as path_file:
+                content = json.load(path_file)
+            extras[KEYFILE_DICT_EXTRA] = json.dumps(content)
+            if extras.get(KEYPATH_EXTRA):
+                del extras[KEYPATH_EXTRA]
+            extras[SCOPE_EXTRA] = 'https://www.googleapis.com/auth/cloud-platform'
+            extras[PROJECT_EXTRA] = self.project_extra
+            conn.extra = json.dumps(extras)
+            session.commit()
+        except BaseException as ex:
+            self.log.info('Airflow DB Session error:' + str(ex))
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+    def _set_key_path(self):
+        """
+        Sets full key path - if GCP_CONFIG_DIR points to absolute
+            directory, it tries to find the key in this directory. Otherwise it assumes
+            that Airflow is run from directory where configuration i checked out
+            next to airflow directory in config directory
+            it tries to find the key folder in the workspace's config
+            directory.
+        :param : name of the key file to find.
+        """
+        if "GCP_CONFIG_DIR" in os.environ:
+            gcp_config_dir = os.environ["GCP_CONFIG_DIR"]
+        else:
+            gcp_config_dir = os.path.join(AIRFLOW_MAIN_FOLDER, os.pardir, "config")
+        if not os.path.isdir(gcp_config_dir):
+            self.log.info("The {} is not a directory".format(gcp_config_dir))
+        key_dir = os.path.join(gcp_config_dir, "keys")
+        if not os.path.isdir(key_dir):
+            self.log.info("The {} is not a directory".format(key_dir))
+            return
+        key_path = os.path.join(key_dir, self.gcp_key)
+        if not os.path.isfile(key_path):
+            self.log.info("The {} is missing".format(key_path))
+        self.full_key_path = key_path
+
+    def _validate_key_set(self):
+        if self.full_key_path is None:
+            raise AirflowException("The gcp_key is not set!")
+        if not os.path.isfile(self.full_key_path):
+            raise AirflowException(
+                "The key {} could not be found. Please copy it to the {} path.".format(
+                    self.gcp_key, self.full_key_path
+                )
+            )
+
+    def gcp_authenticate(self):
+        """
+        Authenticate with service account specified via key name.
+        """
+        self._validate_key_set()
+        self.log.info("Setting the GCP key to {}".format(self.full_key_path))
+        # Checking if we can authenticate using service account credentials provided
+        self.execute_cmd(
+            [
+                'gcloud',
+                'auth',
+                'activate-service-account',
+                '--key-file={}'.format(self.full_key_path),
+                '--project={}'.format(self.project_id),
+            ]
+        )
+        self.set_key_path_in_airflow_connection()
+
+    def gcp_revoke_authentication(self):
+        """
+        Change default authentication to none - which is not existing one.
+        """
+        self._validate_key_set()
+        self.log.info("Revoking authentication - setting it to none")
+        self.execute_cmd(['gcloud', 'config', 'get-value', 'account', '--project={}'.format(self.project_id)])
+        self.execute_cmd(
+            ['gcloud', 'config', 'set', 'account', 'none', '--project={}'.format(self.project_id)]
+        )
+
+    def gcp_store_authentication(self):
+        """
+        Store authentication as it was originally so it can be restored and revoke
+        authentication.
+        """
+        self._validate_key_set()
+        if not GcpAuthenticator.original_account:
+            GcpAuthenticator.original_account = self.check_output(
+                ['gcloud', 'config', 'get-value', 'account', '--project={}'.format(self.project_id)]
+            ).decode('utf-8')
+            self.log.info("Storing account: to restore it later {}".format(GcpAuthenticator.original_account))
+
+    def gcp_restore_authentication(self):
+        """
+        Restore authentication to the original one one.
+        """
+        self._validate_key_set()
+        if GcpAuthenticator.original_account:
+            self.log.info("Restoring original account stored: {}".format(GcpAuthenticator.original_account))
+            subprocess.call(
+                [
+                    'gcloud',
+                    'config',
+                    'set',
+                    'account',
+                    GcpAuthenticator.original_account,
+                    '--project={}'.format(self.project_id),
+                ]
+            )
+        else:
+            self.log.info("Not restoring the original GCP account: it is not set")
diff --git a/tests/contrib/utils/logging_command_executor.py b/tests/contrib/utils/logging_command_executor.py
new file mode 100644
index 0000000000000..e0833f4a77b9e
--- /dev/null
+++ b/tests/contrib/utils/logging_command_executor.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+import subprocess
+
+from airflow import LoggingMixin, AirflowException
+
+
+class LoggingCommandExecutor(LoggingMixin):
+
+    def execute_cmd(self, cmd, silent=False):
+        if silent:
+            self.log.info("Executing in silent mode: '{}'".format(" ".join(cmd)))
+            with open(os.devnull, 'w') as FNULL:
+                return subprocess.call(args=cmd, stdout=FNULL, stderr=subprocess.STDOUT)
+        else:
+            self.log.info("Executing: '{}'".format(" ".join(cmd)))
+            process = subprocess.Popen(args=cmd, stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE, universal_newlines=True)
+            output, err = process.communicate()
+            retcode = process.poll()
+            self.log.info("Stdout: {}".format(output))
+            self.log.info("Stderr: {}".format(err))
+            if retcode:
+                print("Error when executing '{}'".format(" ".join(cmd)))
+            return retcode
+
+    def check_output(self, cmd):
+        self.log.info("Executing for output: '{}'".format(" ".join(cmd)))
+        process = subprocess.Popen(args=cmd, stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+        output, err = process.communicate()
+        retcode = process.poll()
+        if retcode:
+            self.log.info("Error when executing '{}'".format(" ".join(cmd)))
+            self.log.info("Stdout: {}".format(output))
+            self.log.info("Stderr: {}".format(err))
+            raise AirflowException("Retcode {} on {} with stdout: {}, stderr: {}".
+                                   format(retcode, " ".join(cmd), output, err))
+        return output
diff --git a/tests/contrib/utils/run_once_decorator.py b/tests/contrib/utils/run_once_decorator.py
new file mode 100644
index 0000000000000..934250ec2ffa7
--- /dev/null
+++ b/tests/contrib/utils/run_once_decorator.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from functools import wraps
+
+
+def run_once(f):
+    """Runs a function (successfully) only once.
+    The running can be reset by setting the `has_run` attribute to False
+    """
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        if not wrapper.has_run:
+            result = f(*args, **kwargs)
+            wrapper.has_run = True
+            return result
+    wrapper.has_run = False
+    return wrapper
diff --git a/tests/contrib/operators/test_mlengine_operator_utils.py b/tests/contrib/utils/test_mlengine_operator_utils.py
similarity index 88%
rename from tests/contrib/operators/test_mlengine_operator_utils.py
rename to tests/contrib/utils/test_mlengine_operator_utils.py
index 0cb106da6bb8d..c6c0849250040 100644
--- a/tests/contrib/operators/test_mlengine_operator_utils.py
+++ b/tests/contrib/utils/test_mlengine_operator_utils.py
@@ -23,8 +23,7 @@
 import unittest
 
 from airflow import configuration, DAG
-from airflow.contrib.operators import mlengine_operator_utils
-from airflow.contrib.operators.mlengine_operator_utils import create_evaluate_ops
+from airflow.contrib.utils import mlengine_operator_utils
 from airflow.exceptions import AirflowException
 from airflow.version import version
 
@@ -36,7 +35,6 @@
 
 
 class CreateEvaluateOpsTest(unittest.TestCase):
-
     INPUT_MISSING_ORIGIN = {
         'dataFormat': 'TEXT',
         'inputPaths': ['gs://legal-bucket/fake-input-path/*'],
@@ -77,7 +75,7 @@ def setUp(self):
     def testSuccessfulRun(self):
         input_with_model = self.INPUT_MISSING_ORIGIN.copy()
 
-        pred, summary, validate = create_evaluate_ops(
+        pred, summary, validate = mlengine_operator_utils.create_evaluate_ops(
             task_prefix='eval-test',
             batch_prediction_job_id='eval-test-prediction',
             data_format=input_with_model['dataFormat'],
@@ -89,7 +87,6 @@ def testSuccessfulRun(self):
 
         with patch('airflow.contrib.operators.mlengine_operator.'
                    'MLEngineHook') as mock_mlengine_hook:
-
             success_message = self.SUCCESS_MESSAGE_MISSING_INPUT.copy()
             success_message['predictionInput'] = input_with_model
             hook_instance = mock_mlengine_hook.return_value
@@ -107,26 +104,24 @@ def testSuccessfulRun(self):
 
         with patch('airflow.contrib.operators.dataflow_operator.'
                    'DataFlowHook') as mock_dataflow_hook:
-
             hook_instance = mock_dataflow_hook.return_value
             hook_instance.start_python_dataflow.return_value = None
             summary.execute(None)
             mock_dataflow_hook.assert_called_with(
                 gcp_conn_id='google_cloud_default', delegate_to=None, poll_sleep=10)
             hook_instance.start_python_dataflow.assert_called_once_with(
-                'eval-test-summary',
+                '{{task.task_id}}',
                 {
                     'prediction_path': 'gs://legal-bucket/fake-output-path',
                     'labels': {'airflow-version': TEST_VERSION},
                     'metric_keys': 'err',
                     'metric_fn_encoded': self.metric_fn_encoded,
                 },
-                'airflow.contrib.operators.mlengine_prediction_summary',
+                'airflow.contrib.utils.mlengine_prediction_summary',
                 ['-m'])
 
-        with patch('airflow.contrib.operators.mlengine_operator_utils.'
+        with patch('airflow.contrib.utils.mlengine_operator_utils.'
                    'GoogleCloudStorageHook') as mock_gcs_hook:
-
             hook_instance = mock_gcs_hook.return_value
             hook_instance.download.return_value = '{"err": 0.9, "count": 9}'
             result = validate.execute({})
@@ -159,27 +154,27 @@ def testFailures(self):
         }
 
         with self.assertRaisesRegexp(AirflowException, 'Missing model origin'):
-            _ = create_evaluate_ops(**other_params_but_models)
+            mlengine_operator_utils.create_evaluate_ops(**other_params_but_models)
 
         with self.assertRaisesRegexp(AirflowException, 'Ambiguous model origin'):
-            _ = create_evaluate_ops(model_uri='abc', model_name='cde',
-                                    **other_params_but_models)
+            mlengine_operator_utils.create_evaluate_ops(model_uri='abc', model_name='cde',
+                                                        **other_params_but_models)
 
         with self.assertRaisesRegexp(AirflowException, 'Ambiguous model origin'):
-            _ = create_evaluate_ops(model_uri='abc', version_name='vvv',
-                                    **other_params_but_models)
+            mlengine_operator_utils.create_evaluate_ops(model_uri='abc', version_name='vvv',
+                                                        **other_params_but_models)
 
         with self.assertRaisesRegexp(AirflowException,
                                      '`metric_fn` param must be callable'):
             params = other_params_but_models.copy()
             params['metric_fn_and_keys'] = (None, ['abc'])
-            _ = create_evaluate_ops(model_uri='gs://blah', **params)
+            mlengine_operator_utils.create_evaluate_ops(model_uri='gs://blah', **params)
 
         with self.assertRaisesRegexp(AirflowException,
                                      '`validate_fn` param must be callable'):
             params = other_params_but_models.copy()
             params['validate_fn'] = None
-            _ = create_evaluate_ops(model_uri='gs://blah', **params)
+            mlengine_operator_utils.create_evaluate_ops(model_uri='gs://blah', **params)
 
 
 if __name__ == '__main__':
diff --git a/tests/contrib/utils/test_sendgrid.py b/tests/contrib/utils/test_sendgrid.py
index 6710076c2da80..d8a3c9439a1b7 100644
--- a/tests/contrib/utils/test_sendgrid.py
+++ b/tests/contrib/utils/test_sendgrid.py
@@ -22,14 +22,7 @@
 import unittest
 
 from airflow.contrib.utils.sendgrid import send_email
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class SendEmailSendGridTest(unittest.TestCase):
diff --git a/tests/contrib/utils/test_weekday.py b/tests/contrib/utils/test_weekday.py
new file mode 100644
index 0000000000000..961652a4ea8e6
--- /dev/null
+++ b/tests/contrib/utils/test_weekday.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+from enum import Enum
+
+from airflow.contrib.utils.weekday import WeekDay
+
+
+class WeekDayTest(unittest.TestCase):
+    def test_weekday_enum_length(self):
+        self.assertEqual(len(WeekDay), 7)
+
+    def test_weekday_name_value(self):
+        weekdays = "MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY SUNDAY"
+        weekdays = weekdays.split()
+        for i, weekday in enumerate(weekdays, start=1):
+            e = WeekDay(i)
+            self.assertEqual(e, i)
+            self.assertEqual(int(e), i)
+            self.assertEqual(e.name, weekday)
+            self.assertTrue(e in WeekDay)
+            self.assertTrue(0 < e < 8)
+            self.assertTrue(type(e) is WeekDay)
+            self.assertTrue(isinstance(e, int))
+            self.assertTrue(isinstance(e, Enum))
diff --git a/tests/core.py b/tests/core.py
index c3adb3c5005aa..20b714dca74ba 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -22,7 +22,6 @@
 import json
 import unittest
 
-import bleach
 import doctest
 import mock
 import multiprocessing
@@ -38,18 +37,21 @@
 from email.mime.application import MIMEApplication
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
-from freezegun import freeze_time
 from numpy.testing import assert_array_almost_equal
 from six.moves.urllib.parse import urlencode
 from time import sleep
 
+from bs4 import BeautifulSoup
+
 from airflow import configuration
 from airflow.executors import SequentialExecutor
-from airflow.models import Variable
+from airflow.models import Variable, TaskInstance
+
 
-configuration.conf.load_test_config()
 from airflow import jobs, models, DAG, utils, macros, settings, exceptions
 from airflow.models import BaseOperator
+from airflow.models.connection import Connection
+from airflow.models.taskfail import TaskFail
 from airflow.operators.bash_operator import BashOperator
 from airflow.operators.check_operator import CheckOperator, ValueCheckOperator
 from airflow.operators.dagrun_operator import TriggerDagRunOperator
@@ -64,16 +66,16 @@
 from airflow.utils import timezone
 from airflow.utils.timezone import datetime
 from airflow.utils.state import State
-from airflow.utils.dates import infer_time_unit, round_time, scale_time_units
-from lxml import html
+from airflow.utils.dates import days_ago, infer_time_unit, round_time, scale_time_units
 from airflow.exceptions import AirflowException
 from airflow.configuration import AirflowConfigException, run_command
-from jinja2.sandbox import SecurityError
+from jinja2.exceptions import SecurityError
 from jinja2 import UndefinedError
+from pendulum import utcnow
 
 import six
 
-NUM_EXAMPLE_DAGS = 20
+NUM_EXAMPLE_DAGS = 18
 DEV_NULL = '/dev/null'
 TEST_DAG_FOLDER = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), 'dags')
@@ -81,23 +83,13 @@
 DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat()
 DEFAULT_DATE_DS = DEFAULT_DATE_ISO[:10]
 TEST_DAG_ID = 'unit_tests'
+EXAMPLE_DAG_DEFAULT_DATE = days_ago(2)
 
 try:
     import cPickle as pickle
 except ImportError:
     # Python 3
-    import pickle
-
-
-def reset(dag_id=TEST_DAG_ID):
-    session = Session()
-    tis = session.query(models.TaskInstance).filter_by(dag_id=dag_id)
-    tis.delete()
-    session.commit()
-    session.close()
-
-
-reset()
+    import pickle  # type: ignore
 
 
 class OperatorSubclass(BaseOperator):
@@ -128,6 +120,16 @@ def setUp(self):
         self.run_after_loop = self.dag_bash.get_task('run_after_loop')
         self.run_this_last = self.dag_bash.get_task('run_this_last')
 
+    def tearDown(self):
+        if os.environ.get('KUBERNETES_VERSION') is None:
+            session = Session()
+            session.query(models.TaskInstance).filter_by(
+                dag_id=TEST_DAG_ID).delete()
+            session.query(TaskFail).filter_by(
+                dag_id=TEST_DAG_ID).delete()
+            session.commit()
+            session.close()
+
     def test_schedule_dag_no_previous_runs(self):
         """
         Tests scheduling a dag with no previous runs
@@ -153,6 +155,46 @@ def test_schedule_dag_no_previous_runs(self):
         self.assertFalse(dag_run.external_trigger)
         dag.clear()
 
+    def test_schedule_dag_relativedelta(self):
+        """
+        Tests scheduling a dag with a relativedelta schedule_interval
+        """
+        delta = relativedelta(hours=+1)
+        dag = DAG(TEST_DAG_ID + 'test_schedule_dag_relativedelta',
+                  schedule_interval=delta)
+        dag.add_task(models.BaseOperator(
+            task_id="faketastic",
+            owner='Also fake',
+            start_date=datetime(2015, 1, 2, 0, 0)))
+
+        dag_run = jobs.SchedulerJob(**self.default_scheduler_args).create_dag_run(dag)
+        self.assertIsNotNone(dag_run)
+        self.assertEqual(dag.dag_id, dag_run.dag_id)
+        self.assertIsNotNone(dag_run.run_id)
+        self.assertNotEqual('', dag_run.run_id)
+        self.assertEqual(
+            datetime(2015, 1, 2, 0, 0),
+            dag_run.execution_date,
+            msg='dag_run.execution_date did not match expectation: {0}'
+            .format(dag_run.execution_date)
+        )
+        self.assertEqual(State.RUNNING, dag_run.state)
+        self.assertFalse(dag_run.external_trigger)
+        dag_run2 = jobs.SchedulerJob(**self.default_scheduler_args).create_dag_run(dag)
+        self.assertIsNotNone(dag_run2)
+        self.assertEqual(dag.dag_id, dag_run2.dag_id)
+        self.assertIsNotNone(dag_run2.run_id)
+        self.assertNotEqual('', dag_run2.run_id)
+        self.assertEqual(
+            datetime(2015, 1, 2, 0, 0) + delta,
+            dag_run2.execution_date,
+            msg='dag_run2.execution_date did not match expectation: {0}'
+            .format(dag_run2.execution_date)
+        )
+        self.assertEqual(State.RUNNING, dag_run2.state)
+        self.assertFalse(dag_run2.external_trigger)
+        dag.clear()
+
     def test_schedule_dag_fake_scheduled_previous(self):
         """
         Test scheduling a dag where there is a prior DagRun
@@ -261,7 +303,6 @@ def test_schedule_dag_start_end_dates(self):
 
         self.assertIsNone(additional_dag_run)
 
-    @freeze_time('2016-01-01')
     def test_schedule_dag_no_end_date_up_to_today_only(self):
         """
         Tests that a Dag created without an end_date can only be scheduled up
@@ -273,8 +314,11 @@ def test_schedule_dag_no_end_date_up_to_today_only(self):
         """
         session = settings.Session()
         delta = timedelta(days=1)
-        start_date = DEFAULT_DATE
-        runs = 365
+        now = utcnow()
+        start_date = now.subtract(weeks=1)
+
+        runs = (now - start_date).days
+
         dag = DAG(TEST_DAG_ID + 'test_schedule_dag_no_end_date_up_to_today_only',
                   start_date=start_date,
                   schedule_interval=delta)
@@ -392,7 +436,7 @@ def test_illegal_args(self):
         Tests that Operators reject illegal arguments
         """
         with warnings.catch_warnings(record=True) as w:
-            t = BashOperator(
+            BashOperator(
                 task_id='test_illegal_args',
                 bash_command='echo success',
                 dag=self.dag,
@@ -400,7 +444,8 @@ def test_illegal_args(self):
             self.assertTrue(
                 issubclass(w[0].category, PendingDeprecationWarning))
             self.assertIn(
-                'Invalid arguments were passed to BashOperator.',
+                ('Invalid arguments were passed to BashOperator '
+                 '(task_id: test_illegal_args).'),
                 w[0].message.args[0])
 
     def test_bash_operator(self):
@@ -439,6 +484,26 @@ def test_bash_operator_kill(self):
             os.kill(pid, signal.SIGTERM)
             self.fail("BashOperator's subprocess still running after stopping on timeout!")
 
+    def test_on_failure_callback(self):
+        # Annoying workaround for nonlocal not existing in python 2
+        data = {'called': False}
+
+        def check_failure(context, test_case=self):
+            data['called'] = True
+            error = context.get('exception')
+            test_case.assertIsInstance(error, AirflowException)
+
+        t = BashOperator(
+            task_id='check_on_failure_callback',
+            bash_command="exit 1",
+            dag=self.dag,
+            on_failure_callback=check_failure)
+        self.assertRaises(
+            exceptions.AirflowException,
+            t.run,
+            start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+        self.assertTrue(data['called'])
+
     def test_trigger_dagrun(self):
         def trigga(context, obj):
             if True:
@@ -584,6 +649,36 @@ def __bool__(self):
             dag=self.dag)
         t.resolve_template_files()
 
+    def test_task_get_template(self):
+        TI = models.TaskInstance
+        ti = TI(
+            task=self.runme_0, execution_date=DEFAULT_DATE)
+        ti.dag = self.dag_bash
+        ti.run(ignore_ti_state=True)
+        context = ti.get_template_context()
+
+        # DEFAULT DATE is 2015-01-01
+        self.assertEqual(context['ds'], '2015-01-01')
+        self.assertEqual(context['ds_nodash'], '20150101')
+
+        # next_ds is 2015-01-02 as the dag interval is daily
+        self.assertEqual(context['next_ds'], '2015-01-02')
+        self.assertEqual(context['next_ds_nodash'], '20150102')
+
+        # prev_ds is 2014-12-31 as the dag interval is daily
+        self.assertEqual(context['prev_ds'], '2014-12-31')
+        self.assertEqual(context['prev_ds_nodash'], '20141231')
+
+        self.assertEqual(context['ts'], '2015-01-01T00:00:00+00:00')
+        self.assertEqual(context['ts_nodash'], '20150101T000000')
+        self.assertEqual(context['ts_nodash_with_tz'], '20150101T000000+0000')
+
+        self.assertEqual(context['yesterday_ds'], '2014-12-31')
+        self.assertEqual(context['yesterday_ds_nodash'], '20141231')
+
+        self.assertEqual(context['tomorrow_ds'], '2015-01-02')
+        self.assertEqual(context['tomorrow_ds_nodash'], '20150102')
+
     def test_import_examples(self):
         self.assertEqual(len(self.dagbag.dags), NUM_EXAMPLE_DAGS)
 
@@ -622,6 +717,13 @@ def test_get_non_existing_var_should_return_default(self):
         self.assertEqual(default_value, Variable.get("thisIdDoesNotExist",
                                                      default_var=default_value))
 
+    def test_get_non_existing_var_should_raise_key_error(self):
+        with self.assertRaises(KeyError):
+            Variable.get("thisIdDoesNotExist")
+
+    def test_get_non_existing_var_with_none_default_should_return_none(self):
+        self.assertIsNone(Variable.get("thisIdDoesNotExist", default_var=None))
+
     def test_get_non_existing_var_should_not_deserialize_json_default(self):
         default_value = "}{ this is a non JSON default }{"
         self.assertEqual(default_value, Variable.get("thisIdDoesNotExist",
@@ -774,22 +876,11 @@ def test_scale_time_units(self):
         arr4 = scale_time_units([200000, 100000], 'days')
         assert_array_almost_equal(arr4, [2.315, 1.157], decimal=3)
 
-    def test_duplicate_dependencies(self):
-
-        regexp = "Dependency (.*)runme_0(.*)run_after_loop(.*) " \
-                 "already registered"
-
-        with self.assertRaisesRegexp(AirflowException, regexp):
-            self.runme_0.set_downstream(self.run_after_loop)
-
-        with self.assertRaisesRegexp(AirflowException, regexp):
-            self.run_after_loop.set_upstream(self.runme_0)
-
     def test_bad_trigger_rule(self):
         with self.assertRaises(AirflowException):
             DummyOperator(
                 task_id='test_bad_trigger',
-                trigger_rule="non_existant",
+                trigger_rule="non_existent",
                 dag=self.dag)
 
     def test_terminate_task(self):
@@ -844,97 +935,25 @@ def test_task_fail_duration(self):
         session = settings.Session()
         try:
             p.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
-        except:
+        except Exception:
             pass
         try:
             f.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
-        except:
+        except Exception:
             pass
-        p_fails = session.query(models.TaskFail).filter_by(
+        p_fails = session.query(TaskFail).filter_by(
             task_id='pass_sleepy',
             dag_id=self.dag.dag_id,
             execution_date=DEFAULT_DATE).all()
-        f_fails = session.query(models.TaskFail).filter_by(
+        f_fails = session.query(TaskFail).filter_by(
             task_id='fail_sleepy',
             dag_id=self.dag.dag_id,
             execution_date=DEFAULT_DATE).all()
-        print(f_fails)
+
         self.assertEqual(0, len(p_fails))
         self.assertEqual(1, len(f_fails))
-        # C
         self.assertGreaterEqual(sum([f.duration for f in f_fails]), 3)
 
-    def test_dag_stats(self):
-        """Correctly sets/dirties/cleans rows of DagStat table"""
-
-        session = settings.Session()
-
-        session.query(models.DagRun).delete()
-        session.query(models.DagStat).delete()
-        session.commit()
-
-        models.DagStat.update([], session=session)
-
-        run1 = self.dag_bash.create_dagrun(
-            run_id="run1",
-            execution_date=DEFAULT_DATE,
-            state=State.RUNNING)
-
-        models.DagStat.update([self.dag_bash.dag_id], session=session)
-
-        qry = session.query(models.DagStat).all()
-
-        self.assertEqual(3, len(qry))
-        self.assertEqual(self.dag_bash.dag_id, qry[0].dag_id)
-        for stats in qry:
-            if stats.state == State.RUNNING:
-                self.assertEqual(stats.count, 1)
-            else:
-                self.assertEqual(stats.count, 0)
-            self.assertFalse(stats.dirty)
-
-        run2 = self.dag_bash.create_dagrun(
-            run_id="run2",
-            execution_date=DEFAULT_DATE + timedelta(days=1),
-            state=State.RUNNING)
-
-        models.DagStat.update([self.dag_bash.dag_id], session=session)
-
-        qry = session.query(models.DagStat).all()
-
-        self.assertEqual(3, len(qry))
-        self.assertEqual(self.dag_bash.dag_id, qry[0].dag_id)
-        for stats in qry:
-            if stats.state == State.RUNNING:
-                self.assertEqual(stats.count, 2)
-            else:
-                self.assertEqual(stats.count, 0)
-            self.assertFalse(stats.dirty)
-
-        session.query(models.DagRun).first().state = State.SUCCESS
-        session.commit()
-
-        models.DagStat.update([self.dag_bash.dag_id], session=session)
-
-        qry = session.query(models.DagStat).filter(models.DagStat.state == State.SUCCESS).all()
-        self.assertEqual(1, len(qry))
-        self.assertEqual(self.dag_bash.dag_id, qry[0].dag_id)
-        self.assertEqual(State.SUCCESS, qry[0].state)
-        self.assertEqual(1, qry[0].count)
-        self.assertFalse(qry[0].dirty)
-
-        qry = session.query(models.DagStat).filter(models.DagStat.state == State.RUNNING).all()
-        self.assertEqual(1, len(qry))
-        self.assertEqual(self.dag_bash.dag_id, qry[0].dag_id)
-        self.assertEqual(State.RUNNING, qry[0].state)
-        self.assertEqual(1, qry[0].count)
-        self.assertFalse(qry[0].dirty)
-
-        session.query(models.DagRun).delete()
-        session.query(models.DagStat).delete()
-        session.commit()
-        session.close()
-
     def test_run_command(self):
         if six.PY3:
             write = r'sys.stdout.buffer.write("\u1000foo".encode("utf8"))'
@@ -965,9 +984,77 @@ def payload_generator(context, object):
         task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
         dag_runs = models.DagRun.find(dag_id='example_bash_operator',
                                       run_id=run_id)
-        self.assertEquals(len(dag_runs), 1)
+        self.assertEqual(len(dag_runs), 1)
         dag_run = dag_runs[0]
-        self.assertEquals(dag_run.execution_date, utc_now)
+        self.assertEqual(dag_run.execution_date, utc_now)
+
+    def test_trigger_dagrun_with_str_execution_date(self):
+        utc_now_str = timezone.utcnow().isoformat()
+        self.assertIsInstance(utc_now_str, six.string_types)
+        run_id = 'trig__' + utc_now_str
+
+        def payload_generator(context, object):
+            object.run_id = run_id
+            return object
+
+        task = TriggerDagRunOperator(
+            task_id='test_trigger_dagrun_with_str_execution_date',
+            trigger_dag_id='example_bash_operator',
+            python_callable=payload_generator,
+            execution_date=utc_now_str,
+            dag=self.dag)
+        task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+        dag_runs = models.DagRun.find(dag_id='example_bash_operator',
+                                      run_id=run_id)
+        self.assertEqual(len(dag_runs), 1)
+        dag_run = dag_runs[0]
+        self.assertEqual(dag_run.execution_date.isoformat(), utc_now_str)
+
+    def test_trigger_dagrun_with_templated_execution_date(self):
+        task = TriggerDagRunOperator(
+            task_id='test_trigger_dagrun_with_str_execution_date',
+            trigger_dag_id='example_bash_operator',
+            execution_date='{{ execution_date }}',
+            dag=self.dag)
+
+        self.assertTrue(isinstance(task.execution_date, six.string_types))
+        self.assertEqual(task.execution_date, '{{ execution_date }}')
+
+        ti = TaskInstance(task=task, execution_date=DEFAULT_DATE)
+        ti.render_templates()
+        self.assertEqual(timezone.parse(task.execution_date), DEFAULT_DATE)
+
+    def test_externally_triggered_dagrun(self):
+        TI = models.TaskInstance
+
+        # Create the dagrun between two "scheduled" execution dates of the DAG
+        EXECUTION_DATE = DEFAULT_DATE + timedelta(days=2)
+        EXECUTION_DS = EXECUTION_DATE.strftime('%Y-%m-%d')
+        EXECUTION_DS_NODASH = EXECUTION_DS.replace('-', '')
+
+        dag = DAG(
+            TEST_DAG_ID,
+            default_args=self.args,
+            schedule_interval=timedelta(weeks=1),
+            start_date=DEFAULT_DATE)
+        task = DummyOperator(task_id='test_externally_triggered_dag_context',
+                             dag=dag)
+        dag.create_dagrun(run_id=models.DagRun.id_for_date(EXECUTION_DATE),
+                          execution_date=EXECUTION_DATE,
+                          state=State.RUNNING,
+                          external_trigger=True)
+        task.run(
+            start_date=EXECUTION_DATE, end_date=EXECUTION_DATE)
+
+        ti = TI(task=task, execution_date=EXECUTION_DATE)
+        context = ti.get_template_context()
+
+        # next_ds/prev_ds should be the execution date for manually triggered runs
+        self.assertEqual(context['next_ds'], EXECUTION_DS)
+        self.assertEqual(context['next_ds_nodash'], EXECUTION_DS_NODASH)
+
+        self.assertEqual(context['prev_ds'], EXECUTION_DS)
+        self.assertEqual(context['prev_ds_nodash'], EXECUTION_DS_NODASH)
 
 
 class CliTests(unittest.TestCase):
@@ -979,12 +1066,15 @@ def setUpClass(cls):
 
     def setUp(self):
         super(CliTests, self).setUp()
+        from airflow.www_rbac import app as application
         configuration.load_test_config()
-        app = application.create_app()
-        app.config['TESTING'] = True
+        self.app, self.appbuilder = application.create_app(session=Session, testing=True)
+        self.app.config['TESTING'] = True
+
         self.parser = cli.CLIFactory.get_parser()
         self.dagbag = models.DagBag(dag_folder=DEV_NULL, include_examples=True)
-        self.session = Session()
+        settings.configure_orm()
+        self.session = Session
 
     def tearDown(self):
         self._cleanup(session=self.session)
@@ -1004,6 +1094,14 @@ def test_cli_list_dags(self):
         args = self.parser.parse_args(['list_dags', '--report'])
         cli.list_dags(args)
 
+    def test_cli_list_dag_runs(self):
+        cli.trigger_dag(self.parser.parse_args([
+            'trigger_dag', 'example_bash_operator', ]))
+        args = self.parser.parse_args(['list_dag_runs',
+                                       'example_bash_operator',
+                                       '--no_backfill'])
+        cli.list_dag_runs(args)
+
     def test_cli_create_user_random_password(self):
         args = self.parser.parse_args([
             'create_user', '-u', 'test1', '-l', 'doe', '-f', 'jon',
@@ -1018,6 +1116,39 @@ def test_cli_create_user_supplied_password(self):
         ])
         cli.create_user(args)
 
+    def test_cli_delete_user(self):
+        args = self.parser.parse_args([
+            'create_user', '-u', 'test3', '-l', 'doe', '-f', 'jon',
+            '-e', 'jdoe@example.com', '-r', 'Viewer', '--use_random_password'
+        ])
+        cli.create_user(args)
+        args = self.parser.parse_args([
+            'delete_user', '-u', 'test3',
+        ])
+        cli.delete_user(args)
+
+    def test_cli_list_users(self):
+        for i in range(0, 3):
+            args = self.parser.parse_args([
+                'create_user', '-u', 'user{}'.format(i), '-l', 'doe', '-f', 'jon',
+                '-e', 'jdoe+{}@gmail.com'.format(i), '-r', 'Viewer',
+                '--use_random_password'
+            ])
+            cli.create_user(args)
+        with mock.patch('sys.stdout',
+                        new_callable=six.StringIO) as mock_stdout:
+            cli.list_users(self.parser.parse_args(['list_users']))
+            stdout = mock_stdout.getvalue()
+        for i in range(0, 3):
+            self.assertIn('user{}'.format(i), stdout)
+
+    def test_cli_sync_perm(self):
+        # test whether sync_perm cli will throw exceptions or not
+        args = self.parser.parse_args([
+            'sync_perm'
+        ])
+        cli.sync_perm(args)
+
     def test_cli_list_tasks(self):
         for dag_id in self.dagbag.dags.keys():
             args = self.parser.parse_args(['list_tasks', dag_id])
@@ -1027,13 +1158,13 @@ def test_cli_list_tasks(self):
             'list_tasks', 'example_bash_operator', '--tree'])
         cli.list_tasks(args)
 
-    @mock.patch("airflow.bin.cli.db_utils.initdb")
+    @mock.patch("airflow.bin.cli.db.initdb")
     def test_cli_initdb(self, initdb_mock):
         cli.initdb(self.parser.parse_args(['initdb']))
 
         initdb_mock.assert_called_once_with(False)
 
-    @mock.patch("airflow.bin.cli.db_utils.resetdb")
+    @mock.patch("airflow.bin.cli.db.resetdb")
     def test_cli_resetdb(self, resetdb_mock):
         cli.resetdb(self.parser.parse_args(['resetdb', '--yes']))
 
@@ -1044,7 +1175,7 @@ def test_cli_connections_list(self):
                         new_callable=six.StringIO) as mock_stdout:
             cli.connections(self.parser.parse_args(['connections', '--list']))
             stdout = mock_stdout.getvalue()
-        conns = [[x.strip("'") for x in re.findall("'\w+'", line)[:2]]
+        conns = [[x.strip("'") for x in re.findall(r"'\w+'", line)[:2]]
                  for ii, line in enumerate(stdout.split('\n'))
                  if ii % 2 == 1]
         conns = [conn for conn in conns if len(conn) > 0]
@@ -1183,8 +1314,8 @@ def test_cli_connections_add_delete(self):
         for index in range(1, 6):
             conn_id = 'new%s' % index
             result = (session
-                      .query(models.Connection)
-                      .filter(models.Connection.conn_id == conn_id)
+                      .query(Connection)
+                      .filter(Connection.conn_id == conn_id)
                       .first())
             result = (result.conn_id, result.conn_type, result.host,
                       result.port, result.get_extra())
@@ -1229,8 +1360,8 @@ def test_cli_connections_add_delete(self):
         # Check deletions
         for index in range(1, 7):
             conn_id = 'new%s' % index
-            result = (session.query(models.Connection)
-                      .filter(models.Connection.conn_id == conn_id)
+            result = (session.query(Connection)
+                      .filter(Connection.conn_id == conn_id)
                       .first())
 
             self.assertTrue(result is None)
@@ -1314,8 +1445,18 @@ def test_subdag_clear(self):
             'clear', 'example_subdag_operator', '--no_confirm', '--exclude_subdags'])
         cli.clear(args)
 
+    def test_parentdag_downstream_clear(self):
+        args = self.parser.parse_args([
+            'clear', 'example_subdag_operator.section-1', '--no_confirm'])
+        cli.clear(args)
+        args = self.parser.parse_args([
+            'clear', 'example_subdag_operator.section-1', '--no_confirm',
+            '--exclude_parentdag'])
+        cli.clear(args)
+
     def test_get_dags(self):
-        dags = cli.get_dags(self.parser.parse_args(['clear', 'example_subdag_operator', '-c']))
+        dags = cli.get_dags(self.parser.parse_args(['clear', 'example_subdag_operator',
+                                                    '-c']))
         self.assertEqual(len(dags), 1)
 
         dags = cli.get_dags(self.parser.parse_args(['clear', 'subdag', '-dx', '-c']))
@@ -1397,6 +1538,42 @@ def test_pool_no_args(self):
         except Exception as e:
             self.fail("The 'pool' command raised unexpectedly: %s" % e)
 
+    def test_pool_import_export(self):
+        # Create two pools first
+        pool_config_input = {
+            "foo": {
+                "description": "foo_test",
+                "slots": 1
+            },
+            "baz": {
+                "description": "baz_test",
+                "slots": 2
+            }
+        }
+        with open('pools_import.json', mode='w') as f:
+            json.dump(pool_config_input, f)
+
+        # Import json
+        try:
+            cli.pool(self.parser.parse_args(['pool', '-i', 'pools_import.json']))
+        except Exception as e:
+            self.fail("The 'pool -i pools_import.json' failed: %s" % e)
+
+        # Export json
+        try:
+            cli.pool(self.parser.parse_args(['pool', '-e', 'pools_export.json']))
+        except Exception as e:
+            self.fail("The 'pool -e pools_export.json' failed: %s" % e)
+
+        with open('pools_export.json', mode='r') as f:
+            pool_config_output = json.load(f)
+            self.assertEqual(
+                pool_config_input,
+                pool_config_output,
+                "Input and output pool files are not same")
+        os.remove('pools_import.json')
+        os.remove('pools_export.json')
+
     def test_variables(self):
         # Checks if all subcommands are properly received
         cli.variables(self.parser.parse_args([
@@ -1457,7 +1634,7 @@ def _wait_pidfile(self, pidfile):
             try:
                 with open(pidfile) as f:
                     return int(f.read())
-            except:
+            except Exception:
                 sleep(1)
 
     def test_cli_webserver_foreground(self):
@@ -1544,10 +1721,8 @@ def setUp(self):
         self.runme_0 = self.dag_bash.get_task('runme_0')
 
     def get_csrf(self, response):
-        tree = html.fromstring(response.data)
-        form = tree.find('.//form')
-
-        return form.find('.//input[@name="_csrf_token"]').value
+        tree = BeautifulSoup(response.data, 'html.parser')
+        return tree.find('input', attrs={'name': '_csrf_token'})['value']
 
     def test_csrf_rejection(self):
         endpoints = ([
@@ -1567,11 +1742,11 @@ def test_csrf_acceptance(self):
     def test_xss(self):
         try:
             self.app.get("/admin/airflow/tree?dag_id=<script>alert(123456)</script>")
-        except:
+        except Exception:
             # exception is expected here since dag doesnt exist
             pass
         response = self.app.get("/admin/log", follow_redirects=True)
-        self.assertIn(bleach.clean("<script>alert(123456)</script>"), response.data.decode('UTF-8'))
+        self.assertNotIn("<script>alert(123456)</script>", response.data.decode('UTF-8'))
 
     def test_chart_data_template(self):
         """Protect chart_data from being able to do RCE."""
@@ -1639,21 +1814,21 @@ def setUp(self):
 
         self.dagrun_python = self.dag_python.create_dagrun(
             run_id="test_{}".format(models.DagRun.id_for_date(timezone.utcnow())),
-            execution_date=DEFAULT_DATE,
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
             start_date=timezone.utcnow(),
             state=State.RUNNING
         )
 
         self.sub_dag.create_dagrun(
             run_id="test_{}".format(models.DagRun.id_for_date(timezone.utcnow())),
-            execution_date=DEFAULT_DATE,
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
             start_date=timezone.utcnow(),
             state=State.RUNNING
         )
 
         self.example_xcom.create_dagrun(
             run_id="test_{}".format(models.DagRun.id_for_date(timezone.utcnow())),
-            execution_date=DEFAULT_DATE,
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
             start_date=timezone.utcnow(),
             state=State.RUNNING
         )
@@ -1685,8 +1860,69 @@ def test_query(self):
         self.assertIn("TEST", response.data.decode('utf-8'))
 
     def test_health(self):
-        response = self.app.get('/health')
-        self.assertIn('The server is healthy!', response.data.decode('utf-8'))
+        BJ = jobs.BaseJob
+        session = Session()
+
+        # case-1: healthy scheduler status
+        last_scheduler_heartbeat_for_testing_1 = timezone.utcnow()
+        session.add(BJ(job_type='SchedulerJob',
+                       state='running',
+                       latest_heartbeat=last_scheduler_heartbeat_for_testing_1))
+        session.commit()
+
+        response_json = json.loads(self.app.get('/health').data.decode('utf-8'))
+
+        self.assertEqual('healthy', response_json['metadatabase']['status'])
+        self.assertEqual('healthy', response_json['scheduler']['status'])
+        self.assertEqual(str(last_scheduler_heartbeat_for_testing_1),
+                         response_json['scheduler']['latest_scheduler_heartbeat'])
+
+        session.query(BJ).\
+            filter(BJ.job_type == 'SchedulerJob',
+                   BJ.state == 'running',
+                   BJ.latest_heartbeat == last_scheduler_heartbeat_for_testing_1).\
+            delete()
+        session.commit()
+
+        # case-2: unhealthy scheduler status - scenario 1 (SchedulerJob is running too slowly)
+        last_scheduler_heartbeat_for_testing_2 = timezone.utcnow() - timedelta(minutes=1)
+        (session.query(BJ)
+                .filter(BJ.job_type == 'SchedulerJob')
+                .update({'latest_heartbeat': last_scheduler_heartbeat_for_testing_2 - timedelta(seconds=1)}))
+        session.add(BJ(job_type='SchedulerJob',
+                       state='running',
+                       latest_heartbeat=last_scheduler_heartbeat_for_testing_2))
+        session.commit()
+
+        response_json = json.loads(self.app.get('/health').data.decode('utf-8'))
+
+        self.assertEqual('healthy', response_json['metadatabase']['status'])
+        self.assertEqual('unhealthy', response_json['scheduler']['status'])
+        self.assertEqual(str(last_scheduler_heartbeat_for_testing_2),
+                         response_json['scheduler']['latest_scheduler_heartbeat'])
+
+        session.query(BJ).\
+            filter(BJ.job_type == 'SchedulerJob',
+                   BJ.state == 'running',
+                   BJ.latest_heartbeat == last_scheduler_heartbeat_for_testing_1).\
+            delete()
+        session.commit()
+
+        # case-3: unhealthy scheduler status - scenario 2 (no running SchedulerJob)
+        session.query(BJ).\
+            filter(BJ.job_type == 'SchedulerJob',
+                   BJ.state == 'running').\
+            delete()
+        session.commit()
+
+        response_json = json.loads(self.app.get('/health').data.decode('utf-8'))
+
+        self.assertEqual('healthy', response_json['metadatabase']['status'])
+        self.assertEqual('unhealthy', response_json['scheduler']['status'])
+        self.assertEqual('None',
+                         response_json['scheduler']['latest_scheduler_heartbeat'])
+
+        session.close()
 
     def test_noaccess(self):
         response = self.app.get('/admin/airflow/noaccess')
@@ -1710,6 +1946,10 @@ def test_dag_views(self):
         response = self.app.get(
             '/admin/airflow/duration?days=30&dag_id=example_bash_operator')
         self.assertIn("example_bash_operator", response.data.decode('utf-8'))
+        response = self.app.get(
+            '/admin/airflow/duration?days=30&dag_id=missing_dag',
+            follow_redirects=True)
+        self.assertIn("seems to be missing", response.data.decode('utf-8'))
         response = self.app.get(
             '/admin/airflow/tries?days=30&dag_id=example_bash_operator')
         self.assertIn("example_bash_operator", response.data.decode('utf-8'))
@@ -1746,7 +1986,7 @@ def test_dag_views(self):
         response = self.app.get(
             '/admin/airflow/task?'
             'task_id=runme_0&dag_id=example_bash_operator&'
-            'execution_date={}'.format(DEFAULT_DATE_DS))
+            'execution_date={}'.format(EXAMPLE_DAG_DEFAULT_DATE))
         self.assertIn("Attributes", response.data.decode('utf-8'))
         response = self.app.get(
             '/admin/airflow/dag_stats')
@@ -1754,48 +1994,101 @@ def test_dag_views(self):
         response = self.app.get(
             '/admin/airflow/task_stats')
         self.assertIn("example_bash_operator", response.data.decode('utf-8'))
-        url = (
-            "/admin/airflow/success?task_id=print_the_context&"
-            "dag_id=example_python_operator&upstream=false&downstream=false&"
-            "future=false&past=false&execution_date={}&"
-            "origin=/admin".format(DEFAULT_DATE_DS))
-        response = self.app.get(url)
+
+        response = self.app.post("/admin/airflow/success", data=dict(
+            task_id="print_the_context",
+            dag_id="example_python_operator",
+            upstream="false",
+            downstream="false",
+            future="false",
+            past="false",
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
+            origin="/admin"))
         self.assertIn("Wait a minute", response.data.decode('utf-8'))
-        response = self.app.get(url + "&confirmed=true")
-        response = self.app.get(
-            '/admin/airflow/clear?task_id=print_the_context&'
-            'dag_id=example_python_operator&future=true&past=false&'
-            'upstream=true&downstream=false&'
-            'execution_date={}&'
-            'origin=/admin'.format(DEFAULT_DATE_DS))
+
+        response = self.app.post('/admin/airflow/clear', data=dict(
+            task_id="print_the_context",
+            dag_id="example_python_operator",
+            future="true",
+            past="false",
+            upstream="true",
+            downstream="false",
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
+            origin="/admin"))
         self.assertIn("Wait a minute", response.data.decode('utf-8'))
-        url = (
-            "/admin/airflow/success?task_id=section-1&"
-            "dag_id=example_subdag_operator&upstream=true&downstream=true&"
-            "future=false&past=false&execution_date={}&"
-            "origin=/admin".format(DEFAULT_DATE_DS))
-        response = self.app.get(url)
+
+        form = dict(
+            task_id="section-1",
+            dag_id="example_subdag_operator",
+            upstream="true",
+            downstream="true",
+            future="false",
+            past="false",
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
+            origin="/admin")
+        response = self.app.post("/admin/airflow/success", data=form)
         self.assertIn("Wait a minute", response.data.decode('utf-8'))
         self.assertIn("section-1-task-1", response.data.decode('utf-8'))
         self.assertIn("section-1-task-2", response.data.decode('utf-8'))
         self.assertIn("section-1-task-3", response.data.decode('utf-8'))
         self.assertIn("section-1-task-4", response.data.decode('utf-8'))
         self.assertIn("section-1-task-5", response.data.decode('utf-8'))
-        response = self.app.get(url + "&confirmed=true")
-        url = (
-            "/admin/airflow/clear?task_id=print_the_context&"
-            "dag_id=example_python_operator&future=false&past=false&"
-            "upstream=false&downstream=true&"
-            "execution_date={}&"
-            "origin=/admin".format(DEFAULT_DATE_DS))
-        response = self.app.get(url)
+        form["confirmed"] = "true"
+        response = self.app.post("/admin/airflow/success", data=form)
+        self.assertEqual(response.status_code, 302)
+
+        form = dict(
+            task_id="print_the_context",
+            dag_id="example_python_operator",
+            future="false",
+            past="false",
+            upstream="false",
+            downstream="true",
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
+            origin="/admin")
+        response = self.app.post("/admin/airflow/clear", data=form)
         self.assertIn("Wait a minute", response.data.decode('utf-8'))
-        response = self.app.get(url + "&confirmed=true")
+        form["confirmed"] = "true"
+        response = self.app.post("/admin/airflow/clear", data=form)
+        self.assertEqual(response.status_code, 302)
+
+        form = dict(
+            task_id="section-1-task-1",
+            dag_id="example_subdag_operator.section-1",
+            future="false",
+            past="false",
+            upstream="false",
+            downstream="true",
+            recursive="true",
+            execution_date=EXAMPLE_DAG_DEFAULT_DATE,
+            origin="/admin")
+        response = self.app.post("/admin/airflow/clear", data=form)
+        self.assertIn("Wait a minute", response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.end",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-1.section-1-task-1",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-1",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-2",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-2.section-2-task-1",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-2.section-2-task-2",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-2.section-2-task-3",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-2.section-2-task-4",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.section-2.section-2-task-5",
+                      response.data.decode('utf-8'))
+        self.assertIn("example_subdag_operator.some-other-task",
+                      response.data.decode('utf-8'))
         url = (
             "/admin/airflow/run?task_id=runme_0&"
             "dag_id=example_bash_operator&ignore_all_deps=false&ignore_ti_state=true&"
             "ignore_task_deps=true&execution_date={}&"
-            "origin=/admin".format(DEFAULT_DATE_DS))
+            "origin=/admin".format(EXAMPLE_DAG_DEFAULT_DATE))
         response = self.app.get(url)
         response = self.app.get(
             "/admin/airflow/refresh?dag_id=example_bash_operator")
@@ -1830,13 +2123,28 @@ def test_fetch_task_instance(self):
         url = (
             "/admin/airflow/object/task_instances?"
             "dag_id=example_python_operator&"
-            "execution_date={}".format(DEFAULT_DATE_DS))
+            "execution_date={}".format(EXAMPLE_DAG_DEFAULT_DATE))
         response = self.app.get(url)
         self.assertIn("print_the_context", response.data.decode('utf-8'))
 
+    def test_dag_view_task_with_python_operator_using_partial(self):
+        response = self.app.get(
+            '/admin/airflow/task?'
+            'task_id=test_dagrun_functool_partial&dag_id=test_task_view_type_check&'
+            'execution_date={}'.format(EXAMPLE_DAG_DEFAULT_DATE))
+        self.assertIn("A function with two args", response.data.decode('utf-8'))
+
+    def test_dag_view_task_with_python_operator_using_instance(self):
+        response = self.app.get(
+            '/admin/airflow/task?'
+            'task_id=test_dagrun_instance&dag_id=test_task_view_type_check&'
+            'execution_date={}'.format(EXAMPLE_DAG_DEFAULT_DATE))
+        self.assertIn("A __call__ method", response.data.decode('utf-8'))
+
     def tearDown(self):
         configuration.conf.set("webserver", "expose_config", "False")
-        self.dag_bash.clear(start_date=DEFAULT_DATE, end_date=timezone.utcnow())
+        self.dag_bash.clear(start_date=EXAMPLE_DAG_DEFAULT_DATE,
+                            end_date=timezone.utcnow())
         session = Session()
         session.query(models.DagRun).delete()
         session.query(models.TaskInstance).delete()
@@ -1889,6 +2197,11 @@ def test_password_user_authenticate(self):
         self.password_user.password = "secure_password"
         self.assertTrue(self.password_user.authenticate("secure_password"))
 
+    def test_password_unicode_user_authenticate(self):
+        self.password_user.username = u"🐼"  # This is a panda
+        self.password_user.password = "secure_password"
+        self.assertTrue(self.password_user.authenticate("secure_password"))
+
     def test_password_authenticate_session(self):
         from airflow.contrib.auth.backends.password_auth import PasswordUser
         self.password_user.password = 'test_password'
@@ -1924,10 +2237,8 @@ def setUp(self):
         session.close()
 
     def get_csrf(self, response):
-        tree = html.fromstring(response.data)
-        form = tree.find('.//form')
-
-        return form.find('.//input[@name="_csrf_token"]').value
+        tree = BeautifulSoup(response.data, 'html.parser')
+        return tree.find('input', attrs={'name': '_csrf_token'})['value']
 
     def login(self, username, password):
         response = self.app.get('/admin/airflow/login')
@@ -1976,9 +2287,9 @@ def setUp(self):
         configuration.conf.set("webserver", "auth_backend", "airflow.contrib.auth.backends.ldap_auth")
         try:
             configuration.conf.add_section("ldap")
-        except:
+        except Exception:
             pass
-        configuration.conf.set("ldap", "uri", "ldap://localhost:3890")
+        configuration.conf.set("ldap", "uri", "ldap://openldap:389")
         configuration.conf.set("ldap", "user_filter", "objectClass=*")
         configuration.conf.set("ldap", "user_name_attr", "uid")
         configuration.conf.set("ldap", "bind_user", "cn=Manager,dc=example,dc=com")
@@ -1991,10 +2302,8 @@ def setUp(self):
         self.app = app.test_client()
 
     def get_csrf(self, response):
-        tree = html.fromstring(response.data)
-        form = tree.find('.//form')
-
-        return form.find('.//input[@name="_csrf_token"]').value
+        tree = BeautifulSoup(response.data, 'html.parser')
+        return tree.find('input', attrs={'name': '_csrf_token'})['value']
 
     def login(self, username, password):
         response = self.app.get('/admin/airflow/login')
@@ -2063,9 +2372,9 @@ def setUp(self):
         configuration.conf.set("webserver", "auth_backend", "airflow.contrib.auth.backends.ldap_auth")
         try:
             configuration.conf.add_section("ldap")
-        except:
+        except Exception:
             pass
-        configuration.conf.set("ldap", "uri", "ldap://localhost:3890")
+        configuration.conf.set("ldap", "uri", "ldap://openldap:389")
         configuration.conf.set("ldap", "user_filter", "objectClass=*")
         configuration.conf.set("ldap", "user_name_attr", "uid")
         configuration.conf.set("ldap", "bind_user", "cn=Manager,dc=example,dc=com")
@@ -2284,9 +2593,9 @@ def test_using_unix_socket_env_var(self):
         self.assertIsNone(c.port)
 
     def test_param_setup(self):
-        c = models.Connection(conn_id='local_mysql', conn_type='mysql',
-                              host='localhost', login='airflow',
-                              password='airflow', schema='airflow')
+        c = Connection(conn_id='local_mysql', conn_type='mysql',
+                       host='localhost', login='airflow',
+                       password='airflow', schema='airflow')
         self.assertEqual('localhost', c.host)
         self.assertEqual('airflow', c.schema)
         self.assertEqual('airflow', c.login)
@@ -2331,13 +2640,6 @@ def test_get_connections_env_var(self):
         assert conns[0].password == 'password'
         assert conns[0].port == 5432
 
-    def test_get_connections_db(self):
-        conns = BaseHook.get_connections(conn_id='airflow_db')
-        assert len(conns) == 1
-        assert conns[0].host == 'localhost'
-        assert conns[0].schema == 'airflow'
-        assert conns[0].login == 'root'
-
 
 class WebHDFSHookTest(unittest.TestCase):
     def setUp(self):
@@ -2365,7 +2667,7 @@ def test_init_proxy_user(self):
 class HDFSHookTest(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
-        os.environ['AIRFLOW_CONN_HDFS_DEFAULT'] = ('hdfs://localhost:8020')
+        os.environ['AIRFLOW_CONN_HDFS_DEFAULT'] = 'hdfs://localhost:8020'
 
     def test_get_client(self):
         client = HDFSHook(proxy_user='foo').get_conn()
@@ -2378,9 +2680,9 @@ def test_get_client(self):
     @mock.patch('airflow.hooks.hdfs_hook.HDFSHook.get_connections')
     def test_get_autoconfig_client(self, mock_get_connections,
                                    MockAutoConfigClient):
-        c = models.Connection(conn_id='hdfs', conn_type='hdfs',
-                              host='localhost', port=8020, login='foo',
-                              extra=json.dumps({'autoconfig': True}))
+        c = Connection(conn_id='hdfs', conn_type='hdfs',
+                       host='localhost', port=8020, login='foo',
+                       extra=json.dumps({'autoconfig': True}))
         mock_get_connections.return_value = [c]
         HDFSHook(hdfs_conn_id='hdfs').get_conn()
         MockAutoConfigClient.assert_called_once_with(effective_user='foo',
@@ -2394,14 +2696,15 @@ def test_get_autoconfig_client_no_conn(self, MockAutoConfigClient):
 
     @mock.patch('airflow.hooks.hdfs_hook.HDFSHook.get_connections')
     def test_get_ha_client(self, mock_get_connections):
-        c1 = models.Connection(conn_id='hdfs_default', conn_type='hdfs',
-                               host='localhost', port=8020)
-        c2 = models.Connection(conn_id='hdfs_default', conn_type='hdfs',
-                               host='localhost2', port=8020)
+        c1 = Connection(conn_id='hdfs_default', conn_type='hdfs',
+                        host='localhost', port=8020)
+        c2 = Connection(conn_id='hdfs_default', conn_type='hdfs',
+                        host='localhost2', port=8020)
         mock_get_connections.return_value = [c1, c2]
         client = HDFSHook().get_conn()
         self.assertIsInstance(client, snakebite.client.HAClient)
 
+
 send_email_test = mock.Mock()
 
 
diff --git a/tests/dags/.airflowignore b/tests/dags/.airflowignore
new file mode 100644
index 0000000000000..93361c17e24d8
--- /dev/null
+++ b/tests/dags/.airflowignore
@@ -0,0 +1 @@
+.*_invalid.*
diff --git a/tests/dags/.gitignore b/tests/dags/.gitignore
new file mode 100644
index 0000000000000..30db57ad4bacb
--- /dev/null
+++ b/tests/dags/.gitignore
@@ -0,0 +1,4 @@
+# This line is to avoid accidental commits of example dags for integration testing
+# In order to test example dags easily we often create symbolic links in this directory
+# this line prevents accidental commiting of such symbolic links
+example_*
diff --git a/tests/dags/README.md b/tests/dags/README.md
index c5d6acb0f4c2b..615e8ab7f6968 100644
--- a/tests/dags/README.md
+++ b/tests/dags/README.md
@@ -1,3 +1,22 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Unit Tests DAGs Folder
 
 This folder contains DAGs for Airflow unit testing.
diff --git a/tests/dags/no_dags.py b/tests/dags/no_dags.py
index db5ba598d7c23..b7f8352944d3f 100644
--- a/tests/dags/no_dags.py
+++ b/tests/dags/no_dags.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_backfill_pooled_tasks.py b/tests/dags/test_backfill_pooled_tasks.py
index 4b4f2a8661e14..8320f51c86e76 100644
--- a/tests/dags/test_backfill_pooled_tasks.py
+++ b/tests/dags/test_backfill_pooled_tasks.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_cli_triggered_dags.py b/tests/dags/test_cli_triggered_dags.py
index b0683be80f57a..9f53ca4c3ab0b 100644
--- a/tests/dags/test_cli_triggered_dags.py
+++ b/tests/dags/test_cli_triggered_dags.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -42,14 +42,15 @@ def success(ti=None, *args, **kwargs):
 
 # DAG tests that tasks ignore all dependencies
 
-dag1 = DAG(dag_id='test_run_ignores_all_dependencies', default_args=dict(depends_on_past=True, **default_args))
+dag1 = DAG(dag_id='test_run_ignores_all_dependencies',
+           default_args=dict(depends_on_past=True, **default_args))
 dag1_task1 = PythonOperator(
     task_id='test_run_dependency_task',
     python_callable=fail,
-    dag=dag1,)
+    dag=dag1)
 dag1_task2 = PythonOperator(
     task_id='test_run_dependent_task',
     python_callable=success,
     provide_context=True,
-    dag=dag1,)
+    dag=dag1)
 dag1_task1.set_downstream(dag1_task2)
diff --git a/tests/dags/test_default_impersonation.py b/tests/dags/test_default_impersonation.py
index 42b7198bd6ca1..537fee9a7cb93 100644
--- a/tests/dags/test_default_impersonation.py
+++ b/tests/dags/test_default_impersonation.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_double_trigger.py b/tests/dags/test_double_trigger.py
index eea0aa1ddc69f..a53d7638dfefc 100644
--- a/tests/dags/test_double_trigger.py
+++ b/tests/dags/test_double_trigger.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_example_bash_operator.py b/tests/dags/test_example_bash_operator.py
index 94b89ffa52cd2..a87db8dd7ccd7 100644
--- a/tests/dags/test_example_bash_operator.py
+++ b/tests/dags/test_example_bash_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -27,6 +27,7 @@
 
 args = {
     'owner': 'airflow',
+    'retries': 3,
     'start_date': airflow.utils.dates.days_ago(2)
 }
 
@@ -45,7 +46,7 @@
 for i in range(3):
     i = str(i)
     task = BashOperator(
-        task_id='runme_'+i,
+        task_id='runme_' + i,
         bash_command='echo "{{ task_instance_key_str }}" && sleep 1',
         dag=dag)
     task.set_downstream(run_this)
diff --git a/tests/dags/test_impersonation.py b/tests/dags/test_impersonation.py
index 4e554e146d7c6..bc327d2953844 100644
--- a/tests/dags/test_impersonation.py
+++ b/tests/dags/test_impersonation.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_impersonation_custom.py b/tests/dags/test_impersonation_custom.py
index 43a52355c4095..1157812abca02 100644
--- a/tests/dags/test_impersonation_custom.py
+++ b/tests/dags/test_impersonation_custom.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_impersonation_subdag.py b/tests/dags/test_impersonation_subdag.py
index 423c55266485b..01275c67c0264 100644
--- a/tests/dags/test_impersonation_subdag.py
+++ b/tests/dags/test_impersonation_subdag.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_invalid_cron.py b/tests/dags/test_invalid_cron.py
new file mode 100755
index 0000000000000..51a0e43cb500a
--- /dev/null
+++ b/tests/dags/test_invalid_cron.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.models import DAG
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.utils.timezone import datetime
+
+# The schedule_interval specified here is an INVALID
+# Cron expression. This invalid DAG will be used to
+# test whether dagbag.process_file() can identify
+# invalid Cron expression.
+dag1 = DAG(
+    dag_id='test_invalid_cron',
+    start_date=datetime(2015, 1, 1),
+    schedule_interval="0 100 * * *")
+dag1_task1 = DummyOperator(
+    task_id='task1',
+    dag=dag1,
+    owner='airflow')
diff --git a/tests/dags/test_issue_1225.py b/tests/dags/test_issue_1225.py
index b7bd5d5a24e7b..f9dea3794facb 100644
--- a/tests/dags/test_issue_1225.py
+++ b/tests/dags/test_issue_1225.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,7 +23,7 @@
 by a BackfillJob.
 Addresses issue #1225.
 """
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from airflow.models import DAG
 from airflow.operators.dummy_operator import DummyOperator
@@ -37,9 +37,11 @@
     start_date=DEFAULT_DATE,
     owner='airflow')
 
+
 def fail():
     raise ValueError('Expected failure.')
 
+
 def delayed_fail():
     """
     Delayed failure to make sure that processes are running before the error
@@ -50,6 +52,7 @@ def delayed_fail():
     time.sleep(5)
     raise ValueError('Expected failure.')
 
+
 # DAG tests backfill with pooled tasks
 # Previously backfill would queue the task but never run it
 dag1 = DAG(dag_id='test_backfill_pooled_task_dag', default_args=default_args)
@@ -147,3 +150,15 @@ def delayed_fail():
     dag=dag8,
     python_callable=fail,
 )
+
+# DAG tests that a Dag run that completes but has a root in the future is marked as success
+dag9 = DAG(dag_id='test_dagrun_states_root_future', default_args=default_args)
+dag9_task1 = DummyOperator(
+    task_id='current',
+    dag=dag9,
+)
+dag8_task2 = DummyOperator(
+    task_id='future',
+    dag=dag9,
+    start_date=DEFAULT_DATE + timedelta(days=1),
+)
diff --git a/tests/dags/test_latest_runs.py b/tests/dags/test_latest_runs.py
index a39a6bd948e25..e8cd99adc907e 100644
--- a/tests/dags/test_latest_runs.py
+++ b/tests/dags/test_latest_runs.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_mark_success.py b/tests/dags/test_mark_success.py
index 50ba32d2e3d73..dfbcbc1b5d16b 100644
--- a/tests/dags/test_mark_success.py
+++ b/tests/dags/test_mark_success.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_no_impersonation.py b/tests/dags/test_no_impersonation.py
index cb5bb66a6124c..26cb935d8679e 100644
--- a/tests/dags/test_no_impersonation.py
+++ b/tests/dags/test_no_impersonation.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_on_kill.py b/tests/dags/test_on_kill.py
index 353b4573921c3..d9307f150843b 100644
--- a/tests/dags/test_on_kill.py
+++ b/tests/dags/test_on_kill.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags/test_retry_handling_job.py b/tests/dags/test_retry_handling_job.py
index aae072e0bf4b5..d8e314dbabde5 100644
--- a/tests/dags/test_retry_handling_job.py
+++ b/tests/dags/test_retry_handling_job.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,7 @@
 default_args = {
     'owner': 'airflow',
     'depends_on_past': False,
-    'start_date': datetime(2016,10,5,19),
+    'start_date': datetime(2016, 10, 5, 19),
     'email': ['airflow@example.com'],
     'email_on_failure': False,
     'email_on_retry': False,
@@ -38,4 +38,3 @@
     task_id='test_retry_handling_op',
     bash_command='exit 1',
     dag=dag)
-
diff --git a/tests/dags/test_scheduler_dags.py b/tests/dags/test_scheduler_dags.py
index 990086693771a..94e6f8216d226 100644
--- a/tests/dags/test_scheduler_dags.py
+++ b/tests/dags/test_scheduler_dags.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,18 +17,34 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from airflow.models import DAG
 from airflow.operators.dummy_operator import DummyOperator
-DEFAULT_DATE = datetime(2100, 1, 1)
+DEFAULT_DATE = datetime(2016, 1, 1)
 
 # DAG tests backfill with pooled tasks
 # Previously backfill would queue the task but never run it
 dag1 = DAG(
     dag_id='test_start_date_scheduling',
-    start_date=datetime(2100, 1, 1))
+    start_date=datetime.utcnow() + timedelta(days=1))
 dag1_task1 = DummyOperator(
     task_id='dummy',
     dag=dag1,
     owner='airflow')
+
+dag2 = DAG(
+    dag_id='test_task_start_date_scheduling',
+    start_date=DEFAULT_DATE
+)
+dag2_task1 = DummyOperator(
+    task_id='dummy1',
+    dag=dag2,
+    owner='airflow',
+    start_date=DEFAULT_DATE + timedelta(days=3)
+)
+dag2_task2 = DummyOperator(
+    task_id='dummy2',
+    dag=dag2,
+    owner='airflow'
+)
diff --git a/tests/dags/test_task_view_type_check.py b/tests/dags/test_task_view_type_check.py
new file mode 100644
index 0000000000000..a03db8a65cec3
--- /dev/null
+++ b/tests/dags/test_task_view_type_check.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+"""
+DAG designed to test a PythonOperator that calls a functool.partial
+"""
+import functools
+import logging
+
+from datetime import datetime
+
+from airflow.models import DAG
+from airflow.operators.python_operator import PythonOperator
+
+DEFAULT_DATE = datetime(2016, 1, 1)
+default_args = dict(
+    start_date=DEFAULT_DATE,
+    owner='airflow')
+
+
+class CallableClass(object):
+    def __call__(self):
+        """ A __call__ method """
+        pass
+
+
+def a_function(arg_x, arg_y):
+    """ A function with two args """
+    pass
+
+
+partial_function = functools.partial(a_function, arg_x=1)
+class_instance = CallableClass()
+
+logging.info('class_instance type: {}'.format(type(class_instance)))
+
+dag = DAG(dag_id='test_task_view_type_check', default_args=default_args)
+
+dag_task1 = PythonOperator(
+    task_id='test_dagrun_functool_partial',
+    dag=dag,
+    python_callable=partial_function,
+)
+
+dag_task2 = PythonOperator(
+    task_id='test_dagrun_instance',
+    dag=dag,
+    python_callable=class_instance,
+)
diff --git a/tests/dags/test_zip_invalid_cron.zip b/tests/dags/test_zip_invalid_cron.zip
new file mode 100644
index 0000000000000..fe45153abe85a
Binary files /dev/null and b/tests/dags/test_zip_invalid_cron.zip differ
diff --git a/tests/dags_with_system_exit/a_system_exit.py b/tests/dags_with_system_exit/a_system_exit.py
index 42d96a3d1c9be..8d51ddcbb833e 100644
--- a/tests/dags_with_system_exit/a_system_exit.py
+++ b/tests/dags_with_system_exit/a_system_exit.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags_with_system_exit/b_test_scheduler_dags.py b/tests/dags_with_system_exit/b_test_scheduler_dags.py
index 2d925cd79e172..bde36ba5dfecf 100644
--- a/tests/dags_with_system_exit/b_test_scheduler_dags.py
+++ b/tests/dags_with_system_exit/b_test_scheduler_dags.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/dags_with_system_exit/c_system_exit.py b/tests/dags_with_system_exit/c_system_exit.py
index 2581dfecebeea..bebaa3e8f3945 100644
--- a/tests/dags_with_system_exit/c_system_exit.py
+++ b/tests/dags_with_system_exit/c_system_exit.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/executors/__init__.py b/tests/executors/__init__.py
index 4e16cd0a8e1f1..114d189da14ab 100644
--- a/tests/executors/__init__.py
+++ b/tests/executors/__init__.py
@@ -7,14 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from .dask_executor import *
diff --git a/tests/executors/test_base_executor.py b/tests/executors/test_base_executor.py
index f349355d32efa..ae7c76fabd8ab 100644
--- a/tests/executors/test_base_executor.py
+++ b/tests/executors/test_base_executor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -30,10 +30,10 @@ def test_get_event_buffer(self):
         executor = BaseExecutor()
 
         date = datetime.utcnow()
-
-        key1 = ("my_dag1", "my_task1", date)
-        key2 = ("my_dag2", "my_task1", date)
-        key3 = ("my_dag2", "my_task2", date)
+        try_number = 1
+        key1 = ("my_dag1", "my_task1", date, try_number)
+        key2 = ("my_dag2", "my_task1", date, try_number)
+        key3 = ("my_dag2", "my_task2", date, try_number)
         state = State.SUCCESS
         executor.event_buffer[key1] = state
         executor.event_buffer[key2] = state
@@ -42,4 +42,3 @@ def test_get_event_buffer(self):
         self.assertEqual(len(executor.get_event_buffer(("my_dag1",))), 1)
         self.assertEqual(len(executor.get_event_buffer()), 2)
         self.assertEqual(len(executor.event_buffer), 0)
-
diff --git a/tests/executors/test_celery_executor.py b/tests/executors/test_celery_executor.py
index 1169c8d11b95c..821f265f1070e 100644
--- a/tests/executors/test_celery_executor.py
+++ b/tests/executors/test_celery_executor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,30 +18,63 @@
 # under the License.
 import sys
 import unittest
+from multiprocessing import Pool
+
+import mock
 from celery.contrib.testing.worker import start_worker
 
-from airflow.executors.celery_executor import CeleryExecutor
+from airflow.executors import celery_executor
+from airflow.executors.celery_executor import (CeleryExecutor, celery_configuration,
+                                               send_task_to_executor, execute_command)
 from airflow.executors.celery_executor import app
+from celery import states as celery_states
 from airflow.utils.state import State
 
+from airflow import configuration
+configuration.load_test_config()
+
 # leave this it is used by the test worker
-import celery.contrib.testing.tasks
+import celery.contrib.testing.tasks  # noqa: F401
+
 
 class CeleryExecutorTest(unittest.TestCase):
+    @unittest.skipIf('sqlite' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                     "sqlite is configured with SequentialExecutor")
     def test_celery_integration(self):
         executor = CeleryExecutor()
         executor.start()
         with start_worker(app=app, logfile=sys.stdout, loglevel='debug'):
+            success_command = ['true', 'some_parameter']
+            fail_command = ['false', 'some_parameter']
+
+            cached_celery_backend = execute_command.backend
+            task_tuples_to_send = [('success', 'fake_simple_ti', success_command,
+                                    celery_configuration['task_default_queue'],
+                                    execute_command),
+                                   ('fail', 'fake_simple_ti', fail_command,
+                                    celery_configuration['task_default_queue'],
+                                    execute_command)]
+
+            chunksize = executor._num_tasks_per_send_process(len(task_tuples_to_send))
+            num_processes = min(len(task_tuples_to_send), executor._sync_parallelism)
+
+            send_pool = Pool(processes=num_processes)
+            key_and_async_results = send_pool.map(
+                send_task_to_executor,
+                task_tuples_to_send,
+                chunksize=chunksize)
+
+            send_pool.close()
+            send_pool.join()
 
-            success_command = 'echo 1'
-            fail_command = 'exit 1'
+            for key, command, result in key_and_async_results:
+                # Only pops when enqueued successfully, otherwise keep it
+                # and expect scheduler loop to deal with it.
+                result.backend = cached_celery_backend
+                executor.running[key] = command
+                executor.tasks[key] = result
+                executor.last_state[key] = celery_states.PENDING
 
-            executor.execute_async(key='success', command=success_command)
-            # errors are propagated for some reason
-            try:
-                executor.execute_async(key='fail', command=fail_command)
-            except:
-                pass
             executor.running['success'] = True
             executor.running['fail'] = True
 
@@ -53,6 +86,45 @@ def test_celery_integration(self):
         self.assertNotIn('success', executor.tasks)
         self.assertNotIn('fail', executor.tasks)
 
+        self.assertNotIn('success', executor.last_state)
+        self.assertNotIn('fail', executor.last_state)
+
+    @unittest.skipIf('sqlite' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                     "sqlite is configured with SequentialExecutor")
+    def test_error_sending_task(self):
+        @app.task
+        def fake_execute_command():
+            pass
+
+        # fake_execute_command takes no arguments while execute_command takes 1,
+        # which will cause TypeError when calling task.apply_async()
+        celery_executor.execute_command = fake_execute_command
+        executor = CeleryExecutor()
+        value_tuple = 'command', '_', 'queue', 'should_be_a_simple_ti'
+        executor.queued_tasks['key'] = value_tuple
+        executor.heartbeat()
+        self.assertEqual(1, len(executor.queued_tasks))
+        self.assertEqual(executor.queued_tasks['key'], value_tuple)
+
+    def test_exception_propagation(self):
+        @app.task
+        def fake_celery_task():
+            return {}
+
+        mock_log = mock.MagicMock()
+        executor = CeleryExecutor()
+        executor._log = mock_log
+
+        executor.tasks = {'key': fake_celery_task()}
+        executor.sync()
+        assert mock_log.error.call_count == 1
+        args, kwargs = mock_log.error.call_args_list[0]
+        # Result of queuing is not a celery task but a dict,
+        # and it should raise AttributeError and then get propagated
+        # to the error log.
+        self.assertIn(celery_executor.CELERY_FETCH_ERR_MSG_HEADER, args[0])
+        self.assertIn('AttributeError', args[1])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/executors/dask_executor.py b/tests/executors/test_dask_executor.py
similarity index 98%
rename from tests/executors/dask_executor.py
rename to tests/executors/test_dask_executor.py
index 7937b1b8a3c95..4f0009e1ccb21 100644
--- a/tests/executors/dask_executor.py
+++ b/tests/executors/test_dask_executor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -55,8 +55,8 @@ def assert_tasks_on_executor(self, executor):
         # start the executor
         executor.start()
 
-        success_command = 'echo 1'
-        fail_command = 'exit 1'
+        success_command = ['true', 'some_parameter']
+        fail_command = ['false', 'some_parameter']
 
         executor.execute_async(key='success', command=success_command)
         executor.execute_async(key='fail', command=fail_command)
diff --git a/tests/executors/test_executor.py b/tests/executors/test_executor.py
index a6fc54020f04e..366ea8c967cce 100644
--- a/tests/executors/test_executor.py
+++ b/tests/executors/test_executor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -26,6 +26,7 @@ class TestExecutor(BaseExecutor):
     """
     TestExecutor is used for unit testing purposes.
     """
+
     def __init__(self, do_update=False, *args, **kwargs):
         self.do_update = do_update
         self._running = []
@@ -45,7 +46,8 @@ def heartbeat(self):
                 ti = self._running.pop()
                 ti.set_state(State.SUCCESS, session)
             for key, val in list(self.queued_tasks.items()):
-                (command, priority, queue, ti) = val
+                (command, priority, queue, simple_ti) = val
+                ti = simple_ti.construct_task_instance()
                 ti.set_state(State.RUNNING, session)
                 self._running.append(ti)
                 self.queued_tasks.pop(key)
@@ -58,4 +60,3 @@ def terminate(self):
 
     def end(self):
         self.sync()
-
diff --git a/tests/executors/test_local_executor.py b/tests/executors/test_local_executor.py
index 91d1ec8196246..49939d702ecc1 100644
--- a/tests/executors/test_local_executor.py
+++ b/tests/executors/test_local_executor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -33,24 +33,24 @@ def execution_parallelism(self, parallelism=0):
         executor.start()
 
         success_key = 'success {}'
-        success_command = 'echo {}'
-        fail_command = 'exit 1'
+        success_command = ['true', 'some_parameter']
+        fail_command = ['false', 'some_parameter']
 
         for i in range(self.TEST_SUCCESS_COMMANDS):
-            key, command = success_key.format(i), success_command.format(i)
+            key, command = success_key.format(i), success_command
             executor.execute_async(key=key, command=command)
             executor.running[key] = True
 
         # errors are propagated for some reason
         try:
             executor.execute_async(key='fail', command=fail_command)
-        except:
+        except Exception:
             pass
 
         executor.running['fail'] = True
 
         if parallelism == 0:
-            with timeout(seconds=5):
+            with timeout(seconds=10):
                 executor.end()
         else:
             executor.end()
diff --git a/tests/hooks/__init__.py b/tests/hooks/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/hooks/__init__.py
+++ b/tests/hooks/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/hooks/test_dbapi_hook.py b/tests/hooks/test_dbapi_hook.py
index 3484ee99da99b..05a571d8430b3 100644
--- a/tests/hooks/test_dbapi_hook.py
+++ b/tests/hooks/test_dbapi_hook.py
@@ -51,8 +51,8 @@ def test_get_records(self):
 
         self.assertEqual(rows, self.db_hook.get_records(statement))
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
         self.cur.execute.assert_called_once_with(statement)
 
     def test_get_records_parameters(self):
@@ -65,8 +65,8 @@ def test_get_records_parameters(self):
 
         self.assertEqual(rows, self.db_hook.get_records(statement, parameters))
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
         self.cur.execute.assert_called_once_with(statement, parameters)
 
     def test_get_records_exception(self):
@@ -76,8 +76,8 @@ def test_get_records_exception(self):
         with self.assertRaises(RuntimeError):
             self.db_hook.get_records(statement)
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        self.conn.close.call_count == 1
+        self.cur.close.call_count == 1
         self.cur.execute.assert_called_once_with(statement)
 
     def test_insert_rows(self):
@@ -87,8 +87,8 @@ def test_insert_rows(self):
 
         self.db_hook.insert_rows(table, rows)
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
 
         commit_count = 2  # The first and last commit
         self.assertEqual(commit_count, self.conn.commit.call_count)
@@ -104,8 +104,8 @@ def test_insert_rows_replace(self):
 
         self.db_hook.insert_rows(table, rows, replace=True)
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
 
         commit_count = 2  # The first and last commit
         self.assertEqual(commit_count, self.conn.commit.call_count)
@@ -122,8 +122,8 @@ def test_insert_rows_target_fields(self):
 
         self.db_hook.insert_rows(table, rows, target_fields)
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
 
         commit_count = 2  # The first and last commit
         self.assertEqual(commit_count, self.conn.commit.call_count)
@@ -140,8 +140,8 @@ def test_insert_rows_commit_every(self):
 
         self.db_hook.insert_rows(table, rows, commit_every=commit_every)
 
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
 
         commit_count = 2 + divmod(len(rows), commit_every)[0]
         self.assertEqual(commit_count, self.conn.commit.call_count)
diff --git a/tests/hooks/test_docker_hook.py b/tests/hooks/test_docker_hook.py
index b8a0132e505ca..438c70134f0a0 100644
--- a/tests/hooks/test_docker_hook.py
+++ b/tests/hooks/test_docker_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,30 +20,23 @@
 import unittest
 
 from airflow import configuration
-from airflow import models
 from airflow.exceptions import AirflowException
+from airflow.models.connection import Connection
 from airflow.utils import db
+from tests.compat import mock
 
 try:
     from airflow.hooks.docker_hook import DockerHook
 except ImportError:
     pass
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
 
 @mock.patch('airflow.hooks.docker_hook.APIClient', autospec=True)
 class DockerHookTest(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='docker_default',
                 conn_type='docker',
                 host='some.docker.registry.com',
@@ -52,7 +45,7 @@ def setUp(self):
             )
         )
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='docker_with_extras',
                 conn_type='docker',
                 host='some.docker.registry.com',
@@ -101,7 +94,7 @@ def test_get_conn_with_standard_config(self, _):
             )
             client = hook.get_conn()
             self.assertIsNotNone(client)
-        except:
+        except Exception:
             self.fail('Could not get connection from Airflow')
 
     def test_get_conn_with_extra_config(self, _):
@@ -113,7 +106,7 @@ def test_get_conn_with_extra_config(self, _):
             )
             client = hook.get_conn()
             self.assertIsNotNone(client)
-        except:
+        except Exception:
             self.fail('Could not get connection from Airflow')
 
     def test_conn_with_standard_config_passes_parameters(self, _):
@@ -148,7 +141,7 @@ def test_conn_with_extra_config_passes_parameters(self, _):
 
     def test_conn_with_broken_config_missing_username_fails(self, _):
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='docker_without_username',
                 conn_type='docker',
                 host='some.docker.registry.com',
@@ -157,7 +150,7 @@ def test_conn_with_broken_config_missing_username_fails(self, _):
             )
         )
         with self.assertRaises(AirflowException):
-            hook = DockerHook(
+            DockerHook(
                 docker_conn_id='docker_without_username',
                 base_url='unix://var/run/docker.sock',
                 version='auto'
@@ -165,7 +158,7 @@ def test_conn_with_broken_config_missing_username_fails(self, _):
 
     def test_conn_with_broken_config_missing_host_fails(self, _):
         db.merge_conn(
-            models.Connection(
+            Connection(
                 conn_id='docker_without_host',
                 conn_type='docker',
                 login='some_user',
@@ -173,7 +166,7 @@ def test_conn_with_broken_config_missing_host_fails(self, _):
             )
         )
         with self.assertRaises(AirflowException):
-            hook = DockerHook(
+            DockerHook(
                 docker_conn_id='docker_without_host',
                 base_url='unix://var/run/docker.sock',
                 version='auto'
diff --git a/tests/hooks/test_druid_hook.py b/tests/hooks/test_druid_hook.py
index 6fd7b3cc76552..633d985d4a6e3 100644
--- a/tests/hooks/test_druid_hook.py
+++ b/tests/hooks/test_druid_hook.py
@@ -98,7 +98,7 @@ def test_submit_unknown_response(self, m):
 
     @requests_mock.mock()
     def test_submit_timeout(self, m):
-        self.db_hook.timeout = 0
+        self.db_hook.timeout = 1
         self.db_hook.max_ingestion_time = 5
         task_post = m.post(
             'http://druid-overlord:8081/druid/indexer/v1/task',
@@ -131,8 +131,8 @@ def test_get_conn_url(self, mock_get_connection):
         get_conn_value.port = '1'
         get_conn_value.extra_dejson = {'endpoint': 'ingest'}
         mock_get_connection.return_value = get_conn_value
-        hook = DruidHook(timeout=0, max_ingestion_time=5)
-        self.assertEquals(hook.get_conn_url(), 'https://test_host:1/ingest')
+        hook = DruidHook(timeout=1, max_ingestion_time=5)
+        self.assertEqual(hook.get_conn_url(), 'https://test_host:1/ingest')
 
 
 class TestDruidDbApiHook(unittest.TestCase):
@@ -158,7 +158,7 @@ def get_connection(self, conn_id):
 
     def test_get_uri(self):
         db_hook = self.db_hook()
-        self.assertEquals('druid://host:1000/druid/v2/sql', db_hook.get_uri())
+        self.assertEqual('druid://host:1000/druid/v2/sql', db_hook.get_uri())
 
     def test_get_first_record(self):
         statement = 'SQL'
@@ -166,8 +166,8 @@ def test_get_first_record(self):
         self.cur.fetchone.return_value = result_sets[0]
 
         self.assertEqual(result_sets[0], self.db_hook().get_first(statement))
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
         self.cur.execute.assert_called_once_with(statement)
 
     def test_get_records(self):
@@ -176,8 +176,8 @@ def test_get_records(self):
         self.cur.fetchall.return_value = result_sets
 
         self.assertEqual(result_sets, self.db_hook().get_records(statement))
-        self.conn.close.assert_called_once()
-        self.cur.close.assert_called_once()
+        assert self.conn.close.call_count == 1
+        assert self.cur.close.call_count == 1
         self.cur.execute.assert_called_once_with(statement)
 
 
diff --git a/tests/hooks/test_hive_hook.py b/tests/hooks/test_hive_hook.py
index 81270c635b042..aa1e95149021c 100644
--- a/tests/hooks/test_hive_hook.py
+++ b/tests/hooks/test_hive_hook.py
@@ -21,30 +21,29 @@
 import datetime
 import itertools
 import os
-
-import pandas as pd
 import random
-
-import mock
+import sys
 import unittest
-
 from collections import OrderedDict
+
+import mock
+import pandas as pd
 from hmsclient import HMSClient
 
+from airflow import DAG, configuration
 from airflow.exceptions import AirflowException
 from airflow.hooks.hive_hooks import HiveCliHook, HiveMetastoreHook, HiveServer2Hook
-from airflow import DAG, configuration
 from airflow.operators.hive_operator import HiveOperator
 from airflow.utils import timezone
+from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING
 from airflow.utils.tests import assertEqualIgnoreMultipleSpaces
 
-
 configuration.load_test_config()
 
-
 DEFAULT_DATE = timezone.datetime(2015, 1, 1)
 DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat()
 DEFAULT_DATE_DS = DEFAULT_DATE_ISO[:10]
+NOT_ASSERTLOGS_VERSION = sys.version_info.major + sys.version_info.minor / 10
 
 
 class HiveEnvironmentTest(unittest.TestCase):
@@ -97,6 +96,39 @@ def test_run_cli(self):
         hook = HiveCliHook()
         hook.run_cli("SHOW DATABASES")
 
+    def test_run_cli_with_hive_conf(self):
+        hql = "set key;\n" \
+              "set airflow.ctx.dag_id;\nset airflow.ctx.dag_run_id;\n" \
+              "set airflow.ctx.task_id;\nset airflow.ctx.execution_date;\n"
+
+        dag_id_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_DAG_ID']['env_var_format']
+        task_id_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_TASK_ID']['env_var_format']
+        execution_date_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_EXECUTION_DATE'][
+                'env_var_format']
+        dag_run_id_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_DAG_RUN_ID'][
+                'env_var_format']
+        os.environ[dag_id_ctx_var_name] = 'test_dag_id'
+        os.environ[task_id_ctx_var_name] = 'test_task_id'
+        os.environ[execution_date_ctx_var_name] = 'test_execution_date'
+        os.environ[dag_run_id_ctx_var_name] = 'test_dag_run_id'
+
+        hook = HiveCliHook()
+        output = hook.run_cli(hql=hql, hive_conf={'key': 'value'})
+        self.assertIn('value', output)
+        self.assertIn('test_dag_id', output)
+        self.assertIn('test_task_id', output)
+        self.assertIn('test_execution_date', output)
+        self.assertIn('test_dag_run_id', output)
+
+        del os.environ[dag_id_ctx_var_name]
+        del os.environ[task_id_ctx_var_name]
+        del os.environ[execution_date_ctx_var_name]
+        del os.environ[dag_run_id_ctx_var_name]
+
     @mock.patch('airflow.hooks.hive_hooks.HiveCliHook.run_cli')
     def test_load_file(self, mock_run_cli):
         filepath = "/path/to/input/file"
@@ -107,7 +139,7 @@ def test_load_file(self, mock_run_cli):
 
         query = (
             "LOAD DATA LOCAL INPATH '{filepath}' "
-            "OVERWRITE INTO TABLE {table} \n"
+            "OVERWRITE INTO TABLE {table} ;\n"
             .format(filepath=filepath, table=table)
         )
         mock_run_cli.assert_called_with(query)
@@ -126,13 +158,13 @@ def test_load_df(self, mock_to_csv, mock_load_file):
                      delimiter=delimiter,
                      encoding=encoding)
 
-        mock_to_csv.assert_called_once()
+        assert mock_to_csv.call_count == 1
         kwargs = mock_to_csv.call_args[1]
         self.assertEqual(kwargs["header"], False)
         self.assertEqual(kwargs["index"], False)
         self.assertEqual(kwargs["sep"], delimiter)
 
-        mock_load_file.assert_called_once()
+        assert mock_load_file.call_count == 1
         kwargs = mock_load_file.call_args[1]
         self.assertEqual(kwargs["delimiter"], delimiter)
         self.assertEqual(kwargs["field_dict"], {"c": u"STRING"})
@@ -151,7 +183,7 @@ def test_load_df_with_optional_parameters(self, mock_to_csv, mock_load_file):
                          create=create,
                          recreate=recreate)
 
-            mock_load_file.assert_called_once()
+            assert mock_load_file.call_count == 1
             kwargs = mock_load_file.call_args[1]
             self.assertEqual(kwargs["create"], create)
             self.assertEqual(kwargs["recreate"], recreate)
@@ -399,7 +431,23 @@ def test_get_results_data(self):
         results = hook.get_results(query, schema=self.database)
         self.assertListEqual(results['data'], [(1, 1), (2, 2)])
 
-    def test_to_csv(self):
+    @unittest.skipIf(NOT_ASSERTLOGS_VERSION < 3.4, 'assertLogs not support before python 3.4')
+    def test_to_csv_assertlogs(self):
+        hook = HiveServer2Hook()
+        query = "SELECT * FROM {}".format(self.table)
+        csv_filepath = 'query_results.csv'
+        with self.assertLogs() as cm:
+            hook.to_csv(query, csv_filepath, schema=self.database,
+                        delimiter=',', lineterminator='\n', output_header=True, fetch_size=2)
+            df = pd.read_csv(csv_filepath, sep=',')
+            self.assertListEqual(df.columns.tolist(), self.columns)
+            self.assertListEqual(df[self.columns[0]].values.tolist(), [1, 2])
+            self.assertEqual(len(df), 2)
+            self.assertIn('INFO:airflow.hooks.hive_hooks.HiveServer2Hook:'
+                          'Written 2 rows so far.', cm.output)
+
+    @unittest.skipIf(NOT_ASSERTLOGS_VERSION >= 3.4, 'test could cover by test_to_csv_assertLogs')
+    def test_to_csv_without_assertlogs(self):
         hook = HiveServer2Hook()
         query = "SELECT * FROM {}".format(self.table)
         csv_filepath = 'query_results.csv'
@@ -419,3 +467,41 @@ def test_multi_statements(self):
         hook = HiveServer2Hook()
         results = hook.get_records(sqls, schema=self.database)
         self.assertListEqual(results, [(1, 1), (2, 2)])
+
+    def test_get_results_with_hive_conf(self):
+        hql = ["set key",
+               "set airflow.ctx.dag_id",
+               "set airflow.ctx.dag_run_id",
+               "set airflow.ctx.task_id",
+               "set airflow.ctx.execution_date"]
+
+        dag_id_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_DAG_ID']['env_var_format']
+        task_id_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_TASK_ID']['env_var_format']
+        execution_date_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_EXECUTION_DATE'][
+                'env_var_format']
+        dag_run_id_ctx_var_name = \
+            AIRFLOW_VAR_NAME_FORMAT_MAPPING['AIRFLOW_CONTEXT_DAG_RUN_ID'][
+                'env_var_format']
+        os.environ[dag_id_ctx_var_name] = 'test_dag_id'
+        os.environ[task_id_ctx_var_name] = 'test_task_id'
+        os.environ[execution_date_ctx_var_name] = 'test_execution_date'
+        os.environ[dag_run_id_ctx_var_name] = 'test_dag_run_id'
+
+        hook = HiveServer2Hook()
+        output = '\n'.join(res_tuple[0]
+                           for res_tuple
+                           in hook.get_results(hql=hql,
+                                               hive_conf={'key': 'value'})['data'])
+        self.assertIn('value', output)
+        self.assertIn('test_dag_id', output)
+        self.assertIn('test_task_id', output)
+        self.assertIn('test_execution_date', output)
+        self.assertIn('test_dag_run_id', output)
+
+        del os.environ[dag_id_ctx_var_name]
+        del os.environ[task_id_ctx_var_name]
+        del os.environ[execution_date_ctx_var_name]
+        del os.environ[dag_run_id_ctx_var_name]
diff --git a/tests/hooks/test_http_hook.py b/tests/hooks/test_http_hook.py
index c8163322f942b..f569c51f2f047 100644
--- a/tests/hooks/test_http_hook.py
+++ b/tests/hooks/test_http_hook.py
@@ -1,16 +1,21 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 import unittest
 
 import json
@@ -20,21 +25,15 @@
 
 import tenacity
 
-from airflow import configuration, models
+from airflow import configuration
 from airflow.exceptions import AirflowException
 from airflow.hooks.http_hook import HttpHook
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from airflow.models.connection import Connection
+from tests.compat import mock
 
 
 def get_airflow_connection(conn_id=None):
-    return models.Connection(
+    return Connection(
         conn_id='http_default',
         conn_type='http',
         host='test:8080/',
@@ -42,6 +41,15 @@ def get_airflow_connection(conn_id=None):
     )
 
 
+def get_airflow_connection_with_port(conn_id=None):
+    return Connection(
+        conn_id='http_default',
+        conn_type='http',
+        host='test.com',
+        port=1234
+    )
+
+
 class TestHttpHook(unittest.TestCase):
     """Test get, post and raise_for_status"""
     def setUp(self):
@@ -67,7 +75,33 @@ def test_raise_for_status_with_200(self, m):
         ):
 
             resp = self.get_hook.run('v1/test')
-            self.assertEquals(resp.text, '{"status":{"status": 200}}')
+            self.assertEqual(resp.text, '{"status":{"status": 200}}')
+
+    @requests_mock.mock()
+    @mock.patch('requests.Request')
+    def test_get_request_with_port(self, m, request_mock):
+        from requests.exceptions import MissingSchema
+
+        with mock.patch(
+            'airflow.hooks.base_hook.BaseHook.get_connection',
+            side_effect=get_airflow_connection_with_port
+        ):
+            expected_url = 'http://test.com:1234/some/endpoint'
+            for endpoint in ['some/endpoint', '/some/endpoint']:
+
+                try:
+                    self.get_hook.run(endpoint)
+                except MissingSchema:
+                    pass
+
+                request_mock.assert_called_once_with(
+                    mock.ANY,
+                    expected_url,
+                    headers=mock.ANY,
+                    params=mock.ANY
+                )
+
+                request_mock.reset_mock()
 
     @requests_mock.mock()
     def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, m):
@@ -84,7 +118,7 @@ def test_get_request_do_not_raise_for_status_if_check_response_is_false(self, m)
             side_effect=get_airflow_connection
         ):
             resp = self.get_hook.run('v1/test', extra_options={'check_response': False})
-            self.assertEquals(resp.text, '{"status":{"status": 404}}')
+            self.assertEqual(resp.text, '{"status":{"status": 404}}')
 
     @requests_mock.mock()
     def test_hook_contains_header_from_extra_field(self, m):
@@ -95,17 +129,17 @@ def test_hook_contains_header_from_extra_field(self, m):
             expected_conn = get_airflow_connection()
             conn = self.get_hook.get_conn()
             self.assertDictContainsSubset(json.loads(expected_conn.extra), conn.headers)
-            self.assertEquals(conn.headers.get('bareer'), 'test')
+            self.assertEqual(conn.headers.get('bareer'), 'test')
 
     @requests_mock.mock()
     def test_hook_uses_provided_header(self, m):
-            conn = self.get_hook.get_conn(headers={"bareer": "newT0k3n"})
-            self.assertEquals(conn.headers.get('bareer'), "newT0k3n")
+        conn = self.get_hook.get_conn(headers={"bareer": "newT0k3n"})
+        self.assertEqual(conn.headers.get('bareer'), "newT0k3n")
 
     @requests_mock.mock()
     def test_hook_has_no_header_from_extra(self, m):
-            conn = self.get_hook.get_conn()
-            self.assertIsNone(conn.headers.get('bareer'))
+        conn = self.get_hook.get_conn()
+        self.assertIsNone(conn.headers.get('bareer'))
 
     @requests_mock.mock()
     def test_hooks_header_from_extra_is_overridden(self, m):
@@ -114,7 +148,7 @@ def test_hooks_header_from_extra_is_overridden(self, m):
             side_effect=get_airflow_connection
         ):
             conn = self.get_hook.get_conn(headers={"bareer": "newT0k3n"})
-            self.assertEquals(conn.headers.get('bareer'), 'newT0k3n')
+            self.assertEqual(conn.headers.get('bareer'), 'newT0k3n')
 
     @requests_mock.mock()
     def test_post_request(self, m):
@@ -131,7 +165,7 @@ def test_post_request(self, m):
             side_effect=get_airflow_connection
         ):
             resp = self.post_hook.run('v1/test')
-            self.assertEquals(resp.status_code, 200)
+            self.assertEqual(resp.status_code, 200)
 
     @requests_mock.mock()
     def test_post_request_with_error_code(self, m):
@@ -165,7 +199,7 @@ def test_post_request_do_not_raise_for_status_if_check_response_is_false(self, m
             side_effect=get_airflow_connection
         ):
             resp = self.post_hook.run('v1/test', extra_options={'check_response': False})
-            self.assertEquals(resp.status_code, 418)
+            self.assertEqual(resp.status_code, 418)
 
     @mock.patch('airflow.hooks.http_hook.requests.Session')
     def test_retry_on_conn_error(self, mocked_session):
@@ -186,7 +220,7 @@ def send_and_raise(request, **kwargs):
                 endpoint='v1/test',
                 _retry_args=retry_args
             )
-        self.assertEquals(
+        self.assertEqual(
             self.get_hook._retry_obj.stop.max_attempt_number + 1,
             mocked_session.call_count
         )
@@ -207,13 +241,13 @@ def run_and_return(session, prepped_request, extra_options, **kwargs):
             ):
                 pr = self.get_hook.run('v1/test', headers={'some_other_header': 'test'})
                 actual = dict(pr.headers)
-                self.assertEquals(actual.get('bareer'), 'test')
-                self.assertEquals(actual.get('some_other_header'), 'test')
+                self.assertEqual(actual.get('bareer'), 'test')
+                self.assertEqual(actual.get('some_other_header'), 'test')
 
     @mock.patch('airflow.hooks.http_hook.HttpHook.get_connection')
     def test_http_connection(self, mock_get_connection):
-        c = models.Connection(conn_id='http_default', conn_type='http',
-                              host='localhost', schema='http')
+        c = Connection(conn_id='http_default', conn_type='http',
+                       host='localhost', schema='http')
         mock_get_connection.return_value = c
         hook = HttpHook()
         hook.get_conn({})
@@ -221,8 +255,8 @@ def test_http_connection(self, mock_get_connection):
 
     @mock.patch('airflow.hooks.http_hook.HttpHook.get_connection')
     def test_https_connection(self, mock_get_connection):
-        c = models.Connection(conn_id='http_default', conn_type='http',
-                              host='localhost', schema='https')
+        c = Connection(conn_id='http_default', conn_type='http',
+                       host='localhost', schema='https')
         mock_get_connection.return_value = c
         hook = HttpHook()
         hook.get_conn({})
@@ -230,8 +264,8 @@ def test_https_connection(self, mock_get_connection):
 
     @mock.patch('airflow.hooks.http_hook.HttpHook.get_connection')
     def test_host_encoded_http_connection(self, mock_get_connection):
-        c = models.Connection(conn_id='http_default', conn_type='http',
-                              host='http://localhost')
+        c = Connection(conn_id='http_default', conn_type='http',
+                       host='http://localhost')
         mock_get_connection.return_value = c
         hook = HttpHook()
         hook.get_conn({})
@@ -239,8 +273,8 @@ def test_host_encoded_http_connection(self, mock_get_connection):
 
     @mock.patch('airflow.hooks.http_hook.HttpHook.get_connection')
     def test_host_encoded_https_connection(self, mock_get_connection):
-        c = models.Connection(conn_id='http_default', conn_type='http',
-                              host='https://localhost')
+        c = Connection(conn_id='http_default', conn_type='http',
+                       host='https://localhost')
         mock_get_connection.return_value = c
         hook = HttpHook()
         hook.get_conn({})
diff --git a/tests/hooks/test_mysql_hook.py b/tests/hooks/test_mysql_hook.py
index d112f880e9e3a..94694618481a2 100644
--- a/tests/hooks/test_mysql_hook.py
+++ b/tests/hooks/test_mysql_hook.py
@@ -18,10 +18,112 @@
 # under the License.
 #
 
+import json
 import mock
 import unittest
 
+import MySQLdb.cursors
+
 from airflow.hooks.mysql_hook import MySqlHook
+from airflow.models.connection import Connection
+
+SSL_DICT = {
+    'cert': '/tmp/client-cert.pem',
+    'ca': '/tmp/server-ca.pem',
+    'key': '/tmp/client-key.pem'
+}
+
+
+class TestMySqlHookConn(unittest.TestCase):
+
+    def setUp(self):
+        super(TestMySqlHookConn, self).setUp()
+
+        self.connection = Connection(
+            login='login',
+            password='password',
+            host='host',
+            schema='schema',
+        )
+
+        self.db_hook = MySqlHook()
+        self.db_hook.get_connection = mock.Mock()
+        self.db_hook.get_connection.return_value = self.connection
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn(self, mock_connect):
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['user'], 'login')
+        self.assertEqual(kwargs['passwd'], 'password')
+        self.assertEqual(kwargs['host'], 'host')
+        self.assertEqual(kwargs['db'], 'schema')
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn_port(self, mock_connect):
+        self.connection.port = 3307
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['port'], 3307)
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn_charset(self, mock_connect):
+        self.connection.extra = json.dumps({'charset': 'utf-8'})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['charset'], 'utf-8')
+        self.assertEqual(kwargs['use_unicode'], True)
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn_cursor(self, mock_connect):
+        self.connection.extra = json.dumps({'cursor': 'sscursor'})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['cursorclass'], MySQLdb.cursors.SSCursor)
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn_local_infile(self, mock_connect):
+        self.connection.extra = json.dumps({'local_infile': True})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['local_infile'], 1)
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_con_unix_socket(self, mock_connect):
+        self.connection.extra = json.dumps({'unix_socket': "/tmp/socket"})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['unix_socket'], '/tmp/socket')
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn_ssl_as_dictionary(self, mock_connect):
+        self.connection.extra = json.dumps({'ssl': SSL_DICT})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['ssl'], SSL_DICT)
+
+    @mock.patch('airflow.hooks.mysql_hook.MySQLdb.connect')
+    def test_get_conn_ssl_as_string(self, mock_connect):
+        self.connection.extra = json.dumps({'ssl': json.dumps(SSL_DICT)})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['ssl'], SSL_DICT)
 
 
 class TestMySqlHook(unittest.TestCase):
@@ -57,7 +159,7 @@ def test_run_without_autocommit(self):
         self.db_hook.run(sql, autocommit=False)
         self.conn.autocommit.assert_called_once_with(False)
         self.cur.execute.assert_called_once_with(sql)
-        self.conn.commit.assert_called_once()
+        assert self.conn.commit.call_count == 1
 
     def test_run_with_autocommit(self):
         sql = 'SQL'
@@ -85,3 +187,20 @@ def test_run_multi_queries(self):
             self.assertEqual(kwargs, {})
         self.cur.execute.assert_called_with(sql[1])
         self.conn.commit.assert_not_called()
+
+    def test_bulk_load(self):
+        self.db_hook.bulk_load('table', '/tmp/file')
+        self.cur.execute.assert_called_once_with("""
+            LOAD DATA LOCAL INFILE '/tmp/file'
+            INTO TABLE table
+            """)
+
+    def test_bulk_dump(self):
+        self.db_hook.bulk_dump('table', '/tmp/file')
+        self.cur.execute.assert_called_once_with("""
+            SELECT * INTO OUTFILE '/tmp/file'
+            FROM table
+            """)
+
+    def test_serialize_cell(self):
+        self.assertEqual('foo', self.db_hook._serialize_cell('foo', None))
diff --git a/tests/hooks/test_oracle_hook.py b/tests/hooks/test_oracle_hook.py
new file mode 100644
index 0000000000000..927a7b8f110b7
--- /dev/null
+++ b/tests/hooks/test_oracle_hook.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import unittest
+from datetime import datetime
+
+try:
+    import cx_Oracle
+except ImportError:
+    cx_Oracle = None
+import numpy
+
+from airflow.hooks.oracle_hook import OracleHook
+from airflow.models import Connection
+from tests.compat import mock
+
+
+@unittest.skipIf(cx_Oracle is None, 'cx_Oracle package not present')
+class TestOracleHookConn(unittest.TestCase):
+
+    def setUp(self):
+        super(TestOracleHookConn, self).setUp()
+
+        self.connection = Connection(
+            login='login',
+            password='password',
+            host='host',
+            port=1521
+        )
+
+        self.db_hook = OracleHook()
+        self.db_hook.get_connection = mock.Mock()
+        self.db_hook.get_connection.return_value = self.connection
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_host(self, mock_connect):
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['user'], 'login')
+        self.assertEqual(kwargs['password'], 'password')
+        self.assertEqual(kwargs['dsn'], 'host')
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_sid(self, mock_connect):
+        dsn_sid = {'dsn': 'dsn', 'sid': 'sid'}
+        self.connection.extra = json.dumps(dsn_sid)
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['dsn'],
+                         cx_Oracle.makedsn(dsn_sid['dsn'],
+                                           self.connection.port, dsn_sid['sid']))
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_service_name(self, mock_connect):
+        dsn_service_name = {'dsn': 'dsn', 'service_name': 'service_name'}
+        self.connection.extra = json.dumps(dsn_service_name)
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['dsn'], cx_Oracle.makedsn(
+            dsn_service_name['dsn'], self.connection.port,
+            service_name=dsn_service_name['service_name']))
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_encoding_without_nencoding(self, mock_connect):
+        self.connection.extra = json.dumps({'encoding': 'UTF-8'})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['encoding'], 'UTF-8')
+        self.assertEqual(kwargs['nencoding'], 'UTF-8')
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_encoding_with_nencoding(self, mock_connect):
+        self.connection.extra = json.dumps({'encoding': 'UTF-8', 'nencoding': 'gb2312'})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['encoding'], 'UTF-8')
+        self.assertEqual(kwargs['nencoding'], 'gb2312')
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_nencoding(self, mock_connect):
+        self.connection.extra = json.dumps({'nencoding': 'UTF-8'})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertNotIn('encoding', kwargs)
+        self.assertEqual(kwargs['nencoding'], 'UTF-8')
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_mode(self, mock_connect):
+        mode = {
+            'sysdba': cx_Oracle.SYSDBA,
+            'sysasm': cx_Oracle.SYSASM,
+            'sysoper': cx_Oracle.SYSOPER,
+            'sysbkp': cx_Oracle.SYSBKP,
+            'sysdgd': cx_Oracle.SYSDGD,
+            'syskmt': cx_Oracle.SYSKMT,
+        }
+        first = True
+        for m in mode:
+            self.connection.extra = json.dumps({'mode': m})
+            self.db_hook.get_conn()
+            if first:
+                assert mock_connect.call_count == 1
+                first = False
+            args, kwargs = mock_connect.call_args
+            self.assertEqual(args, ())
+            self.assertEqual(kwargs['mode'], mode.get(m))
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_threaded(self, mock_connect):
+        self.connection.extra = json.dumps({'threaded': True})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['threaded'], True)
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_events(self, mock_connect):
+        self.connection.extra = json.dumps({'events': True})
+        self.db_hook.get_conn()
+        assert mock_connect.call_count == 1
+        args, kwargs = mock_connect.call_args
+        self.assertEqual(args, ())
+        self.assertEqual(kwargs['events'], True)
+
+    @mock.patch('airflow.hooks.oracle_hook.cx_Oracle.connect')
+    def test_get_conn_purity(self, mock_connect):
+        purity = {
+            'new': cx_Oracle.ATTR_PURITY_NEW,
+            'self': cx_Oracle.ATTR_PURITY_SELF,
+            'default': cx_Oracle.ATTR_PURITY_DEFAULT
+        }
+        first = True
+        for p in purity:
+            self.connection.extra = json.dumps({'purity': p})
+            self.db_hook.get_conn()
+            if first:
+                assert mock_connect.call_count == 1
+                first = False
+            args, kwargs = mock_connect.call_args
+            self.assertEqual(args, ())
+            self.assertEqual(kwargs['purity'], purity.get(p))
+
+
+@unittest.skipIf(cx_Oracle is None, 'cx_Oracle package not present')
+class TestOracleHook(unittest.TestCase):
+    def setUp(self):
+        super(TestOracleHook, self).setUp()
+
+        self.cur = mock.MagicMock()
+        self.conn = mock.MagicMock()
+        self.conn.cursor.return_value = self.cur
+        conn = self.conn
+
+        class UnitTestOracleHook(OracleHook):
+            conn_name_attr = 'test_conn_id'
+
+            def get_conn(self):
+                return conn
+
+        self.db_hook = UnitTestOracleHook()
+
+    def test_run_without_parameters(self):
+        sql = 'SQL'
+        self.db_hook.run(sql)
+        self.cur.execute.assert_called_once_with(sql)
+        assert self.conn.commit.called
+
+    def test_run_with_parameters(self):
+        sql = 'SQL'
+        param = ('p1', 'p2')
+        self.db_hook.run(sql, parameters=param)
+        self.cur.execute.assert_called_once_with(sql, param)
+        assert self.conn.commit.called
+
+    def test_insert_rows_with_fields(self):
+        rows = [("'basestr_with_quote", None, numpy.NAN,
+                 numpy.datetime64('2019-01-24T01:02:03'),
+                 datetime(2019, 1, 24), 1, 10.24, 'str')]
+        target_fields = ['basestring', 'none', 'numpy_nan', 'numpy_datetime64',
+                         'datetime', 'int', 'float', 'str']
+        self.db_hook.insert_rows('table', rows, target_fields)
+        self.cur.execute.assert_called_once_with(
+            "INSERT /*+ APPEND */ INTO table "
+            "(basestring, none, numpy_nan, numpy_datetime64, datetime, int, float, str) "
+            "VALUES ('''basestr_with_quote',NULL,NULL,'2019-01-24T01:02:03',"
+            "to_date('2019-01-24 00:00:00','YYYY-MM-DD HH24:MI:SS'),1,10.24,'str')")
+
+    def test_insert_rows_without_fields(self):
+        rows = [("'basestr_with_quote", None, numpy.NAN,
+                 numpy.datetime64('2019-01-24T01:02:03'),
+                 datetime(2019, 1, 24), 1, 10.24, 'str')]
+        self.db_hook.insert_rows('table', rows)
+        self.cur.execute.assert_called_once_with(
+            "INSERT /*+ APPEND */ INTO table "
+            " VALUES ('''basestr_with_quote',NULL,NULL,'2019-01-24T01:02:03',"
+            "to_date('2019-01-24 00:00:00','YYYY-MM-DD HH24:MI:SS'),1,10.24,'str')")
+
+    def test_bulk_insert_rows_with_fields(self):
+        rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        target_fields = ['col1', 'col2', 'col3']
+        self.db_hook.bulk_insert_rows('table', rows, target_fields)
+        self.cur.prepare.assert_called_once_with(
+            "insert into table (col1, col2, col3) values (:1, :2, :3)")
+        self.cur.executemany.assert_called_once_with(None, rows)
+
+    def test_bulk_insert_rows_with_commit_every(self):
+        rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        target_fields = ['col1', 'col2', 'col3']
+        self.db_hook.bulk_insert_rows('table', rows, target_fields, commit_every=2)
+        self.cur.prepare.assert_called_with(
+            "insert into table (col1, col2, col3) values (:1, :2, :3)")
+        self.cur.executemany.assert_called_with(None, rows[2:])
+
+    def test_bulk_insert_rows_without_fields(self):
+        rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        self.db_hook.bulk_insert_rows('table', rows)
+        self.cur.prepare.assert_called_once_with(
+            "insert into table  values (:1, :2, :3)")
+        self.cur.executemany.assert_called_once_with(None, rows)
+
+    def test_bulk_insert_rows_no_rows(self):
+        rows = []
+        self.assertRaises(ValueError, self.db_hook.bulk_insert_rows, 'table', rows)
diff --git a/tests/hooks/test_pig_hook.py b/tests/hooks/test_pig_hook.py
new file mode 100644
index 0000000000000..8f8dfdc6e5387
--- /dev/null
+++ b/tests/hooks/test_pig_hook.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from airflow.hooks.pig_hook import PigCliHook
+from tests.compat import mock
+
+
+class TestPigCliHook(unittest.TestCase):
+
+    def setUp(self):
+        super(TestPigCliHook, self).setUp()
+
+        self.extra_dejson = mock.MagicMock()
+        self.extra_dejson.get.return_value = None
+        self.conn = mock.MagicMock()
+        self.conn.extra_dejson = self.extra_dejson
+        conn = self.conn
+
+        class SubPigCliHook(PigCliHook):
+            def get_connection(self, id):
+                return conn
+
+        self.pig_hook = SubPigCliHook
+
+    def test_init(self):
+        self.pig_hook()
+        self.extra_dejson.get.assert_called_with('pig_properties', '')
+
+    @mock.patch('subprocess.Popen')
+    def test_run_cli_success(self, popen_mock):
+        proc_mock = mock.MagicMock()
+        proc_mock.returncode = 0
+        proc_mock.stdout.readline.return_value = b''
+        popen_mock.return_value = proc_mock
+
+        hook = self.pig_hook()
+        stdout = hook.run_cli("")
+
+        self.assertEqual(stdout, "")
+
+    @mock.patch('subprocess.Popen')
+    def test_run_cli_fail(self, popen_mock):
+        proc_mock = mock.MagicMock()
+        proc_mock.returncode = 1
+        proc_mock.stdout.readline.return_value = b''
+        popen_mock.return_value = proc_mock
+
+        hook = self.pig_hook()
+
+        from airflow.exceptions import AirflowException
+        self.assertRaises(AirflowException, hook.run_cli, "")
+
+    @mock.patch('subprocess.Popen')
+    def test_run_cli_with_properties(self, popen_mock):
+        test_properties = "one two"
+
+        proc_mock = mock.MagicMock()
+        proc_mock.returncode = 0
+        proc_mock.stdout.readline.return_value = b''
+        popen_mock.return_value = proc_mock
+
+        hook = self.pig_hook()
+        hook.pig_properties = test_properties
+
+        stdout = hook.run_cli("")
+        self.assertEqual(stdout, "")
+
+        popen_first_arg = popen_mock.call_args[0][0]
+        for pig_prop in test_properties.split():
+            self.assertIn(pig_prop, popen_first_arg)
+
+    @mock.patch('subprocess.Popen')
+    def test_run_cli_verbose(self, popen_mock):
+        test_stdout_lines = [b"one", b"two", b""]
+        test_stdout_strings = [s.decode('utf-8') for s in test_stdout_lines]
+
+        proc_mock = mock.MagicMock()
+        proc_mock.returncode = 0
+        proc_mock.stdout.readline = mock.Mock(side_effect=test_stdout_lines)
+        popen_mock.return_value = proc_mock
+
+        hook = self.pig_hook()
+        stdout = hook.run_cli("", verbose=True)
+
+        self.assertEqual(stdout, "".join(test_stdout_strings))
+
+    def test_kill_no_sp(self):
+        sp_mock = mock.Mock()
+        hook = self.pig_hook()
+        hook.sp = sp_mock
+
+        hook.kill()
+        self.assertFalse(sp_mock.kill.called)
+
+    def test_kill_sp_done(self):
+        sp_mock = mock.Mock()
+        sp_mock.poll.return_value = 0
+
+        hook = self.pig_hook()
+        hook.sp = sp_mock
+
+        hook.kill()
+        self.assertFalse(sp_mock.kill.called)
+
+    def test_kill(self):
+        sp_mock = mock.Mock()
+        sp_mock.poll.return_value = None
+
+        hook = self.pig_hook()
+        hook.sp = sp_mock
+
+        hook.kill()
+        self.assertTrue(sp_mock.kill.called)
diff --git a/tests/hooks/test_postgres_hook.py b/tests/hooks/test_postgres_hook.py
index 0520239e06ee8..d69f20ce06b82 100644
--- a/tests/hooks/test_postgres_hook.py
+++ b/tests/hooks/test_postgres_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,6 +28,10 @@
 
 class TestPostgresHook(unittest.TestCase):
 
+    def __init__(self, *args, **kwargs):
+        super(TestPostgresHook, self).__init__(*args, **kwargs)
+        self.table = "test_postgres_hook_table"
+
     def setUp(self):
         super(TestPostgresHook, self).setUp()
 
@@ -43,6 +47,13 @@ def get_conn(self):
 
         self.db_hook = UnitTestPostgresHook()
 
+    def tearDown(self):
+        super(TestPostgresHook, self).tearDown()
+
+        with PostgresHook().get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute("DROP TABLE IF EXISTS {}".format(self.table))
+
     def test_copy_expert(self):
         m = mock.mock_open(read_data='{"some": "json"}')
         with mock.patch('airflow.hooks.postgres_hook.open', m):
@@ -53,48 +64,44 @@ def test_copy_expert(self):
 
             self.assertEqual(None, self.db_hook.copy_expert(statement, filename, open=m))
 
-            self.conn.close.assert_called_once()
-            self.cur.close.assert_called_once()
-            self.conn.commit.assert_called_once()
+            assert self.conn.close.call_count == 1
+            assert self.cur.close.call_count == 1
+            assert self.conn.commit.call_count == 1
             self.cur.copy_expert.assert_called_once_with(statement, m.return_value)
             self.assertEqual(m.call_args[0], (filename, "r+"))
 
     def test_bulk_load(self):
         hook = PostgresHook()
-        table = "t"
         input_data = ["foo", "bar", "baz"]
 
         with hook.get_conn() as conn:
             with conn.cursor() as cur:
-                cur.execute("DROP TABLE IF EXISTS {}".format(table))
-                cur.execute("CREATE TABLE {} (c VARCHAR)".format(table))
+                cur.execute("CREATE TABLE {} (c VARCHAR)".format(self.table))
                 conn.commit()
 
                 with NamedTemporaryFile() as f:
                     f.write("\n".join(input_data).encode("utf-8"))
                     f.flush()
-                    hook.bulk_load(table, f.name)
+                    hook.bulk_load(self.table, f.name)
 
-                cur.execute("SELECT * FROM {}".format(table))
+                cur.execute("SELECT * FROM {}".format(self.table))
                 results = [row[0] for row in cur.fetchall()]
 
         self.assertEqual(sorted(input_data), sorted(results))
 
     def test_bulk_dump(self):
         hook = PostgresHook()
-        table = "t"
         input_data = ["foo", "bar", "baz"]
 
         with hook.get_conn() as conn:
             with conn.cursor() as cur:
-                cur.execute("DROP TABLE IF EXISTS {}".format(table))
-                cur.execute("CREATE TABLE {} (c VARCHAR)".format(table))
+                cur.execute("CREATE TABLE {} (c VARCHAR)".format(self.table))
                 values = ",".join("('{}')".format(data) for data in input_data)
-                cur.execute("INSERT INTO {} VALUES {}".format(table, values))
+                cur.execute("INSERT INTO {} VALUES {}".format(self.table, values))
                 conn.commit()
 
                 with NamedTemporaryFile() as f:
-                    hook.bulk_dump(table, f.name)
+                    hook.bulk_dump(self.table, f.name)
                     f.seek(0)
                     results = [line.rstrip().decode("utf-8") for line in f.readlines()]
 
diff --git a/tests/hooks/test_presto_hook.py b/tests/hooks/test_presto_hook.py
index b01782ea4bb68..dbcd0d9660129 100644
--- a/tests/hooks/test_presto_hook.py
+++ b/tests/hooks/test_presto_hook.py
@@ -1,16 +1,21 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 #
 
 import mock
diff --git a/tests/hooks/test_s3_hook.py b/tests/hooks/test_s3_hook.py
index baac20386feb9..45646b0c77462 100644
--- a/tests/hooks/test_s3_hook.py
+++ b/tests/hooks/test_s3_hook.py
@@ -19,15 +19,17 @@
 #
 
 import mock
+import tempfile
 import unittest
 
+from botocore.exceptions import NoCredentialsError
+
 from airflow import configuration
 
 try:
     from airflow.hooks.S3_hook import S3Hook
 except ImportError:
-    S3Hook = None
-
+    S3Hook = None  # type: ignore
 
 try:
     import boto3
@@ -51,6 +53,7 @@ def test_parse_s3_url(self):
         self.assertEqual(parsed,
                          ("test", "this/is/not/a-real-key.txt"),
                          "Incorrect parsing of the s3 url")
+
     @mock_s3
     def test_check_for_bucket(self):
         hook = S3Hook(aws_conn_id=None)
@@ -60,12 +63,43 @@ def test_check_for_bucket(self):
         self.assertTrue(hook.check_for_bucket('bucket'))
         self.assertFalse(hook.check_for_bucket('not-a-bucket'))
 
+    def test_check_for_bucket_raises_error_with_invalid_conn_id(self):
+        hook = S3Hook(aws_conn_id="does_not_exist")
+
+        with self.assertRaises(NoCredentialsError):
+            hook.check_for_bucket('bucket')
+
     @mock_s3
     def test_get_bucket(self):
         hook = S3Hook(aws_conn_id=None)
         b = hook.get_bucket('bucket')
         self.assertIsNotNone(b)
 
+    @mock_s3
+    def test_create_bucket_default_region(self):
+        hook = S3Hook(aws_conn_id=None)
+        hook.create_bucket(bucket_name='new_bucket')
+        b = hook.get_bucket('new_bucket')
+        self.assertIsNotNone(b)
+
+    @mock_s3
+    def test_create_bucket_us_standard_region(self):
+        hook = S3Hook(aws_conn_id=None)
+        hook.create_bucket(bucket_name='new_bucket', region_name='us-east-1')
+        b = hook.get_bucket('new_bucket')
+        self.assertIsNotNone(b)
+        region = b.meta.client.get_bucket_location(Bucket=b.name).get('LocationConstraint', None)
+        self.assertEqual(region, 'us-east-1')
+
+    @mock_s3
+    def test_create_bucket_other_region(self):
+        hook = S3Hook(aws_conn_id=None)
+        hook.create_bucket(bucket_name='new_bucket', region_name='us-east-2')
+        b = hook.get_bucket('new_bucket')
+        self.assertIsNotNone(b)
+        region = b.meta.client.get_bucket_location(Bucket=b.name).get('LocationConstraint', None)
+        self.assertEqual(region, 'us-east-2')
+
     @mock_s3
     def test_check_for_prefix(self):
         hook = S3Hook(aws_conn_id=None)
@@ -146,6 +180,12 @@ def test_check_for_key(self):
         self.assertFalse(hook.check_for_key('b', 'bucket'))
         self.assertFalse(hook.check_for_key('s3://bucket//b'))
 
+    def test_check_for_key_raises_error_with_invalid_conn_id(self):
+        hook = S3Hook(aws_conn_id="does_not_exist")
+
+        with self.assertRaises(NoCredentialsError):
+            hook.check_for_key('a', 'bucket')
+
     @mock_s3
     def test_get_key(self):
         hook = S3Hook(aws_conn_id=None)
@@ -171,7 +211,7 @@ def test_read_key(self):
     @mock.patch('airflow.contrib.hooks.aws_hook.AwsHook.get_client_type')
     def test_select_key(self, mock_get_client_type):
         mock_get_client_type.return_value.select_object_content.return_value = \
-            {'Payload': [{'Records': {'Payload': u'Contént'}}]}
+            {'Payload': [{'Records': {'Payload': b'Cont\xC3\xA9nt'}}]}
         hook = S3Hook(aws_conn_id=None)
         self.assertEqual(hook.select_key('my_key', 'mybucket'), u'Contént')
 
@@ -241,6 +281,21 @@ def test_load_bytes(self):
 
         self.assertEqual(body, b'Content')
 
+    @mock_s3
+    def test_load_fileobj(self):
+        hook = S3Hook(aws_conn_id=None)
+        conn = hook.get_conn()
+        # We need to create the bucket since this is all in Moto's 'virtual'
+        # AWS account
+        conn.create_bucket(Bucket="mybucket")
+        with tempfile.TemporaryFile() as temp_file:
+            temp_file.write(b"Content")
+            temp_file.seek(0)
+            hook.load_file_obj(temp_file, "my_key", "mybucket")
+            body = boto3.resource('s3').Object('mybucket', 'my_key').get()['Body'].read()
+
+            self.assertEqual(body, b'Content')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/hooks/test_slack_hook.py b/tests/hooks/test_slack_hook.py
index 46f8946411b1d..d15c5f251258a 100644
--- a/tests/hooks/test_slack_hook.py
+++ b/tests/hooks/test_slack_hook.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,14 +20,7 @@
 import unittest
 from airflow.exceptions import AirflowException
 from airflow.hooks.slack_hook import SlackHook
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class SlackHookTestCase(unittest.TestCase):
diff --git a/tests/lineage/backend/test_atlas.py b/tests/lineage/backend/test_atlas.py
index 9c98334a9a634..5898e683336d5 100644
--- a/tests/lineage/backend/test_atlas.py
+++ b/tests/lineage/backend/test_atlas.py
@@ -25,19 +25,12 @@
 from airflow.models import DAG, TaskInstance as TI
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.utils import timezone
+from tests.compat import mock
 
 from backports.configparser import DuplicateSectionError
 
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
 
 class TestAtlas(unittest.TestCase):
     def setUp(self):
diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py
index 92b74fe869ed4..451d1f32eb35c 100644
--- a/tests/lineage/test_lineage.py
+++ b/tests/lineage/test_lineage.py
@@ -23,17 +23,10 @@
 from airflow.models import DAG, TaskInstance as TI
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.utils import timezone
+from tests.compat import mock
 
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
 
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
-
 
 class TestLineage(unittest.TestCase):
 
diff --git a/tests/macros/__init__.py b/tests/macros/__init__.py
index db5ba598d7c23..b7f8352944d3f 100644
--- a/tests/macros/__init__.py
+++ b/tests/macros/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/macros/test_hive.py b/tests/macros/test_hive.py
index eaa5050f0ddab..c3c4506b0bc61 100644
--- a/tests/macros/test_hive.py
+++ b/tests/macros/test_hive.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from datetime import datetime, timedelta
+from datetime import datetime
 import unittest
 
 from airflow.macros import hive
@@ -33,13 +33,13 @@ def test_closest_ds_partition(self):
         target_dt = datetime.strptime('2017-04-27', '%Y-%m-%d')
         date_list = [d1, d2, d3, d4, d5]
 
-        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list, True)))
-        self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, date_list, False)))
+        self.assertEqual("2017-04-26", str(hive._closest_date(target_dt, date_list, True)))
+        self.assertEqual("2017-04-28", str(hive._closest_date(target_dt, date_list, False)))
 
         # when before is not set, the closest date should be returned
-        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, [d1, d2, d3, d5], None)))
-        self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, [d1, d2, d4, d5])))
-        self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list)))
+        self.assertEqual("2017-04-26", str(hive._closest_date(target_dt, [d1, d2, d3, d5], None)))
+        self.assertEqual("2017-04-28", str(hive._closest_date(target_dt, [d1, d2, d4, d5])))
+        self.assertEqual("2017-04-26", str(hive._closest_date(target_dt, date_list)))
 
 
 if __name__ == '__main__':
diff --git a/tests/models/__init__.py b/tests/models/__init__.py
new file mode 100644
index 0000000000000..114d189da14ab
--- /dev/null
+++ b/tests/models/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py
new file mode 100644
index 0000000000000..939869e31b688
--- /dev/null
+++ b/tests/models/test_connection.py
@@ -0,0 +1,276 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from collections import namedtuple
+
+from cryptography.fernet import Fernet
+from mock import patch
+from parameterized import parameterized
+
+from airflow import models
+from airflow.models.connection import Connection
+
+ConnectionParts = namedtuple("ConnectionParts", ["conn_type", "login", "password", "host", "port", "schema"])
+
+
+class ConnectionTest(unittest.TestCase):
+    def setUp(self):
+        models._fernet = None
+
+    def tearDown(self):
+        models._fernet = None
+
+    @patch('airflow.models.configuration.conf.get')
+    def test_connection_extra_no_encryption(self, mock_get):
+        """
+        Tests extras on a new connection without encryption. The fernet key
+        is set to a non-base64-encoded string and the extra is stored without
+        encryption.
+        """
+        mock_get.return_value = ''
+        test_connection = Connection(extra='testextra')
+        self.assertFalse(test_connection.is_extra_encrypted)
+        self.assertEqual(test_connection.extra, 'testextra')
+
+    @patch('airflow.models.configuration.conf.get')
+    def test_connection_extra_with_encryption(self, mock_get):
+        """
+        Tests extras on a new connection with encryption.
+        """
+        mock_get.return_value = Fernet.generate_key().decode()
+        test_connection = Connection(extra='testextra')
+        self.assertTrue(test_connection.is_extra_encrypted)
+        self.assertEqual(test_connection.extra, 'testextra')
+
+    @patch('airflow.models.configuration.conf.get')
+    def test_connection_extra_with_encryption_rotate_fernet_key(self, mock_get):
+        """
+        Tests rotating encrypted extras.
+        """
+        key1 = Fernet.generate_key()
+        key2 = Fernet.generate_key()
+
+        mock_get.return_value = key1.decode()
+        test_connection = Connection(extra='testextra')
+        self.assertTrue(test_connection.is_extra_encrypted)
+        self.assertEqual(test_connection.extra, 'testextra')
+        self.assertEqual(Fernet(key1).decrypt(test_connection._extra.encode()), b'testextra')
+
+        # Test decrypt of old value with new key
+        mock_get.return_value = ','.join([key2.decode(), key1.decode()])
+        models._fernet = None
+        self.assertEqual(test_connection.extra, 'testextra')
+
+        # Test decrypt of new value with new key
+        test_connection.rotate_fernet_key()
+        self.assertTrue(test_connection.is_extra_encrypted)
+        self.assertEqual(test_connection.extra, 'testextra')
+        self.assertEqual(Fernet(key2).decrypt(test_connection._extra.encode()), b'testextra')
+
+    def test_connection_from_uri_without_extras(self):
+        uri = 'scheme://user:password@host%2flocation:1234/schema'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host/location')
+        self.assertEqual(connection.schema, 'schema')
+        self.assertEqual(connection.login, 'user')
+        self.assertEqual(connection.password, 'password')
+        self.assertEqual(connection.port, 1234)
+        self.assertIsNone(connection.extra)
+
+    def test_connection_from_uri_with_extras(self):
+        uri = 'scheme://user:password@host%2flocation:1234/schema?' \
+              'extra1=a%20value&extra2=%2fpath%2f'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host/location')
+        self.assertEqual(connection.schema, 'schema')
+        self.assertEqual(connection.login, 'user')
+        self.assertEqual(connection.password, 'password')
+        self.assertEqual(connection.port, 1234)
+        self.assertDictEqual(connection.extra_dejson, {'extra1': 'a value',
+                                                       'extra2': '/path/'})
+
+    def test_connection_from_uri_with_colon_in_hostname(self):
+        uri = 'scheme://user:password@host%2flocation%3ax%3ay:1234/schema?' \
+              'extra1=a%20value&extra2=%2fpath%2f'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host/location:x:y')
+        self.assertEqual(connection.schema, 'schema')
+        self.assertEqual(connection.login, 'user')
+        self.assertEqual(connection.password, 'password')
+        self.assertEqual(connection.port, 1234)
+        self.assertDictEqual(connection.extra_dejson, {'extra1': 'a value',
+                                                       'extra2': '/path/'})
+
+    def test_connection_from_uri_with_encoded_password(self):
+        uri = 'scheme://user:password%20with%20space@host%2flocation%3ax%3ay:1234/schema'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host/location:x:y')
+        self.assertEqual(connection.schema, 'schema')
+        self.assertEqual(connection.login, 'user')
+        self.assertEqual(connection.password, 'password with space')
+        self.assertEqual(connection.port, 1234)
+
+    def test_connection_from_uri_with_encoded_user(self):
+        uri = 'scheme://domain%2fuser:password@host%2flocation%3ax%3ay:1234/schema'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host/location:x:y')
+        self.assertEqual(connection.schema, 'schema')
+        self.assertEqual(connection.login, 'domain/user')
+        self.assertEqual(connection.password, 'password')
+        self.assertEqual(connection.port, 1234)
+
+    def test_connection_from_uri_with_encoded_schema(self):
+        uri = 'scheme://user:password%20with%20space@host:1234/schema%2ftest'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host')
+        self.assertEqual(connection.schema, 'schema/test')
+        self.assertEqual(connection.login, 'user')
+        self.assertEqual(connection.password, 'password with space')
+        self.assertEqual(connection.port, 1234)
+
+    def test_connection_from_uri_no_schema(self):
+        uri = 'scheme://user:password%20with%20space@host:1234'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host')
+        self.assertEqual(connection.schema, '')
+        self.assertEqual(connection.login, 'user')
+        self.assertEqual(connection.password, 'password with space')
+        self.assertEqual(connection.port, 1234)
+
+    def test_connection_from_uri_with_underscore(self):
+        uri = 'google-cloud-platform://?extra__google_cloud_platform__key_' \
+              'path=%2Fkeys%2Fkey.json&extra__google_cloud_platform__scope=' \
+              'https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform&extra' \
+              '__google_cloud_platform__project=airflow'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'google_cloud_platform')
+        self.assertEqual(connection.host, '')
+        self.assertEqual(connection.schema, '')
+        self.assertEqual(connection.login, None)
+        self.assertEqual(connection.password, None)
+        self.assertEqual(connection.extra_dejson, dict(
+            extra__google_cloud_platform__key_path='/keys/key.json',
+            extra__google_cloud_platform__project='airflow',
+            extra__google_cloud_platform__scope='https://www.googleapis.com/'
+                                                'auth/cloud-platform'))
+
+    def test_connection_from_uri_without_authinfo(self):
+        uri = 'scheme://host:1234'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, 'host')
+        self.assertEqual(connection.schema, '')
+        self.assertEqual(connection.login, None)
+        self.assertEqual(connection.password, None)
+        self.assertEqual(connection.port, 1234)
+
+    def test_connection_from_uri_with_path(self):
+        uri = 'scheme://%2FTmP%2F:1234'
+        connection = Connection(uri=uri)
+        self.assertEqual(connection.conn_type, 'scheme')
+        self.assertEqual(connection.host, '/TmP/')
+        self.assertEqual(connection.schema, '')
+        self.assertEqual(connection.login, None)
+        self.assertEqual(connection.password, None)
+        self.assertEqual(connection.port, 1234)
+
+    @parameterized.expand(
+        [
+            (
+                "http://:password@host:80/database",
+                ConnectionParts(
+                    conn_type="http", login='', password="password", host="host", port=80, schema="database"
+                ),
+            ),
+            (
+                "http://user:@host:80/database",
+                ConnectionParts(
+                    conn_type="http", login="user", password=None, host="host", port=80, schema="database"
+                ),
+            ),
+            (
+                "http://user:password@/database",
+                ConnectionParts(
+                    conn_type="http", login="user", password="password", host="", port=None, schema="database"
+                ),
+            ),
+            (
+                "http://user:password@host:80/",
+                ConnectionParts(
+                    conn_type="http", login="user", password="password", host="host", port=80, schema=""
+                ),
+            ),
+            (
+                "http://user:password@/",
+                ConnectionParts(
+                    conn_type="http", login="user", password="password", host="", port=None, schema=""
+                ),
+            ),
+            (
+                "postgresql://user:password@%2Ftmp%2Fz6rqdzqh%2Fexample%3Awest1%3Atestdb/testdb",
+                ConnectionParts(
+                    conn_type="postgres",
+                    login="user",
+                    password="password",
+                    host="/tmp/z6rqdzqh/example:west1:testdb",
+                    port=None,
+                    schema="testdb",
+                ),
+            ),
+            (
+                "postgresql://user@%2Ftmp%2Fz6rqdzqh%2Fexample%3Aeurope-west1%3Atestdb/testdb",
+                ConnectionParts(
+                    conn_type="postgres",
+                    login="user",
+                    password=None,
+                    host="/tmp/z6rqdzqh/example:europe-west1:testdb",
+                    port=None,
+                    schema="testdb",
+                ),
+            ),
+            (
+                "postgresql://%2Ftmp%2Fz6rqdzqh%2Fexample%3Aeurope-west1%3Atestdb",
+                ConnectionParts(
+                    conn_type="postgres",
+                    login=None,
+                    password=None,
+                    host="/tmp/z6rqdzqh/example:europe-west1:testdb",
+                    port=None,
+                    schema="",
+                ),
+            ),
+        ]
+    )
+    def test_connection_from_with_auth_info(self, uri, uri_parts):
+        connection = Connection(uri=uri)
+
+        self.assertEqual(connection.conn_type, uri_parts.conn_type)
+        self.assertEqual(connection.login, uri_parts.login)
+        self.assertEqual(connection.password, uri_parts.password)
+        self.assertEqual(connection.host, uri_parts.host)
+        self.assertEqual(connection.port, uri_parts.port)
+        self.assertEqual(connection.schema, uri_parts.schema)
diff --git a/tests/models/test_kubernetes.py b/tests/models/test_kubernetes.py
new file mode 100644
index 0000000000000..e01a88784cd4a
--- /dev/null
+++ b/tests/models/test_kubernetes.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from mock import patch
+
+from airflow import settings
+from airflow.models.kubernetes import KubeResourceVersion, KubeWorkerIdentifier
+
+
+class TestKubeResourceVersion(unittest.TestCase):
+
+    def test_checkpoint_resource_version(self):
+        session = settings.Session()
+        KubeResourceVersion.checkpoint_resource_version('7', session)
+        self.assertEqual(KubeResourceVersion.get_current_resource_version(session), '7')
+
+    def test_reset_resource_version(self):
+        session = settings.Session()
+        version = KubeResourceVersion.reset_resource_version(session)
+        self.assertEqual(version, '0')
+        self.assertEqual(KubeResourceVersion.get_current_resource_version(session), '0')
+
+
+class TestKubeWorkerIdentifier(unittest.TestCase):
+
+    @patch('airflow.models.kubernetes.uuid.uuid4')
+    def test_get_or_create_not_exist(self, mock_uuid):
+        session = settings.Session()
+        session.query(KubeWorkerIdentifier).update({
+            KubeWorkerIdentifier.worker_uuid: ''
+        })
+        mock_uuid.return_value = 'abcde'
+        worker_uuid = KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid(session)
+        self.assertEqual(worker_uuid, 'abcde')
+
+    def test_get_or_create_exist(self):
+        session = settings.Session()
+        KubeWorkerIdentifier.checkpoint_kube_worker_uuid('fghij', session)
+        worker_uuid = KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid(session)
+        self.assertEqual(worker_uuid, 'fghij')
diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py
new file mode 100644
index 0000000000000..9b1230409dc2c
--- /dev/null
+++ b/tests/models/test_skipmixin.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import unittest
+
+import pendulum
+from mock import patch, Mock
+
+from airflow.models import settings, DAG, TaskInstance as TI
+from airflow.models.skipmixin import SkipMixin
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.utils import timezone
+from airflow.utils.state import State
+
+DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+
+
+class TestSkipMixin(unittest.TestCase):
+
+    @patch('airflow.models.timezone.utcnow')
+    def test_skip(self, mock_now):
+        session = settings.Session()
+        now = datetime.datetime.utcnow().replace(tzinfo=pendulum.timezone('UTC'))
+        mock_now.return_value = now
+        dag = DAG(
+            'dag',
+            start_date=DEFAULT_DATE,
+        )
+        with dag:
+            tasks = [DummyOperator(task_id='task')]
+        dag_run = dag.create_dagrun(
+            run_id='manual__' + now.isoformat(),
+            state=State.FAILED,
+        )
+        SkipMixin().skip(
+            dag_run=dag_run,
+            execution_date=now,
+            tasks=tasks,
+            session=session)
+
+        session.query(TI).filter(
+            TI.dag_id == 'dag',
+            TI.task_id == 'task',
+            TI.state == State.SKIPPED,
+            TI.start_date == now,
+            TI.end_date == now,
+        ).one()
+
+    @patch('airflow.models.timezone.utcnow')
+    def test_skip_none_dagrun(self, mock_now):
+        session = settings.Session()
+        now = datetime.datetime.utcnow().replace(tzinfo=pendulum.timezone('UTC'))
+        mock_now.return_value = now
+        dag = DAG(
+            'dag',
+            start_date=DEFAULT_DATE,
+        )
+        with dag:
+            tasks = [DummyOperator(task_id='task')]
+        SkipMixin().skip(
+            dag_run=None,
+            execution_date=now,
+            tasks=tasks,
+            session=session)
+
+        session.query(TI).filter(
+            TI.dag_id == 'dag',
+            TI.task_id == 'task',
+            TI.state == State.SKIPPED,
+            TI.start_date == now,
+            TI.end_date == now,
+        ).one()
+
+    def test_skip_none_tasks(self):
+        session = Mock()
+        SkipMixin().skip(dag_run=None, execution_date=None, tasks=[], session=session)
+        self.assertFalse(session.query.called)
+        self.assertFalse(session.commit.called)
diff --git a/tests/operators/__init__.py b/tests/operators/__init__.py
index b94e5a9b6ac4a..114d189da14ab 100644
--- a/tests/operators/__init__.py
+++ b/tests/operators/__init__.py
@@ -7,21 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from .docker_operator import *
-from .subdag_operator import *
-from .operators import *
-from .hive_operator import *
-from .s3_to_hive_operator import *
-from .python_operator import *
-from .latest_only_operator import *
-
diff --git a/tests/operators/python_operator.py b/tests/operators/python_operator.py
deleted file mode 100644
index e44cfa42027b3..0000000000000
--- a/tests/operators/python_operator.py
+++ /dev/null
@@ -1,270 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import print_function, unicode_literals
-
-import datetime
-import unittest
-
-from airflow import configuration, DAG
-from airflow.models import TaskInstance as TI
-from airflow.operators.python_operator import PythonOperator, BranchPythonOperator
-from airflow.operators.python_operator import ShortCircuitOperator
-from airflow.operators.dummy_operator import DummyOperator
-from airflow.settings import Session
-from airflow.utils import timezone
-from airflow.utils.state import State
-
-from airflow.exceptions import AirflowException
-import logging
-
-DEFAULT_DATE = timezone.datetime(2016, 1, 1)
-END_DATE = timezone.datetime(2016, 1, 2)
-INTERVAL = datetime.timedelta(hours=12)
-FROZEN_NOW = timezone.datetime(2016, 1, 2, 12, 1, 1)
-
-
-class PythonOperatorTest(unittest.TestCase):
-
-    def setUp(self):
-        super(PythonOperatorTest, self).setUp()
-        configuration.load_test_config()
-        self.dag = DAG(
-            'test_dag',
-            default_args={
-                'owner': 'airflow',
-                'start_date': DEFAULT_DATE},
-            schedule_interval=INTERVAL)
-        self.addCleanup(self.dag.clear)
-        self.clear_run()
-        self.addCleanup(self.clear_run)
-
-    def do_run(self):
-        self.run = True
-
-    def clear_run(self):
-        self.run = False
-
-    def is_run(self):
-        return self.run
-
-    def test_python_operator_run(self):
-        """Tests that the python callable is invoked on task run."""
-        task = PythonOperator(
-            python_callable=self.do_run,
-            task_id='python_operator',
-            dag=self.dag)
-        self.assertFalse(self.is_run())
-        task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-        self.assertTrue(self.is_run())
-
-    def test_python_operator_python_callable_is_callable(self):
-        """Tests that PythonOperator will only instantiate if
-        the python_callable argument is callable."""
-        not_callable = {}
-        with self.assertRaises(AirflowException):
-            PythonOperator(
-                python_callable=not_callable,
-                task_id='python_operator',
-                dag=self.dag)
-        not_callable = None
-        with self.assertRaises(AirflowException):
-            PythonOperator(
-                python_callable=not_callable,
-                task_id='python_operator',
-                dag=self.dag)
-
-
-class BranchOperatorTest(unittest.TestCase):
-    def setUp(self):
-        self.dag = DAG('branch_operator_test',
-                       default_args={
-                           'owner': 'airflow',
-                           'start_date': DEFAULT_DATE},
-                       schedule_interval=INTERVAL)
-        self.branch_op = BranchPythonOperator(task_id='make_choice',
-                                              dag=self.dag,
-                                              python_callable=lambda: 'branch_1')
-
-        self.branch_1 = DummyOperator(task_id='branch_1', dag=self.dag)
-        self.branch_1.set_upstream(self.branch_op)
-        self.branch_2 = DummyOperator(task_id='branch_2', dag=self.dag)
-        self.branch_2.set_upstream(self.branch_op)
-        self.dag.clear()
-
-    def test_without_dag_run(self):
-        """This checks the defensive against non existent tasks in a dag run"""
-        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-
-        session = Session()
-        tis = session.query(TI).filter(
-            TI.dag_id == self.dag.dag_id,
-            TI.execution_date == DEFAULT_DATE
-        )
-        session.close()
-
-        for ti in tis:
-            if ti.task_id == 'make_choice':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'branch_1':
-                # should exist with state None
-                self.assertEquals(ti.state, State.NONE)
-            elif ti.task_id == 'branch_2':
-                self.assertEquals(ti.state, State.SKIPPED)
-            else:
-                raise
-
-    def test_with_dag_run(self):
-        dr = self.dag.create_dagrun(
-            run_id="manual__",
-            start_date=timezone.utcnow(),
-            execution_date=DEFAULT_DATE,
-            state=State.RUNNING
-        )
-
-        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-
-        tis = dr.get_task_instances()
-        for ti in tis:
-            if ti.task_id == 'make_choice':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'branch_1':
-                self.assertEquals(ti.state, State.NONE)
-            elif ti.task_id == 'branch_2':
-                self.assertEquals(ti.state, State.SKIPPED)
-            else:
-                raise
-
-
-class ShortCircuitOperatorTest(unittest.TestCase):
-    def test_without_dag_run(self):
-        """This checks the defensive against non existent tasks in a dag run"""
-        value = False
-        dag = DAG('shortcircuit_operator_test_without_dag_run',
-                  default_args={
-                       'owner': 'airflow',
-                       'start_date': DEFAULT_DATE
-                  },
-                  schedule_interval=INTERVAL)
-        short_op = ShortCircuitOperator(task_id='make_choice',
-                                        dag=dag,
-                                        python_callable=lambda: value)
-        branch_1 = DummyOperator(task_id='branch_1', dag=dag)
-        branch_1.set_upstream(short_op)
-        branch_2 = DummyOperator(task_id='branch_2', dag=dag)
-        branch_2.set_upstream(branch_1)
-        upstream = DummyOperator(task_id='upstream', dag=dag)
-        upstream.set_downstream(short_op)
-        dag.clear()
-
-        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-
-        session = Session()
-        tis = session.query(TI).filter(
-            TI.dag_id == dag.dag_id,
-            TI.execution_date == DEFAULT_DATE
-        )
-
-        for ti in tis:
-            if ti.task_id == 'make_choice':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'upstream':
-                # should not exist
-                raise
-            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
-                self.assertEquals(ti.state, State.SKIPPED)
-            else:
-                raise
-
-        value = True
-        dag.clear()
-
-        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-        for ti in tis:
-            if ti.task_id == 'make_choice':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'upstream':
-                # should not exist
-                raise
-            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
-                self.assertEquals(ti.state, State.NONE)
-            else:
-                raise
-
-        session.close()
-
-    def test_with_dag_run(self):
-        value = False
-        dag = DAG('shortcircuit_operator_test_with_dag_run',
-                  default_args={
-                       'owner': 'airflow',
-                       'start_date': DEFAULT_DATE
-                  },
-                  schedule_interval=INTERVAL)
-        short_op = ShortCircuitOperator(task_id='make_choice',
-                                        dag=dag,
-                                        python_callable=lambda: value)
-        branch_1 = DummyOperator(task_id='branch_1', dag=dag)
-        branch_1.set_upstream(short_op)
-        branch_2 = DummyOperator(task_id='branch_2', dag=dag)
-        branch_2.set_upstream(branch_1)
-        upstream = DummyOperator(task_id='upstream', dag=dag)
-        upstream.set_downstream(short_op)
-        dag.clear()
-
-        logging.error("Tasks {}".format(dag.tasks))
-        dr = dag.create_dagrun(
-            run_id="manual__",
-            start_date=timezone.utcnow(),
-            execution_date=DEFAULT_DATE,
-            state=State.RUNNING
-        )
-
-        upstream.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-
-        tis = dr.get_task_instances()
-        self.assertEqual(len(tis), 4)
-        for ti in tis:
-            if ti.task_id == 'make_choice':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'upstream':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
-                self.assertEquals(ti.state, State.SKIPPED)
-            else:
-                raise
-
-        value = True
-        dag.clear()
-        dr.verify_integrity()
-        upstream.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-
-        tis = dr.get_task_instances()
-        self.assertEqual(len(tis), 4)
-        for ti in tis:
-            if ti.task_id == 'make_choice':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'upstream':
-                self.assertEquals(ti.state, State.SUCCESS)
-            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
-                self.assertEquals(ti.state, State.NONE)
-            else:
-                raise
diff --git a/tests/operators/bash_operator.py b/tests/operators/test_bash_operator.py
similarity index 56%
rename from tests/operators/bash_operator.py
rename to tests/operators/test_bash_operator.py
index 1ce77e93e7ccb..aa62cd1f94ab6 100644
--- a/tests/operators/bash_operator.py
+++ b/tests/operators/test_bash_operator.py
@@ -1,19 +1,24 @@
 # -*- coding: utf-8 -*-
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
-import unittest
 import os
+import unittest
 from datetime import datetime, timedelta
 
 from airflow import DAG
@@ -59,8 +64,15 @@ def test_echo_env_variables(self):
                 task_id='echo_env_vars',
                 dag=self.dag,
                 bash_command='echo $AIRFLOW_HOME>> {0};'
-                             'echo $PYTHONPATH>> {0};'.format(fname)
+                             'echo $PYTHONPATH>> {0};'
+                             'echo $AIRFLOW_CTX_DAG_ID >> {0};'
+                             'echo $AIRFLOW_CTX_TASK_ID>> {0};'
+                             'echo $AIRFLOW_CTX_EXECUTION_DATE>> {0};'
+                             'echo $AIRFLOW_CTX_DAG_RUN_ID>> {0};'.format(fname)
             )
+
+            original_AIRFLOW_HOME = os.environ['AIRFLOW_HOME']
+
             os.environ['AIRFLOW_HOME'] = 'MY_PATH_TO_AIRFLOW_HOME'
             t.run(DEFAULT_DATE, DEFAULT_DATE,
                   ignore_first_depends_on_past=True, ignore_ti_state=True)
@@ -70,3 +82,9 @@ def test_echo_env_variables(self):
                 self.assertIn('MY_PATH_TO_AIRFLOW_HOME', output)
                 # exported in run_unit_tests.sh as part of PYTHONPATH
                 self.assertIn('tests/test_utils', output)
+                self.assertIn('bash_op_test', output)
+                self.assertIn('echo_env_vars', output)
+                self.assertIn(DEFAULT_DATE.isoformat(), output)
+                self.assertIn('manual__' + DEFAULT_DATE.isoformat(), output)
+
+            os.environ['AIRFLOW_HOME'] = original_AIRFLOW_HOME
diff --git a/tests/operators/test_check_operator.py b/tests/operators/test_check_operator.py
index 46a48e42d233b..eeec061715aa0 100644
--- a/tests/operators/test_check_operator.py
+++ b/tests/operators/test_check_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,14 +22,7 @@
 from airflow.models import DAG
 from airflow.exceptions import AirflowException
 from airflow.operators.check_operator import ValueCheckOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class ValueCheckOperatorTest(unittest.TestCase):
diff --git a/tests/operators/docker_operator.py b/tests/operators/test_docker_operator.py
similarity index 91%
rename from tests/operators/docker_operator.py
rename to tests/operators/test_docker_operator.py
index 78a920c30cc13..9747a1b96ad50 100644
--- a/tests/operators/docker_operator.py
+++ b/tests/operators/test_docker_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,14 +28,7 @@
     pass
 
 from airflow.exceptions import AirflowException
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class DockerOperatorTestCase(unittest.TestCase):
@@ -51,7 +44,7 @@ def test_execute(self, client_class_mock, mkdtemp_mock):
         client_mock.images.return_value = []
         client_mock.logs.return_value = ['container log']
         client_mock.pull.return_value = [b'{"status":"pull log"}']
-        client_mock.wait.return_value = 0
+        client_mock.wait.return_value = {"StatusCode": 0}
 
         client_class_mock.return_value = client_mock
 
@@ -64,20 +57,25 @@ def test_execute(self, client_class_mock, mkdtemp_mock):
         client_class_mock.assert_called_with(base_url='unix://var/run/docker.sock', tls=None,
                                              version='1.19')
 
-        client_mock.create_container.assert_called_with(command='env', cpu_shares=1024,
+        client_mock.create_container.assert_called_with(command='env',
                                                         environment={
                                                             'AIRFLOW_TMP_DIR': '/tmp/airflow',
                                                             'UNIT': 'TEST'
                                                         },
                                                         host_config=host_config,
                                                         image='ubuntu:latest',
-                                                        mem_limit=None, user=None,
+                                                        user=None,
                                                         working_dir='/container/path'
                                                         )
         client_mock.create_host_config.assert_called_with(binds=['/host/path:/container/path',
                                                                  '/mkdtemp:/tmp/airflow'],
                                                           network_mode='bridge',
-                                                          shm_size=1000)
+                                                          shm_size=1000,
+                                                          cpu_shares=1024,
+                                                          mem_limit=None,
+                                                          auto_remove=False,
+                                                          dns=None,
+                                                          dns_search=None)
         client_mock.images.assert_called_with(name='ubuntu:latest')
         client_mock.logs.assert_called_with(container='some_id', stream=True)
         client_mock.pull.assert_called_with('ubuntu:latest', stream=True)
@@ -92,7 +90,7 @@ def test_execute_tls(self, client_class_mock, tls_class_mock):
         client_mock.images.return_value = []
         client_mock.logs.return_value = []
         client_mock.pull.return_value = []
-        client_mock.wait.return_value = 0
+        client_mock.wait.return_value = {"StatusCode": 0}
 
         client_class_mock.return_value = client_mock
         tls_mock = mock.Mock()
@@ -118,7 +116,7 @@ def test_execute_unicode_logs(self, client_class_mock):
         client_mock.images.return_value = []
         client_mock.logs.return_value = ['unicode container log 😁']
         client_mock.pull.return_value = []
-        client_mock.wait.return_value = 0
+        client_mock.wait.return_value = {"StatusCode": 0}
 
         client_class_mock.return_value = client_mock
 
@@ -140,7 +138,7 @@ def test_execute_container_fails(self, client_class_mock):
         client_mock.images.return_value = []
         client_mock.logs.return_value = []
         client_mock.pull.return_value = []
-        client_mock.wait.return_value = 1
+        client_mock.wait.return_value = {"StatusCode": 1}
 
         client_class_mock.return_value = client_mock
 
@@ -149,7 +147,8 @@ def test_execute_container_fails(self, client_class_mock):
         with self.assertRaises(AirflowException):
             operator.execute(None)
 
-    def test_on_kill(self):
+    @staticmethod
+    def test_on_kill():
         client_mock = mock.Mock(spec=APIClient)
 
         operator = DockerOperator(image='ubuntu', owner='unittest', task_id='unittest')
@@ -168,7 +167,7 @@ def test_execute_no_docker_conn_id_no_hook(self, operator_client_mock):
         client_mock.create_container.return_value = {'Id': 'some_id'}
         client_mock.logs.return_value = []
         client_mock.pull.return_value = []
-        client_mock.wait.return_value = 0
+        client_mock.wait.return_value = {"StatusCode": 0}
         operator_client_mock.return_value = client_mock
 
         # Create the DockerOperator
@@ -203,7 +202,7 @@ def test_execute_with_docker_conn_id_use_hook(self, operator_client_mock,
         client_mock.create_container.return_value = {'Id': 'some_id'}
         client_mock.logs.return_value = []
         client_mock.pull.return_value = []
-        client_mock.wait.return_value = 0
+        client_mock.wait.return_value = {"StatusCode": 0}
         operator_client_mock.return_value = client_mock
 
         # Create the DockerOperator
@@ -234,5 +233,6 @@ def test_execute_with_docker_conn_id_use_hook(self, operator_client_mock,
             'Image was not pulled using operator client'
         )
 
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/operators/test_druid_check_operator.py b/tests/operators/test_druid_check_operator.py
index f64ad7e32b1d4..c18b56d97115e 100644
--- a/tests/operators/test_druid_check_operator.py
+++ b/tests/operators/test_druid_check_operator.py
@@ -24,14 +24,7 @@
 from airflow.models import DAG
 from airflow.exceptions import AirflowException
 from airflow.operators.druid_check_operator import DruidCheckOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class DruidCheckOperatorTest(unittest.TestCase):
diff --git a/tests/operators/test_email_operator.py b/tests/operators/test_email_operator.py
index a19f450a5cbe9..67baa42cb29df 100644
--- a/tests/operators/test_email_operator.py
+++ b/tests/operators/test_email_operator.py
@@ -62,4 +62,4 @@ def test_execute(self):
         configuration.conf.set('email', 'EMAIL_BACKEND',
                                'tests.operators.test_email_operator.send_email_test')
         self._run_as_operator()
-        send_email_test.assert_called_once()
+        assert send_email_test.call_count == 1
diff --git a/tests/operators/hive_operator.py b/tests/operators/test_hive_operator.py
similarity index 87%
rename from tests/operators/hive_operator.py
rename to tests/operators/test_hive_operator.py
index 1569e7f7e9538..9f13ab43942f8 100644
--- a/tests/operators/hive_operator.py
+++ b/tests/operators/test_hive_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,11 +22,10 @@
 import datetime
 import os
 import unittest
-import mock
 import nose
-import six
 
-from airflow import DAG, configuration, operators
+from airflow import DAG, configuration
+import airflow.operators.hive_operator
 configuration.load_test_config()
 
 
@@ -61,7 +60,7 @@ def setUp(self):
 class HiveOperatorConfigTest(HiveEnvironmentTest):
 
     def test_hive_airflow_default_config_queue(self):
-        t = operators.hive_operator.HiveOperator(
+        t = airflow.operators.hive_operator.HiveOperator(
             task_id='test_default_config_queue',
             hql=self.hql,
             mapred_queue_priority='HIGH',
@@ -77,7 +76,7 @@ def test_hive_airflow_default_config_queue(self):
 
     def test_hive_airflow_default_config_queue_override(self):
         specific_mapred_queue = 'default'
-        t = operators.hive_operator.HiveOperator(
+        t = airflow.operators.hive_operator.HiveOperator(
             task_id='test_default_config_queue',
             hql=self.hql,
             mapred_queue=specific_mapred_queue,
@@ -92,7 +91,7 @@ class HiveOperatorTest(HiveEnvironmentTest):
 
     def test_hiveconf_jinja_translate(self):
         hql = "SELECT ${num_col} FROM ${hiveconf:table};"
-        t = operators.hive_operator.HiveOperator(
+        t = airflow.operators.hive_operator.HiveOperator(
             hiveconf_jinja_translate=True,
             task_id='dry_run_basic_hql', hql=hql, dag=self.dag)
         t.prepare_template()
@@ -100,7 +99,7 @@ def test_hiveconf_jinja_translate(self):
 
     def test_hiveconf(self):
         hql = "SELECT * FROM ${hiveconf:table} PARTITION (${hiveconf:day});"
-        t = operators.hive_operator.HiveOperator(
+        t = airflow.operators.hive_operator.HiveOperator(
             hiveconfs={'table': 'static_babynames', 'day': '{{ ds }}'},
             task_id='dry_run_basic_hql', hql=hql, dag=self.dag)
         t.prepare_template()
@@ -117,13 +116,13 @@ def test_hiveconf(self):
     class HivePrestoTest(HiveEnvironmentTest):
 
         def test_hive(self):
-            t = operators.hive_operator.HiveOperator(
+            t = airflow.operators.hive_operator.HiveOperator(
                 task_id='basic_hql', hql=self.hql, dag=self.dag)
             t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE,
                   ignore_ti_state=True)
 
         def test_hive_queues(self):
-            t = operators.hive_operator.HiveOperator(
+            t = airflow.operators.hive_operator.HiveOperator(
                 task_id='test_hive_queues', hql=self.hql,
                 mapred_queue='default', mapred_queue_priority='HIGH',
                 mapred_job_name='airflow.test_hive_queues',
@@ -132,12 +131,12 @@ def test_hive_queues(self):
                   ignore_ti_state=True)
 
         def test_hive_dryrun(self):
-            t = operators.hive_operator.HiveOperator(
+            t = airflow.operators.hive_operator.HiveOperator(
                 task_id='dry_run_basic_hql', hql=self.hql, dag=self.dag)
             t.dry_run()
 
         def test_beeline(self):
-            t = operators.hive_operator.HiveOperator(
+            t = airflow.operators.hive_operator.HiveOperator(
                 task_id='beeline_hql', hive_cli_conn_id='beeline_default',
                 hql=self.hql, dag=self.dag)
             t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE,
@@ -147,13 +146,13 @@ def test_presto(self):
             sql = """
             SELECT count(1) FROM airflow.static_babynames_partitioned;
             """
-            t = operators.presto_check_operator.PrestoCheckOperator(
+            t = airflow.operators.presto_check_operator.PrestoCheckOperator(
                 task_id='presto_check', sql=sql, dag=self.dag)
             t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE,
                   ignore_ti_state=True)
 
         def test_presto_to_mysql(self):
-            t = operators.presto_to_mysql.PrestoToMySqlTransfer(
+            t = airflow.operators.presto_to_mysql.PrestoToMySqlTransfer(
                 task_id='presto_to_mysql_check',
                 sql="""
                 SELECT name, count(*) as ccount
@@ -167,7 +166,7 @@ def test_presto_to_mysql(self):
                   ignore_ti_state=True)
 
         def test_hdfs_sensor(self):
-            t = operators.sensors.HdfsSensor(
+            t = airflow.operators.sensors.HdfsSensor(
                 task_id='hdfs_sensor_check',
                 filepath='hdfs://user/hive/warehouse/airflow.db/static_babynames',
                 dag=self.dag)
@@ -175,7 +174,7 @@ def test_hdfs_sensor(self):
                   ignore_ti_state=True)
 
         def test_webhdfs_sensor(self):
-            t = operators.sensors.WebHdfsSensor(
+            t = airflow.operators.sensors.WebHdfsSensor(
                 task_id='webhdfs_sensor_check',
                 filepath='hdfs://user/hive/warehouse/airflow.db/static_babynames',
                 timeout=120,
@@ -184,7 +183,7 @@ def test_webhdfs_sensor(self):
                   ignore_ti_state=True)
 
         def test_sql_sensor(self):
-            t = operators.sensors.SqlSensor(
+            t = airflow.operators.sensors.SqlSensor(
                 task_id='hdfs_sensor_check',
                 conn_id='presto_default',
                 sql="SELECT 'x' FROM airflow.static_babynames LIMIT 1;",
@@ -193,7 +192,7 @@ def test_sql_sensor(self):
                   ignore_ti_state=True)
 
         def test_hive_stats(self):
-            t = operators.hive_stats_operator.HiveStatsCollectionOperator(
+            t = airflow.operators.hive_stats_operator.HiveStatsCollectionOperator(
                 task_id='hive_stats_check',
                 table="airflow.static_babynames_partitioned",
                 partition={'ds': DEFAULT_DATE_DS},
@@ -202,7 +201,7 @@ def test_hive_stats(self):
                   ignore_ti_state=True)
 
         def test_named_hive_partition_sensor(self):
-            t = operators.sensors.NamedHivePartitionSensor(
+            t = airflow.operators.sensors.NamedHivePartitionSensor(
                 task_id='hive_partition_check',
                 partition_names=[
                     "airflow.static_babynames_partitioned/ds={{ds}}"
@@ -212,7 +211,7 @@ def test_named_hive_partition_sensor(self):
                   ignore_ti_state=True)
 
         def test_named_hive_partition_sensor_succeeds_on_multiple_partitions(self):
-            t = operators.sensors.NamedHivePartitionSensor(
+            t = airflow.operators.sensors.NamedHivePartitionSensor(
                 task_id='hive_partition_check',
                 partition_names=[
                     "airflow.static_babynames_partitioned/ds={{ds}}",
@@ -223,7 +222,7 @@ def test_named_hive_partition_sensor_succeeds_on_multiple_partitions(self):
                   ignore_ti_state=True)
 
         def test_named_hive_partition_sensor_parses_partitions_with_periods(self):
-            t = operators.sensors.NamedHivePartitionSensor.parse_partition_name(
+            t = airflow.operators.sensors.NamedHivePartitionSensor.parse_partition_name(
                 partition="schema.table/part1=this.can.be.an.issue/part2=ok")
             self.assertEqual(t[0], "schema")
             self.assertEqual(t[1], "table")
@@ -231,7 +230,7 @@ def test_named_hive_partition_sensor_parses_partitions_with_periods(self):
 
         @nose.tools.raises(airflow.exceptions.AirflowSensorTimeout)
         def test_named_hive_partition_sensor_times_out_on_nonexistent_partition(self):
-            t = operators.sensors.NamedHivePartitionSensor(
+            t = airflow.operators.sensors.NamedHivePartitionSensor(
                 task_id='hive_partition_check',
                 partition_names=[
                     "airflow.static_babynames_partitioned/ds={{ds}}",
@@ -244,7 +243,7 @@ def test_named_hive_partition_sensor_times_out_on_nonexistent_partition(self):
                   ignore_ti_state=True)
 
         def test_hive_partition_sensor(self):
-            t = operators.sensors.HivePartitionSensor(
+            t = airflow.operators.sensors.HivePartitionSensor(
                 task_id='hive_partition_check',
                 table='airflow.static_babynames_partitioned',
                 dag=self.dag)
@@ -252,7 +251,7 @@ def test_hive_partition_sensor(self):
                   ignore_ti_state=True)
 
         def test_hive_metastore_sql_sensor(self):
-            t = operators.sensors.MetastorePartitionSensor(
+            t = airflow.operators.sensors.MetastorePartitionSensor(
                 task_id='hive_partition_check',
                 table='airflow.static_babynames_partitioned',
                 partition_name='ds={}'.format(DEFAULT_DATE_DS),
@@ -261,7 +260,7 @@ def test_hive_metastore_sql_sensor(self):
                   ignore_ti_state=True)
 
         def test_hive2samba(self):
-            t = operators.hive_to_samba_operator.Hive2SambaOperator(
+            t = airflow.operators.hive_to_samba_operator.Hive2SambaOperator(
                 task_id='hive2samba_check',
                 samba_conn_id='tableau_samba',
                 hql="SELECT * FROM airflow.static_babynames LIMIT 10000",
@@ -271,7 +270,7 @@ def test_hive2samba(self):
                   ignore_ti_state=True)
 
         def test_hive_to_mysql(self):
-            t = operators.hive_to_mysql.HiveToMySqlTransfer(
+            t = airflow.operators.hive_to_mysql.HiveToMySqlTransfer(
                 mysql_conn_id='airflow_db',
                 task_id='hive_to_mysql_check',
                 create=True,
diff --git a/tests/operators/test_hive_to_druid.py b/tests/operators/test_hive_to_druid.py
index 06e9596a74f26..b657e292a8bc2 100644
--- a/tests/operators/test_hive_to_druid.py
+++ b/tests/operators/test_hive_to_druid.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -47,7 +47,12 @@ class TestDruidHook(unittest.TestCase):
         'num_shards': -1,
         'target_partition_size': 1925,
         'query_granularity': 'month',
-        'segment_granularity': 'week'
+        'segment_granularity': 'week',
+        'job_properties': {
+            "mapreduce.job.user.classpath.first": "false",
+            "mapreduce.map.output.compress": "false",
+            "mapreduce.output.fileoutputformat.compress": "false"
+        }
     }
 
     index_spec_config = {
@@ -112,11 +117,7 @@ def test_construct_ingest_query(self):
                 },
                 "tuningConfig": {
                     "type": "hadoop",
-                    "jobProperties": {
-                        "mapreduce.job.user.classpath.first": "false",
-                        "mapreduce.map.output.compress": "false",
-                        "mapreduce.output.fileoutputformat.compress": "false",
-                    },
+                    "jobProperties": self.hook_config['job_properties'],
                     "partitionsSpec": {
                         "type": "hashed",
                         "targetPartitionSize": self.hook_config['target_partition_size'],
diff --git a/tests/operators/test_http_operator.py b/tests/operators/test_http_operator.py
new file mode 100644
index 0000000000000..12fc6529752b3
--- /dev/null
+++ b/tests/operators/test_http_operator.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import unittest
+
+import requests_mock
+from airflow.operators.http_operator import SimpleHttpOperator
+from airflow.exceptions import AirflowException
+from tests.compat import mock
+
+
+class SimpleHttpOpTests(unittest.TestCase):
+    def setUp(self):
+        os.environ['AIRFLOW_CONN_HTTP_EXAMPLE'] = 'http://www.example.com'
+
+    @requests_mock.mock()
+    def test_response_in_logs(self, m):
+        """
+        Test that when using SimpleHttpOperator with 'GET',
+        the log contains 'Example Domain' in it
+        """
+
+        m.get('http://www.example.com', text='Example.com fake response')
+        operator = SimpleHttpOperator(
+            task_id='test_HTTP_op',
+            method='GET',
+            endpoint='/',
+            http_conn_id='HTTP_EXAMPLE',
+            log_response=True,
+        )
+
+        with mock.patch.object(operator.log, 'info') as mock_info:
+            operator.execute(None)
+            mock_info.assert_called_with('Example.com fake response')
+
+    @requests_mock.mock()
+    def test_response_in_logs_after_failed_check(self, m):
+        """
+        Test that when using SimpleHttpOperator with log_response=True,
+        the reponse is logged even if request_check fails
+        """
+
+        def response_check(response):
+            return response.text != 'invalid response'
+
+        m.get('http://www.example.com', text='invalid response')
+        operator = SimpleHttpOperator(
+            task_id='test_HTTP_op',
+            method='GET',
+            endpoint='/',
+            http_conn_id='HTTP_EXAMPLE',
+            log_response=True,
+            response_check=response_check
+        )
+
+        with mock.patch.object(operator.log, 'info') as mock_info:
+            self.assertRaises(AirflowException, operator.execute, None)
+            mock_info.assert_called_with('invalid response')
diff --git a/tests/operators/latest_only_operator.py b/tests/operators/test_latest_only_operator.py
similarity index 92%
rename from tests/operators/latest_only_operator.py
rename to tests/operators/test_latest_only_operator.py
index 3c21831f05789..7a4efa11b5af1 100644
--- a/tests/operators/latest_only_operator.py
+++ b/tests/operators/test_latest_only_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,11 +20,9 @@
 from __future__ import print_function, unicode_literals
 
 import datetime
-import logging
 import unittest
 
 from airflow import configuration, DAG, settings
-from airflow.jobs import BackfillJob
 from airflow.models import TaskInstance
 from airflow.operators.latest_only_operator import LatestOnlyOperator
 from airflow.operators.dummy_operator import DummyOperator
@@ -93,7 +91,7 @@ def test_skipping(self):
         self.assertEqual({
             timezone.datetime(2016, 1, 1): 'success',
             timezone.datetime(2016, 1, 1, 12): 'success',
-            timezone.datetime(2016, 1, 2): 'success', },
+            timezone.datetime(2016, 1, 2): 'success'},
             exec_date_to_latest_state)
 
         downstream_instances = get_task_instances('downstream')
@@ -102,7 +100,7 @@ def test_skipping(self):
         self.assertEqual({
             timezone.datetime(2016, 1, 1): 'skipped',
             timezone.datetime(2016, 1, 1, 12): 'skipped',
-            timezone.datetime(2016, 1, 2): 'success',},
+            timezone.datetime(2016, 1, 2): 'success'},
             exec_date_to_downstream_state)
 
         downstream_instances = get_task_instances('downstream_2')
@@ -111,7 +109,7 @@ def test_skipping(self):
         self.assertEqual({
             timezone.datetime(2016, 1, 1): 'skipped',
             timezone.datetime(2016, 1, 1, 12): 'skipped',
-            timezone.datetime(2016, 1, 2): 'success',},
+            timezone.datetime(2016, 1, 2): 'success'},
             exec_date_to_downstream_state)
 
     def test_skipping_dagrun(self):
@@ -128,21 +126,21 @@ def test_skipping_dagrun(self):
         downstream_task.set_upstream(latest_task)
         downstream_task2.set_upstream(downstream_task)
 
-        dr1 = self.dag.create_dagrun(
+        self.dag.create_dagrun(
             run_id="manual__1",
             start_date=timezone.utcnow(),
             execution_date=DEFAULT_DATE,
             state=State.RUNNING
         )
 
-        dr2 = self.dag.create_dagrun(
+        self.dag.create_dagrun(
             run_id="manual__2",
             start_date=timezone.utcnow(),
             execution_date=timezone.datetime(2016, 1, 1, 12),
             state=State.RUNNING
         )
 
-        dr2 = self.dag.create_dagrun(
+        self.dag.create_dagrun(
             run_id="manual__3",
             start_date=timezone.utcnow(),
             execution_date=END_DATE,
@@ -159,7 +157,7 @@ def test_skipping_dagrun(self):
         self.assertEqual({
             timezone.datetime(2016, 1, 1): 'success',
             timezone.datetime(2016, 1, 1, 12): 'success',
-            timezone.datetime(2016, 1, 2): 'success', },
+            timezone.datetime(2016, 1, 2): 'success'},
             exec_date_to_latest_state)
 
         downstream_instances = get_task_instances('downstream')
@@ -168,7 +166,7 @@ def test_skipping_dagrun(self):
         self.assertEqual({
             timezone.datetime(2016, 1, 1): 'skipped',
             timezone.datetime(2016, 1, 1, 12): 'skipped',
-            timezone.datetime(2016, 1, 2): 'success',},
+            timezone.datetime(2016, 1, 2): 'success'},
             exec_date_to_downstream_state)
 
         downstream_instances = get_task_instances('downstream_2')
@@ -177,5 +175,5 @@ def test_skipping_dagrun(self):
         self.assertEqual({
             timezone.datetime(2016, 1, 1): 'skipped',
             timezone.datetime(2016, 1, 1, 12): 'skipped',
-            timezone.datetime(2016, 1, 2): 'success',},
+            timezone.datetime(2016, 1, 2): 'success'},
             exec_date_to_downstream_state)
diff --git a/tests/operators/operators.py b/tests/operators/test_operators.py
similarity index 68%
rename from tests/operators/operators.py
rename to tests/operators/test_operators.py
index ae953253cccf8..0d817c001b755 100644
--- a/tests/operators/operators.py
+++ b/tests/operators/test_operators.py
@@ -20,14 +20,12 @@
 from __future__ import print_function
 
 from airflow import DAG, configuration, operators
-from airflow.utils.tests import skipUnlessImported
 from airflow.utils import timezone
 
 from collections import OrderedDict
 
 import os
 import mock
-import six
 import unittest
 
 configuration.load_test_config()
@@ -38,18 +36,25 @@
 TEST_DAG_ID = 'unit_test_dag'
 
 
-@skipUnlessImported('airflow.operators.mysql_operator', 'MySqlOperator')
 class MySqlTest(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
         args = {
             'owner': 'airflow',
-            'mysql_conn_id': 'airflow_db',
             'start_date': DEFAULT_DATE
         }
         dag = DAG(TEST_DAG_ID, default_args=args)
         self.dag = dag
 
+    def tearDown(self):
+        from airflow.hooks.mysql_hook import MySqlHook
+        drop_tables = {'test_mysql_to_mysql', 'test_airflow'}
+        with MySqlHook().get_conn() as conn:
+            for table in drop_tables:
+                conn.execute("DROP TABLE IF EXISTS {}".format(table))
+
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_operator_test(self):
         sql = """
         CREATE TABLE IF NOT EXISTS test_airflow (
@@ -60,22 +65,27 @@ def test_mysql_operator_test(self):
         t = MySqlOperator(
             task_id='basic_mysql',
             sql=sql,
-            mysql_conn_id='airflow_db',
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_operator_test_multi(self):
         sql = [
+            "CREATE TABLE IF NOT EXISTS test_airflow (dummy VARCHAR(50))",
             "TRUNCATE TABLE test_airflow",
             "INSERT INTO test_airflow VALUES ('X')",
         ]
         from airflow.operators.mysql_operator import MySqlOperator
         t = MySqlOperator(
             task_id='mysql_operator_test_multi',
-            mysql_conn_id='airflow_db',
-            sql=sql, dag=self.dag)
+            sql=sql,
+            dag=self.dag,
+        )
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_hook_test_bulk_load(self):
         records = ("foo", "bar", "baz")
 
@@ -85,7 +95,7 @@ def test_mysql_hook_test_bulk_load(self):
             t.flush()
 
             from airflow.hooks.mysql_hook import MySqlHook
-            h = MySqlHook('airflow_ci')
+            h = MySqlHook('airflow_db')
             with h.get_conn() as c:
                 c.execute("""
                     CREATE TABLE IF NOT EXISTS test_airflow (
@@ -98,9 +108,11 @@ def test_mysql_hook_test_bulk_load(self):
                 results = tuple(result[0] for result in c.fetchall())
                 self.assertEqual(sorted(results), sorted(records))
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_hook_test_bulk_dump(self):
         from airflow.hooks.mysql_hook import MySqlHook
-        hook = MySqlHook('airflow_ci')
+        hook = MySqlHook('airflow_db')
         priv = hook.get_first("SELECT @@global.secure_file_priv")
         if priv and priv[0]:
             # Confirm that no error occurs
@@ -109,25 +121,29 @@ def test_mysql_hook_test_bulk_dump(self):
             self.skipTest("Skip test_mysql_hook_test_bulk_load "
                           "since file output is not permitted")
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     @mock.patch('airflow.hooks.mysql_hook.MySqlHook.get_conn')
     def test_mysql_hook_test_bulk_dump_mock(self, mock_get_conn):
         mock_execute = mock.MagicMock()
         mock_get_conn.return_value.cursor.return_value.execute = mock_execute
 
         from airflow.hooks.mysql_hook import MySqlHook
-        hook = MySqlHook('airflow_ci')
+        hook = MySqlHook('airflow_db')
         table = "INFORMATION_SCHEMA.TABLES"
         tmp_file = "/path/to/output/file"
         hook.bulk_dump(table, tmp_file)
 
         from airflow.utils.tests import assertEqualIgnoreMultipleSpaces
-        mock_execute.assert_called_once()
+        assert mock_execute.call_count == 1
         query = """
             SELECT * INTO OUTFILE '{tmp_file}'
             FROM {table}
         """.format(tmp_file=tmp_file, table=table)
         assertEqualIgnoreMultipleSpaces(self, mock_execute.call_args[0][0], query)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_to_mysql(self):
         sql = "SELECT * FROM INFORMATION_SCHEMA.TABLES LIMIT 100;"
         from airflow.operators.generic_transfer import GenericTransfer
@@ -145,6 +161,8 @@ def test_mysql_to_mysql(self):
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_overwrite_schema(self):
         """
         Verifies option to overwrite connection schema
@@ -167,7 +185,6 @@ def test_overwrite_schema(self):
             assert "Unknown database 'foobar'" in str(e)
 
 
-@skipUnlessImported('airflow.operators.postgres_operator', 'PostgresOperator')
 class PostgresTest(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
@@ -175,6 +192,16 @@ def setUp(self):
         dag = DAG(TEST_DAG_ID, default_args=args)
         self.dag = dag
 
+    def tearDown(self):
+        tables_to_drop = ['test_postgres_to_postgres', 'test_airflow']
+        from airflow.hooks.postgres_hook import PostgresHook
+        with PostgresHook().get_conn() as conn:
+            with conn.cursor() as cur:
+                for t in tables_to_drop:
+                    cur.execute("DROP TABLE IF EXISTS {}".format(t))
+
+    @unittest.skipUnless('postgres' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a Postgres test")
     def test_postgres_operator_test(self):
         sql = """
         CREATE TABLE IF NOT EXISTS test_airflow (
@@ -195,8 +222,11 @@ def test_postgres_operator_test(self):
             end_date=DEFAULT_DATE,
             ignore_ti_state=True)
 
+    @unittest.skipUnless('postgres' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a Postgres test")
     def test_postgres_operator_test_multi(self):
         sql = [
+            "CREATE TABLE IF NOT EXISTS test_airflow (dummy VARCHAR(50))",
             "TRUNCATE TABLE test_airflow",
             "INSERT INTO test_airflow VALUES ('X')",
         ]
@@ -205,6 +235,8 @@ def test_postgres_operator_test_multi(self):
             task_id='postgres_operator_test_multi', sql=sql, dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('postgres' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a Postgres test")
     def test_postgres_to_postgres(self):
         sql = "SELECT * FROM INFORMATION_SCHEMA.TABLES LIMIT 100;"
         from airflow.operators.generic_transfer import GenericTransfer
@@ -222,6 +254,8 @@ def test_postgres_to_postgres(self):
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('postgres' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a Postgres test")
     def test_vacuum(self):
         """
         Verifies the VACUUM operation runs well with the PostgresOperator
@@ -236,6 +270,8 @@ def test_vacuum(self):
             autocommit=True)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('postgres' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a Postgres test")
     def test_overwrite_schema(self):
         """
         Verifies option to overwrite connection schema
@@ -259,7 +295,6 @@ def test_overwrite_schema(self):
             assert 'database "foobar" does not exist' in str(e)
 
 
-@skipUnlessImported('airflow.operators.mysql_to_hive', 'MySqlToHiveTransfer')
 class TransferTests(unittest.TestCase):
     def setUp(self):
         configuration.load_test_config()
@@ -267,17 +302,95 @@ def setUp(self):
         dag = DAG(TEST_DAG_ID, default_args=args)
         self.dag = dag
 
+        rows = [
+            (1880, "John", 0.081541, "boy"),
+            (1880, "William", 0.080511, "boy"),
+            (1880, "James", 0.050057, "boy"),
+            (1880, "Charles", 0.045167, "boy"),
+            (1880, "George", 0.043292, "boy"),
+            (1880, "Frank", 0.02738, "boy"),
+            (1880, "Joseph", 0.022229, "boy"),
+            (1880, "Thomas", 0.021401, "boy"),
+            (1880, "Henry", 0.020641, "boy"),
+            (1880, "Robert", 0.020404, "boy"),
+            (1880, "Edward", 0.019965, "boy"),
+            (1880, "Harry", 0.018175, "boy"),
+            (1880, "Walter", 0.014822, "boy"),
+            (1880, "Arthur", 0.013504, "boy"),
+            (1880, "Fred", 0.013251, "boy"),
+            (1880, "Albert", 0.012609, "boy"),
+            (1880, "Samuel", 0.008648, "boy"),
+            (1880, "David", 0.007339, "boy"),
+            (1880, "Louis", 0.006993, "boy"),
+            (1880, "Joe", 0.006174, "boy"),
+            (1880, "Charlie", 0.006165, "boy"),
+            (1880, "Clarence", 0.006165, "boy"),
+            (1880, "Richard", 0.006148, "boy"),
+            (1880, "Andrew", 0.005439, "boy"),
+            (1880, "Daniel", 0.00543, "boy"),
+            (1880, "Ernest", 0.005194, "boy"),
+            (1880, "Will", 0.004966, "boy"),
+            (1880, "Jesse", 0.004805, "boy"),
+            (1880, "Oscar", 0.004594, "boy"),
+            (1880, "Lewis", 0.004366, "boy"),
+            (1880, "Peter", 0.004189, "boy"),
+            (1880, "Benjamin", 0.004138, "boy"),
+            (1880, "Frederick", 0.004079, "boy"),
+            (1880, "Willie", 0.00402, "boy"),
+            (1880, "Alfred", 0.003961, "boy"),
+            (1880, "Sam", 0.00386, "boy"),
+            (1880, "Roy", 0.003716, "boy"),
+            (1880, "Herbert", 0.003581, "boy"),
+            (1880, "Jacob", 0.003412, "boy"),
+            (1880, "Tom", 0.00337, "boy"),
+            (1880, "Elmer", 0.00315, "boy"),
+            (1880, "Carl", 0.003142, "boy"),
+            (1880, "Lee", 0.003049, "boy"),
+            (1880, "Howard", 0.003015, "boy"),
+            (1880, "Martin", 0.003015, "boy"),
+            (1880, "Michael", 0.00299, "boy"),
+            (1880, "Bert", 0.002939, "boy"),
+            (1880, "Herman", 0.002931, "boy"),
+            (1880, "Jim", 0.002914, "boy"),
+            (1880, "Francis", 0.002905, "boy"),
+            (1880, "Harvey", 0.002905, "boy"),
+            (1880, "Earl", 0.002829, "boy"),
+            (1880, "Eugene", 0.00277, "boy"),
+        ]
+
+        from airflow.hooks.mysql_hook import MySqlHook
+        with MySqlHook().get_conn() as cur:
+            cur.execute('''
+            CREATE TABLE IF NOT EXISTS baby_names (
+              org_year integer(4),
+              baby_name VARCHAR(25),
+              rate FLOAT(7,6),
+              sex VARCHAR(4)
+            )
+            ''')
+
+        for row in rows:
+            cur.execute("INSERT INTO baby_names VALUES(%s, %s, %s, %s);", row)
+
+    def tearDown(self):
+        from airflow.hooks.mysql_hook import MySqlHook
+        with MySqlHook().get_conn() as cur:
+            cur.execute("DROP TABLE IF EXISTS baby_names CASCADE;")
+
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_clear(self):
         self.dag.clear(
             start_date=DEFAULT_DATE,
             end_date=timezone.utcnow())
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_to_hive(self):
         from airflow.operators.mysql_to_hive import MySqlToHiveTransfer
         sql = "SELECT * FROM baby_names LIMIT 1000;"
         t = MySqlToHiveTransfer(
             task_id='test_m2h',
-            mysql_conn_id='airflow_ci',
             hive_cli_conn_id='beeline_default',
             sql=sql,
             hive_table='test_mysql_to_hive',
@@ -286,12 +399,13 @@ def test_mysql_to_hive(self):
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_to_hive_partition(self):
         from airflow.operators.mysql_to_hive import MySqlToHiveTransfer
         sql = "SELECT * FROM baby_names LIMIT 1000;"
         t = MySqlToHiveTransfer(
             task_id='test_m2h',
-            mysql_conn_id='airflow_ci',
             hive_cli_conn_id='beeline_default',
             sql=sql,
             hive_table='test_mysql_to_hive_part',
@@ -302,28 +416,30 @@ def test_mysql_to_hive_partition(self):
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_to_hive_tblproperties(self):
         from airflow.operators.mysql_to_hive import MySqlToHiveTransfer
         sql = "SELECT * FROM baby_names LIMIT 1000;"
         t = MySqlToHiveTransfer(
             task_id='test_m2h',
-            mysql_conn_id='airflow_ci',
             hive_cli_conn_id='beeline_default',
             sql=sql,
             hive_table='test_mysql_to_hive',
             recreate=True,
             delimiter=",",
-            tblproperties={'test_property':'test_value'},
+            tblproperties={'test_property': 'test_value'},
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     @mock.patch('airflow.hooks.hive_hooks.HiveCliHook.load_file')
     def test_mysql_to_hive_type_conversion(self, mock_load_file):
-        mysql_conn_id = 'airflow_ci'
         mysql_table = 'test_mysql_to_hive'
 
         from airflow.hooks.mysql_hook import MySqlHook
-        m = MySqlHook(mysql_conn_id)
+        m = MySqlHook()
 
         try:
             with m.get_conn() as c:
@@ -342,14 +458,13 @@ def test_mysql_to_hive_type_conversion(self, mock_load_file):
             from airflow.operators.mysql_to_hive import MySqlToHiveTransfer
             t = MySqlToHiveTransfer(
                 task_id='test_m2h',
-                mysql_conn_id=mysql_conn_id,
                 hive_cli_conn_id='beeline_default',
                 sql="SELECT * FROM {}".format(mysql_table),
                 hive_table='test_mysql_to_hive',
                 dag=self.dag)
             t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
-            mock_load_file.assert_called_once()
+            assert mock_load_file.call_count == 1
             d = OrderedDict()
             d["c0"] = "SMALLINT"
             d["c1"] = "INT"
@@ -362,13 +477,14 @@ def test_mysql_to_hive_type_conversion(self, mock_load_file):
             with m.get_conn() as c:
                 c.execute("DROP TABLE IF EXISTS {}".format(mysql_table))
 
+    @unittest.skipUnless('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                         "This is a MySQL test")
     def test_mysql_to_hive_verify_loaded_values(self):
-        mysql_conn_id = 'airflow_ci'
         mysql_table = 'test_mysql_to_hive'
         hive_table = 'test_mysql_to_hive'
 
         from airflow.hooks.mysql_hook import MySqlHook
-        m = MySqlHook(mysql_conn_id)
+        m = MySqlHook()
 
         try:
             minmax = (
@@ -409,7 +525,6 @@ def test_mysql_to_hive_verify_loaded_values(self):
             from airflow.operators.mysql_to_hive import MySqlToHiveTransfer
             t = MySqlToHiveTransfer(
                 task_id='test_m2h',
-                mysql_conn_id=mysql_conn_id,
                 hive_cli_conn_id='beeline_default',
                 sql="SELECT * FROM {}".format(mysql_table),
                 hive_table=hive_table,
diff --git a/tests/operators/test_python_operator.py b/tests/operators/test_python_operator.py
new file mode 100644
index 0000000000000..296ef841c5deb
--- /dev/null
+++ b/tests/operators/test_python_operator.py
@@ -0,0 +1,576 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function, unicode_literals
+
+import copy
+import logging
+import os
+import unittest
+from datetime import timedelta, date
+
+from airflow import configuration
+from airflow.exceptions import AirflowException
+from airflow.models import TaskInstance as TI, DAG, DagRun
+from airflow.operators.dummy_operator import DummyOperator
+from airflow.operators.python_operator import PythonOperator, BranchPythonOperator
+from airflow.operators.python_operator import ShortCircuitOperator
+from airflow.settings import Session
+from airflow.utils import timezone
+from airflow.utils.state import State
+
+DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+END_DATE = timezone.datetime(2016, 1, 2)
+INTERVAL = timedelta(hours=12)
+FROZEN_NOW = timezone.datetime(2016, 1, 2, 12, 1, 1)
+
+TI_CONTEXT_ENV_VARS = ['AIRFLOW_CTX_DAG_ID',
+                       'AIRFLOW_CTX_TASK_ID',
+                       'AIRFLOW_CTX_EXECUTION_DATE',
+                       'AIRFLOW_CTX_DAG_RUN_ID']
+
+
+class Call:
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+
+
+def build_recording_function(calls_collection):
+    """
+    We can not use a Mock instance as a PythonOperator callable function or some tests fail with a
+    TypeError: Object of type Mock is not JSON serializable
+    Then using this custom function recording custom Call objects for further testing
+    (replacing Mock.assert_called_with assertion method)
+    """
+    def recording_function(*args, **kwargs):
+        calls_collection.append(Call(*args, **kwargs))
+    return recording_function
+
+
+class PythonOperatorTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(PythonOperatorTest, cls).setUpClass()
+
+        session = Session()
+
+        session.query(DagRun).delete()
+        session.query(TI).delete()
+        session.commit()
+        session.close()
+
+    def setUp(self):
+        super(PythonOperatorTest, self).setUp()
+        configuration.load_test_config()
+        self.dag = DAG(
+            'test_dag',
+            default_args={
+                'owner': 'airflow',
+                'start_date': DEFAULT_DATE},
+            schedule_interval=INTERVAL)
+        self.addCleanup(self.dag.clear)
+        self.clear_run()
+        self.addCleanup(self.clear_run)
+
+    def tearDown(self):
+        super(PythonOperatorTest, self).tearDown()
+
+        session = Session()
+
+        session.query(DagRun).delete()
+        session.query(TI).delete()
+        print(len(session.query(DagRun).all()))
+        session.commit()
+        session.close()
+
+        for var in TI_CONTEXT_ENV_VARS:
+            if var in os.environ:
+                del os.environ[var]
+
+    def do_run(self):
+        self.run = True
+
+    def clear_run(self):
+        self.run = False
+
+    def is_run(self):
+        return self.run
+
+    def test_python_operator_run(self):
+        """Tests that the python callable is invoked on task run."""
+        task = PythonOperator(
+            python_callable=self.do_run,
+            task_id='python_operator',
+            dag=self.dag)
+        self.assertFalse(self.is_run())
+        task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+        self.assertTrue(self.is_run())
+
+    def test_python_operator_python_callable_is_callable(self):
+        """Tests that PythonOperator will only instantiate if
+        the python_callable argument is callable."""
+        not_callable = {}
+        with self.assertRaises(AirflowException):
+            PythonOperator(
+                python_callable=not_callable,
+                task_id='python_operator',
+                dag=self.dag)
+        not_callable = None
+        with self.assertRaises(AirflowException):
+            PythonOperator(
+                python_callable=not_callable,
+                task_id='python_operator',
+                dag=self.dag)
+
+    def _assertCallsEqual(self, first, second):
+        self.assertIsInstance(first, Call)
+        self.assertIsInstance(second, Call)
+        self.assertTupleEqual(first.args, second.args)
+        self.assertDictEqual(first.kwargs, second.kwargs)
+
+    def test_python_callable_arguments_are_templatized(self):
+        """Test PythonOperator op_args are templatized"""
+        recorded_calls = []
+
+        task = PythonOperator(
+            task_id='python_operator',
+            # a Mock instance cannot be used as a callable function or test fails with a
+            # TypeError: Object of type Mock is not JSON serializable
+            python_callable=(build_recording_function(recorded_calls)),
+            op_args=[
+                4,
+                date(2019, 1, 1),
+                "dag {{dag.dag_id}} ran on {{ds}}."
+            ],
+            dag=self.dag)
+
+        self.dag.create_dagrun(
+            run_id='manual__' + DEFAULT_DATE.isoformat(),
+            execution_date=DEFAULT_DATE,
+            start_date=DEFAULT_DATE,
+            state=State.RUNNING
+        )
+        task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        self.assertEqual(1, len(recorded_calls))
+        self._assertCallsEqual(
+            recorded_calls[0],
+            Call(4,
+                 date(2019, 1, 1),
+                 "dag {} ran on {}.".format(self.dag.dag_id, DEFAULT_DATE.date().isoformat()))
+        )
+
+    def test_python_callable_keyword_arguments_are_templatized(self):
+        """Test PythonOperator op_kwargs are templatized"""
+        recorded_calls = []
+
+        task = PythonOperator(
+            task_id='python_operator',
+            # a Mock instance cannot be used as a callable function or test fails with a
+            # TypeError: Object of type Mock is not JSON serializable
+            python_callable=(build_recording_function(recorded_calls)),
+            op_kwargs={
+                'an_int': 4,
+                'a_date': date(2019, 1, 1),
+                'a_templated_string': "dag {{dag.dag_id}} ran on {{ds}}."
+            },
+            dag=self.dag)
+
+        self.dag.create_dagrun(
+            run_id='manual__' + DEFAULT_DATE.isoformat(),
+            execution_date=DEFAULT_DATE,
+            start_date=DEFAULT_DATE,
+            state=State.RUNNING
+        )
+        task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        self.assertEqual(1, len(recorded_calls))
+        self._assertCallsEqual(
+            recorded_calls[0],
+            Call(an_int=4,
+                 a_date=date(2019, 1, 1),
+                 a_templated_string="dag {} ran on {}.".format(
+                     self.dag.dag_id, DEFAULT_DATE.date().isoformat()))
+        )
+
+    def test_python_operator_shallow_copy_attr(self):
+        not_callable = lambda x: x
+        original_task = PythonOperator(
+            python_callable=not_callable,
+            task_id='python_operator',
+            op_kwargs={'certain_attrs': ''},
+            dag=self.dag
+        )
+        new_task = copy.deepcopy(original_task)
+        # shallow copy op_kwargs
+        self.assertEqual(id(original_task.op_kwargs['certain_attrs']),
+                         id(new_task.op_kwargs['certain_attrs']))
+        # shallow copy python_callable
+        self.assertEqual(id(original_task.python_callable),
+                         id(new_task.python_callable))
+
+    def _env_var_check_callback(self):
+        self.assertEqual('test_dag', os.environ['AIRFLOW_CTX_DAG_ID'])
+        self.assertEqual('hive_in_python_op', os.environ['AIRFLOW_CTX_TASK_ID'])
+        self.assertEqual(DEFAULT_DATE.isoformat(),
+                         os.environ['AIRFLOW_CTX_EXECUTION_DATE'])
+        self.assertEqual('manual__' + DEFAULT_DATE.isoformat(),
+                         os.environ['AIRFLOW_CTX_DAG_RUN_ID'])
+
+    def test_echo_env_variables(self):
+        """
+        Test that env variables are exported correctly to the
+        python callback in the task.
+        """
+        self.dag.create_dagrun(
+            run_id='manual__' + DEFAULT_DATE.isoformat(),
+            execution_date=DEFAULT_DATE,
+            start_date=DEFAULT_DATE,
+            state=State.RUNNING,
+            external_trigger=False,
+        )
+
+        t = PythonOperator(task_id='hive_in_python_op',
+                           dag=self.dag,
+                           python_callable=self._env_var_check_callback
+                           )
+        t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+
+class BranchOperatorTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(BranchOperatorTest, cls).setUpClass()
+
+        session = Session()
+
+        session.query(DagRun).delete()
+        session.query(TI).delete()
+        session.commit()
+        session.close()
+
+    def setUp(self):
+        self.dag = DAG('branch_operator_test',
+                       default_args={
+                           'owner': 'airflow',
+                           'start_date': DEFAULT_DATE},
+                       schedule_interval=INTERVAL)
+
+        self.branch_1 = DummyOperator(task_id='branch_1', dag=self.dag)
+        self.branch_2 = DummyOperator(task_id='branch_2', dag=self.dag)
+
+    def tearDown(self):
+        super(BranchOperatorTest, self).tearDown()
+
+        session = Session()
+
+        session.query(DagRun).delete()
+        session.query(TI).delete()
+        print(len(session.query(DagRun).all()))
+        session.commit()
+        session.close()
+
+    def test_without_dag_run(self):
+        """This checks the defensive against non existent tasks in a dag run"""
+        self.branch_op = BranchPythonOperator(task_id='make_choice',
+                                              dag=self.dag,
+                                              python_callable=lambda: 'branch_1')
+        self.branch_1.set_upstream(self.branch_op)
+        self.branch_2.set_upstream(self.branch_op)
+        self.dag.clear()
+
+        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        session = Session()
+        tis = session.query(TI).filter(
+            TI.dag_id == self.dag.dag_id,
+            TI.execution_date == DEFAULT_DATE
+        )
+        session.close()
+
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'branch_1':
+                # should exist with state None
+                self.assertEqual(ti.state, State.NONE)
+            elif ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.SKIPPED)
+            else:
+                raise
+
+    def test_branch_list_without_dag_run(self):
+        """This checks if the BranchPythonOperator supports branching off to a list of tasks."""
+        self.branch_op = BranchPythonOperator(task_id='make_choice',
+                                              dag=self.dag,
+                                              python_callable=lambda: ['branch_1', 'branch_2'])
+        self.branch_1.set_upstream(self.branch_op)
+        self.branch_2.set_upstream(self.branch_op)
+        self.branch_3 = DummyOperator(task_id='branch_3', dag=self.dag)
+        self.branch_3.set_upstream(self.branch_op)
+        self.dag.clear()
+
+        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        session = Session()
+        tis = session.query(TI).filter(
+            TI.dag_id == self.dag.dag_id,
+            TI.execution_date == DEFAULT_DATE
+        )
+        session.close()
+
+        expected = {
+            "make_choice": State.SUCCESS,
+            "branch_1": State.NONE,
+            "branch_2": State.NONE,
+            "branch_3": State.SKIPPED,
+        }
+
+        for ti in tis:
+            if ti.task_id in expected:
+                self.assertEqual(ti.state, expected[ti.task_id])
+            else:
+                raise
+
+    def test_with_dag_run(self):
+        self.branch_op = BranchPythonOperator(task_id='make_choice',
+                                              dag=self.dag,
+                                              python_callable=lambda: 'branch_1')
+
+        self.branch_1.set_upstream(self.branch_op)
+        self.branch_2.set_upstream(self.branch_op)
+        self.dag.clear()
+
+        dr = self.dag.create_dagrun(
+            run_id="manual__",
+            start_date=timezone.utcnow(),
+            execution_date=DEFAULT_DATE,
+            state=State.RUNNING
+        )
+
+        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        tis = dr.get_task_instances()
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'branch_1':
+                self.assertEqual(ti.state, State.NONE)
+            elif ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.SKIPPED)
+            else:
+                raise
+
+    def test_with_skip_in_branch_downstream_dependencies(self):
+        self.branch_op = BranchPythonOperator(task_id='make_choice',
+                                              dag=self.dag,
+                                              python_callable=lambda: 'branch_1')
+
+        self.branch_op >> self.branch_1 >> self.branch_2
+        self.branch_op >> self.branch_2
+        self.dag.clear()
+
+        dr = self.dag.create_dagrun(
+            run_id="manual__",
+            start_date=timezone.utcnow(),
+            execution_date=DEFAULT_DATE,
+            state=State.RUNNING
+        )
+
+        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        tis = dr.get_task_instances()
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'branch_1':
+                self.assertEqual(ti.state, State.NONE)
+            elif ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.NONE)
+            else:
+                raise Exception
+
+    def test_with_skip_in_branch_downstream_dependencies2(self):
+        self.branch_op = BranchPythonOperator(task_id='make_choice',
+                                              dag=self.dag,
+                                              python_callable=lambda: 'branch_2')
+
+        self.branch_op >> self.branch_1 >> self.branch_2
+        self.branch_op >> self.branch_2
+        self.dag.clear()
+
+        dr = self.dag.create_dagrun(
+            run_id="manual__",
+            start_date=timezone.utcnow(),
+            execution_date=DEFAULT_DATE,
+            state=State.RUNNING
+        )
+
+        self.branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        tis = dr.get_task_instances()
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'branch_1':
+                self.assertEqual(ti.state, State.SKIPPED)
+            elif ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.NONE)
+            else:
+                raise Exception
+
+
+class ShortCircuitOperatorTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(ShortCircuitOperatorTest, cls).setUpClass()
+
+        session = Session()
+
+        session.query(DagRun).delete()
+        session.query(TI).delete()
+        session.commit()
+        session.close()
+
+    def tearDown(self):
+        super(ShortCircuitOperatorTest, self).tearDown()
+
+        session = Session()
+
+        session.query(DagRun).delete()
+        session.query(TI).delete()
+        session.commit()
+        session.close()
+
+    def test_without_dag_run(self):
+        """This checks the defensive against non existent tasks in a dag run"""
+        value = False
+        dag = DAG('shortcircuit_operator_test_without_dag_run',
+                  default_args={
+                      'owner': 'airflow',
+                      'start_date': DEFAULT_DATE
+                  },
+                  schedule_interval=INTERVAL)
+        short_op = ShortCircuitOperator(task_id='make_choice',
+                                        dag=dag,
+                                        python_callable=lambda: value)
+        branch_1 = DummyOperator(task_id='branch_1', dag=dag)
+        branch_1.set_upstream(short_op)
+        branch_2 = DummyOperator(task_id='branch_2', dag=dag)
+        branch_2.set_upstream(branch_1)
+        upstream = DummyOperator(task_id='upstream', dag=dag)
+        upstream.set_downstream(short_op)
+        dag.clear()
+
+        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        session = Session()
+        tis = session.query(TI).filter(
+            TI.dag_id == dag.dag_id,
+            TI.execution_date == DEFAULT_DATE
+        )
+
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'upstream':
+                # should not exist
+                raise
+            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.SKIPPED)
+            else:
+                raise
+
+        value = True
+        dag.clear()
+
+        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'upstream':
+                # should not exist
+                raise
+            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.NONE)
+            else:
+                raise
+
+        session.close()
+
+    def test_with_dag_run(self):
+        value = False
+        dag = DAG('shortcircuit_operator_test_with_dag_run',
+                  default_args={
+                      'owner': 'airflow',
+                      'start_date': DEFAULT_DATE
+                  },
+                  schedule_interval=INTERVAL)
+        short_op = ShortCircuitOperator(task_id='make_choice',
+                                        dag=dag,
+                                        python_callable=lambda: value)
+        branch_1 = DummyOperator(task_id='branch_1', dag=dag)
+        branch_1.set_upstream(short_op)
+        branch_2 = DummyOperator(task_id='branch_2', dag=dag)
+        branch_2.set_upstream(branch_1)
+        upstream = DummyOperator(task_id='upstream', dag=dag)
+        upstream.set_downstream(short_op)
+        dag.clear()
+
+        logging.error("Tasks {}".format(dag.tasks))
+        dr = dag.create_dagrun(
+            run_id="manual__",
+            start_date=timezone.utcnow(),
+            execution_date=DEFAULT_DATE,
+            state=State.RUNNING
+        )
+
+        upstream.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 4)
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'upstream':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.SKIPPED)
+            else:
+                raise
+
+        value = True
+        dag.clear()
+        dr.verify_integrity()
+        upstream.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+        short_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
+
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 4)
+        for ti in tis:
+            if ti.task_id == 'make_choice':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'upstream':
+                self.assertEqual(ti.state, State.SUCCESS)
+            elif ti.task_id == 'branch_1' or ti.task_id == 'branch_2':
+                self.assertEqual(ti.state, State.NONE)
+            else:
+                raise
diff --git a/tests/operators/test_redshift_to_s3_operator.py b/tests/operators/test_redshift_to_s3_operator.py
index 379e78744e6cf..0b4309214f227 100644
--- a/tests/operators/test_redshift_to_s3_operator.py
+++ b/tests/operators/test_redshift_to_s3_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -92,8 +92,8 @@ def test_execute(self, mock_get_conn, mock_run, mock_Session):
                            secret_key=secret_key,
                            unload_options=unload_options)
 
-        cur.execute.assert_called_once()
+        assert cur.execute.call_count == 1
         assertEqualIgnoreMultipleSpaces(self, cur.execute.call_args[0][0], columns_query)
 
-        mock_run.assert_called_once()
+        assert mock_run.call_count == 1
         assertEqualIgnoreMultipleSpaces(self, mock_run.call_args[0][0], unload_query)
diff --git a/tests/operators/test_s3_file_transform_operator.py b/tests/operators/test_s3_file_transform_operator.py
index 6b72c58f7857f..da7a4b0bed91f 100644
--- a/tests/operators/test_s3_file_transform_operator.py
+++ b/tests/operators/test_s3_file_transform_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,6 +22,7 @@
 import io
 import os
 import shutil
+import sys
 import unittest
 from tempfile import mkdtemp
 
@@ -29,6 +30,7 @@
 import mock
 from moto import mock_s3
 
+from airflow.exceptions import AirflowException
 from airflow.operators.s3_file_transform_operator import S3FileTransformOperator
 
 
@@ -48,11 +50,15 @@ def tearDown(self):
                 raise e
 
     @mock.patch('subprocess.Popen')
+    @mock.patch.object(S3FileTransformOperator, 'log')
     @mock_s3
-    def test_execute_with_transform_script(self, mock_Popen):
-        transform_script_process = mock_Popen.return_value
-        transform_script_process.communicate.return_value = [None, None]
-        transform_script_process.returncode = 0
+    def test_execute_with_transform_script(self, mock_log, mock_Popen):
+        process_output = [b"Foo", b"Bar", b"Baz"]
+
+        process = mock_Popen.return_value
+        process.stdout.readline.side_effect = process_output
+        process.wait.return_value = None
+        process.returncode = 0
 
         bucket = "bucket"
         input_key = "foo"
@@ -72,6 +78,40 @@ def test_execute_with_transform_script(self, mock_Popen):
             task_id="task_id")
         t.execute(None)
 
+        mock_log.info.assert_has_calls([
+            mock.call(line.decode(sys.getdefaultencoding())) for line in process_output
+        ])
+
+    @mock.patch('subprocess.Popen')
+    @mock_s3
+    def test_execute_with_failing_transform_script(self, mock_Popen):
+        process = mock_Popen.return_value
+        process.stdout.readline.side_effect = []
+        process.wait.return_value = None
+        process.returncode = 42
+
+        bucket = "bucket"
+        input_key = "foo"
+        output_key = "bar"
+        bio = io.BytesIO(b"input")
+
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket=bucket)
+        conn.upload_fileobj(Bucket=bucket, Key=input_key, Fileobj=bio)
+
+        s3_url = "s3://{0}/{1}"
+        t = S3FileTransformOperator(
+            source_s3_key=s3_url.format(bucket, input_key),
+            dest_s3_key=s3_url.format(bucket, output_key),
+            transform_script=self.transform_script,
+            replace=True,
+            task_id="task_id")
+
+        with self.assertRaises(AirflowException) as e:
+            t.execute(None)
+
+        self.assertEqual('Transform script failed: 42', str(e.exception))
+
     @mock.patch('airflow.hooks.S3_hook.S3Hook.select_key', return_value="input")
     @mock_s3
     def test_execute_with_select_expression(self, mock_select_key):
diff --git a/tests/operators/s3_to_hive_operator.py b/tests/operators/test_s3_to_hive_operator.py
similarity index 84%
rename from tests/operators/s3_to_hive_operator.py
rename to tests/operators/test_s3_to_hive_operator.py
index 482e7fefc82ff..8366465a9ebd8 100644
--- a/tests/operators/s3_to_hive_operator.py
+++ b/tests/operators/test_s3_to_hive_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,20 +18,15 @@
 # under the License.
 
 import unittest
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+
+from tests.compat import mock
 import logging
 from itertools import product
 from airflow.operators.s3_to_hive_operator import S3ToHiveTransfer
 from collections import OrderedDict
 from airflow.exceptions import AirflowException
 from tempfile import NamedTemporaryFile, mkdtemp
-import gzip
+from gzip import GzipFile
 import bz2
 import shutil
 import filecmp
@@ -85,29 +80,31 @@ def setUp(self):
                 self._set_fn(f_txt_h.name, '.txt', True)
                 f_txt_h.writelines([header, line1, line2])
             fn_gz = self._get_fn('.txt', True) + ".gz"
-            with gzip.GzipFile(filename=fn_gz,
-                               mode="wb") as f_gz_h:
+            with GzipFile(filename=fn_gz, mode="wb") as f_gz_h:
                 self._set_fn(fn_gz, '.gz', True)
                 f_gz_h.writelines([header, line1, line2])
+            fn_gz_upper = self._get_fn('.txt', True) + ".GZ"
+            with GzipFile(filename=fn_gz_upper, mode="wb") as f_gz_upper_h:
+                self._set_fn(fn_gz_upper, '.GZ', True)
+                f_gz_upper_h.writelines([header, line1, line2])
             fn_bz2 = self._get_fn('.txt', True) + '.bz2'
-            with bz2.BZ2File(filename=fn_bz2,
-                             mode="wb") as f_bz2_h:
+            with bz2.BZ2File(filename=fn_bz2, mode="wb") as f_bz2_h:
                 self._set_fn(fn_bz2, '.bz2', True)
                 f_bz2_h.writelines([header, line1, line2])
             # create sample txt, bz and bz2 without header
-            with NamedTemporaryFile(mode='wb+',
-                                    dir=self.tmp_dir,
-                                    delete=False) as f_txt_nh:
+            with NamedTemporaryFile(mode='wb+', dir=self.tmp_dir, delete=False) as f_txt_nh:
                 self._set_fn(f_txt_nh.name, '.txt', False)
                 f_txt_nh.writelines([line1, line2])
             fn_gz = self._get_fn('.txt', False) + ".gz"
-            with gzip.GzipFile(filename=fn_gz,
-                               mode="wb") as f_gz_nh:
+            with GzipFile(filename=fn_gz, mode="wb") as f_gz_nh:
                 self._set_fn(fn_gz, '.gz', False)
                 f_gz_nh.writelines([line1, line2])
+            fn_gz_upper = self._get_fn('.txt', False) + ".GZ"
+            with GzipFile(filename=fn_gz_upper, mode="wb") as f_gz_upper_nh:
+                self._set_fn(fn_gz_upper, '.GZ', False)
+                f_gz_upper_nh.writelines([line1, line2])
             fn_bz2 = self._get_fn('.txt', False) + '.bz2'
-            with bz2.BZ2File(filename=fn_bz2,
-                             mode="wb") as f_bz2_nh:
+            with bz2.BZ2File(filename=fn_bz2, mode="wb") as f_bz2_nh:
                 self._set_fn(fn_bz2, '.bz2', False)
                 f_bz2_nh.writelines([line1, line2])
         # Base Exception so it catches Keyboard Interrupt
@@ -135,24 +132,24 @@ def _get_fn(self, ext, header):
         key = self._get_key(ext, header)
         return self.fn[key]
 
-    def _get_key(self, ext, header):
+    @staticmethod
+    def _get_key(ext, header):
         key = ext + "_" + ('h' if header else 'nh')
         return key
 
-    def _check_file_equality(self, fn_1, fn_2, ext):
+    @staticmethod
+    def _check_file_equality(fn_1, fn_2, ext):
         # gz files contain mtime and filename in the header that
         # causes filecmp to return False even if contents are identical
         # Hence decompress to test for equality
-        if(ext == '.gz'):
-            with gzip.GzipFile(fn_1, 'rb') as f_1,\
-                 NamedTemporaryFile(mode='wb') as f_txt_1,\
-                 gzip.GzipFile(fn_2, 'rb') as f_2,\
-                 NamedTemporaryFile(mode='wb') as f_txt_2:
-                shutil.copyfileobj(f_1, f_txt_1)
-                shutil.copyfileobj(f_2, f_txt_2)
-                f_txt_1.flush()
-                f_txt_2.flush()
-                return filecmp.cmp(f_txt_1.name, f_txt_2.name, shallow=False)
+        if ext.lower() == '.gz':
+            with GzipFile(fn_1, 'rb') as f_1, NamedTemporaryFile(mode='wb') as f_txt_1:
+                with GzipFile(fn_2, 'rb') as f_2, NamedTemporaryFile(mode='wb') as f_txt_2:
+                    shutil.copyfileobj(f_1, f_txt_1)
+                    shutil.copyfileobj(f_2, f_txt_2)
+                    f_txt_1.flush()
+                    f_txt_2.flush()
+                    return filecmp.cmp(f_txt_1.name, f_txt_2.name, shallow=False)
         else:
             return filecmp.cmp(fn_1, fn_2, shallow=False)
 
@@ -167,20 +164,20 @@ def test_bad_parameters(self):
     def test__get_top_row_as_list(self):
         self.kwargs['delimiter'] = '\t'
         fn_txt = self._get_fn('.txt', True)
-        header_list = S3ToHiveTransfer(**self.kwargs).\
+        header_list = S3ToHiveTransfer(**self.kwargs). \
             _get_top_row_as_list(fn_txt)
         self.assertEqual(header_list, ['Sno', 'Some,Text'],
                          msg="Top row from file doesnt matched expected value")
 
         self.kwargs['delimiter'] = ','
-        header_list = S3ToHiveTransfer(**self.kwargs).\
+        header_list = S3ToHiveTransfer(**self.kwargs). \
             _get_top_row_as_list(fn_txt)
         self.assertEqual(header_list, ['Sno\tSome', 'Text'],
                          msg="Top row from file doesnt matched expected value")
 
     def test__match_headers(self):
         self.kwargs['field_dict'] = OrderedDict([('Sno', 'BIGINT'),
-                                                ('Some,Text', 'STRING')])
+                                                 ('Some,Text', 'STRING')])
         self.assertTrue(S3ToHiveTransfer(**self.kwargs).
                         _match_headers(['Sno', 'Some,Text']),
                         msg="Header row doesnt match expected value")
@@ -220,14 +217,14 @@ def test_execute(self, mock_hiveclihook):
         conn.create_bucket(Bucket='bucket')
 
         # Testing txt, zip, bz2 files with and without header row
-        for (ext, has_header) in product(['.txt', '.gz', '.bz2'], [True, False]):
+        for (ext, has_header) in product(['.txt', '.gz', '.bz2', '.GZ'], [True, False]):
             self.kwargs['headers'] = has_header
             self.kwargs['check_headers'] = has_header
             logging.info("Testing {0} format {1} header".
                          format(ext,
                                 ('with' if has_header else 'without'))
                          )
-            self.kwargs['input_compressed'] = ext != '.txt'
+            self.kwargs['input_compressed'] = ext.lower() != '.txt'
             self.kwargs['s3_key'] = 's3://bucket/' + self.s3_key + ext
             ip_fn = self._get_fn(ext, self.kwargs['headers'])
             op_fn = self._get_fn(ext, False)
@@ -238,8 +235,7 @@ def test_execute(self, mock_hiveclihook):
             # file parameter to HiveCliHook.load_file is compared
             # against expected file output
             mock_hiveclihook().load_file.side_effect = \
-                lambda *args, **kwargs: \
-                self.assertTrue(
+                lambda *args, **kwargs: self.assertTrue(
                     self._check_file_equality(args[0], op_fn, ext),
                     msg='{0} output file not as expected'.format(ext))
             # Execute S3ToHiveTransfer
@@ -260,8 +256,8 @@ def test_execute_with_select_expression(self, mock_hiveclihook):
         # Only testing S3ToHiveTransfer calls S3Hook.select_key with
         # the right parameters and its execute method succeeds here,
         # since Moto doesn't support select_object_content as of 1.3.2.
-        for (ext, has_header) in product(['.txt', '.gz'], [True, False]):
-            input_compressed = ext != '.txt'
+        for (ext, has_header) in product(['.txt', '.gz', '.GZ'], [True, False]):
+            input_compressed = ext.lower() != '.txt'
             key = self.s3_key + ext
 
             self.kwargs['check_headers'] = False
diff --git a/tests/operators/test_s3_to_redshift_operator.py b/tests/operators/test_s3_to_redshift_operator.py
index 2afde6c95d7f3..8c0175311562b 100644
--- a/tests/operators/test_s3_to_redshift_operator.py
+++ b/tests/operators/test_s3_to_redshift_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -67,5 +67,5 @@ def test_execute(self, mock_run, mock_Session):
                    secret_key=secret_key,
                    copy_options=copy_options)
 
-        mock_run.assert_called_once()
+        assert mock_run.call_count == 1
         assertEqualIgnoreMultipleSpaces(self, mock_run.call_args[0][0], copy_query)
diff --git a/tests/operators/slack_operator.py b/tests/operators/test_slack_operator.py
similarity index 97%
rename from tests/operators/slack_operator.py
rename to tests/operators/test_slack_operator.py
index ad916429772a0..08434f9622e72 100644
--- a/tests/operators/slack_operator.py
+++ b/tests/operators/test_slack_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,14 +21,7 @@
 import json
 from airflow.exceptions import AirflowException
 from airflow.operators.slack_operator import SlackAPIPostOperator
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 
 class SlackAPIPostOperatorTestCase(unittest.TestCase):
diff --git a/tests/operators/subdag_operator.py b/tests/operators/test_subdag_operator.py
similarity index 96%
rename from tests/operators/subdag_operator.py
rename to tests/operators/test_subdag_operator.py
index af47c5cfd56f5..bba6f17194de2 100644
--- a/tests/operators/subdag_operator.py
+++ b/tests/operators/test_subdag_operator.py
@@ -85,7 +85,7 @@ def test_subdag_pools(self):
         session.add(pool_10)
         session.commit()
 
-        dummy_1 = DummyOperator(task_id='dummy', dag=subdag, pool='test_pool_1')
+        DummyOperator(task_id='dummy', dag=subdag, pool='test_pool_1')
 
         self.assertRaises(
             AirflowException,
@@ -116,8 +116,7 @@ def test_subdag_pools_no_possible_conflict(self):
         session.add(pool_10)
         session.commit()
 
-        dummy_1 = DummyOperator(
-            task_id='dummy', dag=subdag, pool='test_pool_10')
+        DummyOperator(task_id='dummy', dag=subdag, pool='test_pool_10')
 
         mock_session = Mock()
         SubDagOperator(
diff --git a/tests/operators/test_virtualenv_operator.py b/tests/operators/test_virtualenv_operator.py
index 82abe9c8098de..dc83f2206569e 100644
--- a/tests/operators/test_virtualenv_operator.py
+++ b/tests/operators/test_virtualenv_operator.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,13 +28,10 @@
 from subprocess import CalledProcessError
 
 from airflow import configuration, DAG
-from airflow.models import TaskInstance
 from airflow.operators.python_operator import PythonVirtualenvOperator
-from airflow.settings import Session
 from airflow.utils import timezone
 
 from airflow.exceptions import AirflowException
-import logging
 
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
 END_DATE = timezone.datetime(2016, 1, 2)
@@ -55,9 +52,10 @@ def setUp(self):
             schedule_interval=INTERVAL)
         self.addCleanup(self.dag.clear)
 
-    def _run_as_operator(self, fn, **kwargs):
+    def _run_as_operator(self, fn, python_version=sys.version_info[0], **kwargs):
         task = PythonVirtualenvOperator(
             python_callable=fn,
+            python_version=python_version,
             task_id='task',
             dag=self.dag,
             **kwargs)
@@ -83,7 +81,7 @@ def f():
     def test_no_system_site_packages(self):
         def f():
             try:
-                import funcsigs
+                import funcsigs  # noqa: F401
             except ImportError:
                 return True
             raise Exception
@@ -91,26 +89,31 @@ def f():
 
     def test_system_site_packages(self):
         def f():
-            import funcsigs
+            import funcsigs  # noqa: F401
         self._run_as_operator(f, requirements=['funcsigs'], system_site_packages=True)
 
     def test_with_requirements_pinned(self):
-        self.assertNotEqual('0.4', funcsigs.__version__, 'Please update this string if this fails')
+        self.assertNotEqual(
+            '0.4', funcsigs.__version__, 'Please update this string if this fails')
+
         def f():
-            import funcsigs
+            import funcsigs  # noqa: F401
             if funcsigs.__version__ != '0.4':
                 raise Exception
+
         self._run_as_operator(f, requirements=['funcsigs==0.4'])
 
     def test_unpinned_requirements(self):
         def f():
-            import funcsigs
-        self._run_as_operator(f, requirements=['funcsigs', 'dill'], system_site_packages=False)
+            import funcsigs  # noqa: F401
+        self._run_as_operator(
+            f, requirements=['funcsigs', 'dill'], system_site_packages=False)
 
     def test_range_requirements(self):
         def f():
-            import funcsigs
-        self._run_as_operator(f, requirements=['funcsigs>1.0', 'dill'], system_site_packages=False)
+            import funcsigs  # noqa: F401
+        self._run_as_operator(
+            f, requirements=['funcsigs>1.0', 'dill'], system_site_packages=False)
 
     def test_fail(self):
         def f():
@@ -140,7 +143,8 @@ def f():
             raise Exception
         self._run_as_operator(f, python_version=3, use_dill=False, requirements=['dill'])
 
-    def _invert_python_major_version(self):
+    @staticmethod
+    def _invert_python_major_version():
         if sys.version_info[0] == 2:
             return 3
         else:
@@ -151,8 +155,10 @@ def test_wrong_python_op_args(self):
             version = 3
         else:
             version = 2
+
         def f():
             pass
+
         with self.assertRaises(AirflowException):
             self._run_as_operator(f, python_version=version, op_args=[1])
 
@@ -167,11 +173,12 @@ def f():
             print(virtualenv_string_args)
             if virtualenv_string_args[0] != virtualenv_string_args[2]:
                 raise Exception
-        self._run_as_operator(f, python_version=self._invert_python_major_version(), string_args=[1,2,1])
+        self._run_as_operator(
+            f, python_version=self._invert_python_major_version(), string_args=[1, 2, 1])
 
     def test_with_args(self):
         def f(a, b, c=False, d=False):
-            if a==0 and b==1 and c and not d:
+            if a == 0 and b == 1 and c and not d:
                 return True
             else:
                 raise Exception
@@ -198,3 +205,17 @@ def test_context(self):
         def f(**kwargs):
             return kwargs['templates_dict']['ds']
         self._run_as_operator(f, templates_dict={'ds': '{{ ds }}'})
+
+    def test_provide_context(self):
+        def fn():
+            pass
+        task = PythonVirtualenvOperator(
+            python_callable=fn,
+            python_version=sys.version_info[0],
+            task_id='task',
+            dag=self.dag,
+            provide_context=True,
+        )
+        self.assertTrue(
+            task.provide_context
+        )
diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py
new file mode 100644
index 0000000000000..114d189da14ab
--- /dev/null
+++ b/tests/plugins/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py
index d4c7f27b1c573..8bf0416b220ae 100644
--- a/tests/plugins/test_plugin.py
+++ b/tests/plugins/test_plugin.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,7 @@
 from flask import Blueprint
 from flask_admin import BaseView, expose
 from flask_admin.base import MenuLink
+from flask_appbuilder import BaseView as AppBuilderBaseView
 
 # Importing base classes that we need to derive
 from airflow.hooks.base_hook import BaseHook
@@ -30,10 +31,12 @@
 from airflow.sensors.base_sensor_operator import BaseSensorOperator
 from airflow.executors.base_executor import BaseExecutor
 
+
 # Will show up under airflow.hooks.test_plugin.PluginHook
 class PluginHook(BaseHook):
     pass
 
+
 # Will show up under airflow.operators.test_plugin.PluginOperator
 class PluginOperator(BaseOperator):
     pass
@@ -48,29 +51,57 @@ class PluginSensorOperator(BaseSensorOperator):
 class PluginExecutor(BaseExecutor):
     pass
 
+
 # Will show up under airflow.macros.test_plugin.plugin_macro
 def plugin_macro():
     pass
 
+
 # Creating a flask admin BaseView
-class TestView(BaseView):
+class PluginTestView(BaseView):
     @expose('/')
     def test(self):
-        # in this example, put your test_plugin/test.html template at airflow/plugins/templates/test_plugin/test.html
+        # in this example, put your test_plugin/test.html
+        # template at airflow/plugins/templates/test_plugin/test.html
+        return self.render("test_plugin/test.html", content="Hello galaxy!")
+
+
+v = PluginTestView(category="Test Plugin", name="Test View")
+
+
+# Creating a flask appbuilder BaseView
+class PluginTestAppBuilderBaseView(AppBuilderBaseView):
+    default_view = "test"
+
+    @expose("/")
+    def test(self):
         return self.render("test_plugin/test.html", content="Hello galaxy!")
-v = TestView(category="Test Plugin", name="Test View")
+
+
+v_appbuilder_view = PluginTestAppBuilderBaseView()
+v_appbuilder_package = {"name": "Test View",
+                        "category": "Test Plugin",
+                        "view": v_appbuilder_view}
+
+# Creating a flask appbuilder Menu Item
+appbuilder_mitem = {"name": "Google",
+                    "category": "Search",
+                    "category_icon": "fa-th",
+                    "href": "https://www.google.com"}
+
 
 # Creating a flask blueprint to intergrate the templates and static folder
 bp = Blueprint(
     "test_plugin", __name__,
-    template_folder='templates', # registers airflow/plugins/templates as a Jinja template folder
+    template_folder='templates',  # registers airflow/plugins/templates as a Jinja template folder
     static_folder='static',
     static_url_path='/static/test_plugin')
 
+
 ml = MenuLink(
     category='Test Plugin',
-    name='Test Menu Link',
-    url='https://airflow.incubator.apache.org/')
+    name="Test Menu Link",
+    url="https://airflow.apache.org/")
 
 
 # Defining the plugin class
@@ -84,3 +115,17 @@ class AirflowTestPlugin(AirflowPlugin):
     admin_views = [v]
     flask_blueprints = [bp]
     menu_links = [ml]
+    appbuilder_views = [v_appbuilder_package]
+    appbuilder_menu_items = [appbuilder_mitem]
+
+
+class MockPluginA(AirflowPlugin):
+    name = 'plugin-a'
+
+
+class MockPluginB(AirflowPlugin):
+    name = 'plugin-b'
+
+
+class MockPluginC(AirflowPlugin):
+    name = 'plugin-c'
diff --git a/tests/plugins/test_plugins_manager_rbac.py b/tests/plugins/test_plugins_manager_rbac.py
new file mode 100644
index 0000000000000..fe440debc4e30
--- /dev/null
+++ b/tests/plugins/test_plugins_manager_rbac.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import unittest
+
+
+from airflow.configuration import conf
+from airflow.www_rbac import app as application
+
+
+class PluginsTestRBAC(unittest.TestCase):
+    def setUp(self):
+        conf.load_test_config()
+        self.app, self.appbuilder = application.create_app(testing=True)
+
+    def test_flaskappbuilder_views(self):
+        from tests.plugins.test_plugin import v_appbuilder_package
+        appbuilder_class_name = str(v_appbuilder_package['view'].__class__.__name__)
+        plugin_views = [view for view in self.appbuilder.baseviews
+                        if view.blueprint.name == appbuilder_class_name]
+
+        self.assertTrue(len(plugin_views) == 1)
+
+        # view should have a menu item matching category of v_appbuilder_package
+        links = [menu_item for menu_item in self.appbuilder.menu.menu
+                 if menu_item.name == v_appbuilder_package['category']]
+
+        self.assertTrue(len(links) == 1)
+
+        # menu link should also have a link matching the name of the package.
+        link = links[0]
+        self.assertEqual(link.name, v_appbuilder_package['category'])
+        self.assertEqual(link.childs[0].name, v_appbuilder_package['name'])
+
+    def test_flaskappbuilder_menu_links(self):
+        from tests.plugins.test_plugin import appbuilder_mitem
+
+        # menu item should exist matching appbuilder_mitem
+        links = [menu_item for menu_item in self.appbuilder.menu.menu
+                 if menu_item.name == appbuilder_mitem['category']]
+
+        self.assertTrue(len(links) == 1)
+
+        # menu link should also have a link matching the name of the package.
+        link = links[0]
+        self.assertEqual(link.name, appbuilder_mitem['category'])
+        self.assertEqual(link.childs[0].name, appbuilder_mitem['name'])
+
+    def test_app_blueprints(self):
+        from tests.plugins.test_plugin import bp
+
+        # Blueprint should be present in the app
+        self.assertTrue('test_plugin' in self.app.blueprints)
+        self.assertEqual(self.app.blueprints['test_plugin'].name, bp.name)
diff --git a/tests/plugins_manager.py b/tests/plugins/test_plugins_manager_www.py
similarity index 64%
rename from tests/plugins_manager.py
rename to tests/plugins/test_plugins_manager_www.py
index f9d093f16284e..d47feb6e76a2f 100644
--- a/tests/plugins_manager.py
+++ b/tests/plugins/test_plugins_manager_www.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,19 +22,19 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import inspect
-import logging
 import unittest
+from mock import MagicMock, PropertyMock
 
 from flask.blueprints import Blueprint
-from flask_admin import BaseView
 from flask_admin.menu import MenuLink, MenuView
 
 from airflow.hooks.base_hook import BaseHook
-from airflow.models import  BaseOperator
+from airflow.models import BaseOperator
+from airflow.plugins_manager import load_entrypoint_plugins, is_valid_plugin
 from airflow.sensors.base_sensor_operator import BaseSensorOperator
 from airflow.executors.base_executor import BaseExecutor
-from airflow.www.app import cached_app
+from airflow.www.app import create_app
+from tests.plugins.test_plugin import MockPluginA, MockPluginB, MockPluginC
 
 
 class PluginsTest(unittest.TestCase):
@@ -68,7 +68,7 @@ def test_macros(self):
         self.assertTrue(callable(plugin_macro))
 
     def test_admin_views(self):
-        app = cached_app()
+        app = create_app(testing=True)
         [admin] = app.extensions['admin']
         category = admin._menu_categories['Test Plugin']
         [admin_view] = [v for v in category.get_children()
@@ -76,13 +76,64 @@ def test_admin_views(self):
         self.assertEqual('Test View', admin_view.name)
 
     def test_flask_blueprints(self):
-        app = cached_app()
+        app = create_app(testing=True)
         self.assertIsInstance(app.blueprints['test_plugin'], Blueprint)
 
     def test_menu_links(self):
-        app = cached_app()
+        app = create_app(testing=True)
         [admin] = app.extensions['admin']
         category = admin._menu_categories['Test Plugin']
         [menu_link] = [ml for ml in category.get_children()
                        if isinstance(ml, MenuLink)]
         self.assertEqual('Test Menu Link', menu_link.name)
+
+
+class PluginsTestEntrypointLoad(unittest.TestCase):
+
+    def setUp(self):
+        self.expected = [MockPluginA, MockPluginB, MockPluginC]
+        self.entrypoints = [
+            self._build_mock(plugin_obj)
+            for plugin_obj in self.expected
+        ]
+
+    def _build_mock(self, plugin_obj):
+        m = MagicMock(**{
+            'load.return_value': plugin_obj
+        })
+        type(m).name = PropertyMock(return_value='plugin-' + plugin_obj.name)
+        return m
+
+    def test_load_entrypoint_plugins(self):
+        self.assertListEqual(
+            load_entrypoint_plugins(self.entrypoints, []),
+            self.expected
+        )
+
+    def test_failed_load_entrpoint_plugins(self):
+        self.assertListEqual(
+            load_entrypoint_plugins(
+                self.entrypoints[:2] + [self._build_mock(
+                    MagicMock(name='other')
+                )], []),
+            self.expected[:2]
+        )
+
+
+class PluginsTestValidator(unittest.TestCase):
+    def setUp(self):
+        self.existing = [MockPluginA, MockPluginB]
+
+    def test_valid_plugin(self):
+        c = MockPluginC
+        self.assertTrue(
+            is_valid_plugin(
+                c, self.existing
+            ))
+
+    def test_invalid_plugin(self):
+        c = MockPluginC
+        self.assertFalse(
+            is_valid_plugin(
+                c, self.existing + [c]
+            ))
diff --git a/tests/security/__init__.py b/tests/security/__init__.py
index 7aa5f6adfa814..114d189da14ab 100644
--- a/tests/security/__init__.py
+++ b/tests/security/__init__.py
@@ -7,14 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from .kerberos import *
diff --git a/tests/security/kerberos.py b/tests/security/test_kerberos.py
similarity index 53%
rename from tests/security/kerberos.py
rename to tests/security/test_kerberos.py
index 98c06951cfdbb..36372696682ab 100644
--- a/tests/security/kerberos.py
+++ b/tests/security/test_kerberos.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,9 +19,10 @@
 
 import os
 import unittest
-
+from argparse import Namespace
 from airflow import configuration
 from airflow.security.kerberos import renew_from_kt
+from airflow import LoggingMixin
 
 
 @unittest.skipIf('KRB5_KTNAME' not in os.environ,
@@ -32,13 +33,34 @@ def setUp(self):
 
         if not configuration.conf.has_section("kerberos"):
             configuration.conf.add_section("kerberos")
-
-        configuration.conf.set("kerberos",
-                               "keytab",
+        configuration.conf.set("kerberos", "keytab",
                                os.environ['KRB5_KTNAME'])
+        keytab_from_cfg = configuration.conf.get("kerberos", "keytab")
+        self.args = Namespace(keytab=keytab_from_cfg, principal=None, pid=None,
+                              daemon=None, stdout=None, stderr=None, log_file=None)
 
     def test_renew_from_kt(self):
         """
         We expect no result, but a successful run. No more TypeError
         """
-        self.assertIsNone(renew_from_kt())
+        self.assertIsNone(renew_from_kt(principal=self.args.principal,
+                                        keytab=self.args.keytab))
+
+    def test_args_from_cli(self):
+        """
+        We expect no result, but a run with sys.exit(1) because keytab not exist.
+        """
+        configuration.conf.set("kerberos", "keytab", "")
+        self.args.keytab = "test_keytab"
+
+        with self.assertRaises(SystemExit) as se:
+            renew_from_kt(principal=self.args.principal,
+                          keytab=self.args.keytab)
+
+            with self.assertLogs(LoggingMixin().log) as log:
+                self.assertIn(
+                    'kinit: krb5_init_creds_set_keytab: Failed to find '
+                    'airflow@LUPUS.GRIDDYNAMICS.NET in keytab FILE:{} '
+                    '(unknown enctype)'.format(self.args.keytab), log.output)
+
+        self.assertEqual(se.exception.code, 1)
diff --git a/tests/sensors/__init__.py b/tests/sensors/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/sensors/__init__.py
+++ b/tests/sensors/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/sensors/test_base_sensor.py b/tests/sensors/test_base_sensor.py
index adb7a5d1e31f7..9af8cbf1c3018 100644
--- a/tests/sensors/test_base_sensor.py
+++ b/tests/sensors/test_base_sensor.py
@@ -18,17 +18,22 @@
 # under the License.
 
 import unittest
+from mock import Mock
 
 from airflow import DAG, configuration, settings
-from airflow.exceptions import AirflowSensorTimeout
+from airflow.exceptions import (AirflowSensorTimeout, AirflowException,
+                                AirflowRescheduleException)
 from airflow.models import DagRun, TaskInstance
+from airflow.models.taskreschedule import TaskReschedule
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.sensors.base_sensor_operator import BaseSensorOperator
+from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep
 from airflow.utils import timezone
 from airflow.utils.state import State
 from airflow.utils.timezone import datetime
 from datetime import timedelta
 from time import sleep
+from freezegun import freeze_time
 
 configuration.load_test_config()
 
@@ -57,6 +62,7 @@ def setUp(self):
         self.dag = DAG(TEST_DAG_ID, default_args=args)
 
         session = settings.Session()
+        session.query(TaskReschedule).delete()
         session.query(DagRun).delete()
         session.query(TaskInstance).delete()
         session.commit()
@@ -101,12 +107,12 @@ def test_ok(self):
 
         self._run(sensor)
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 2)
+        self.assertEqual(len(tis), 2)
         for ti in tis:
             if ti.task_id == SENSOR_OP:
-                self.assertEquals(ti.state, State.SUCCESS)
+                self.assertEqual(ti.state, State.SUCCESS)
             if ti.task_id == DUMMY_OP:
-                self.assertEquals(ti.state, State.NONE)
+                self.assertEqual(ti.state, State.NONE)
 
     def test_fail(self):
         sensor = self._make_sensor(False)
@@ -115,12 +121,12 @@ def test_fail(self):
         with self.assertRaises(AirflowSensorTimeout):
             self._run(sensor)
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 2)
+        self.assertEqual(len(tis), 2)
         for ti in tis:
             if ti.task_id == SENSOR_OP:
-                self.assertEquals(ti.state, State.FAILED)
+                self.assertEqual(ti.state, State.FAILED)
             if ti.task_id == DUMMY_OP:
-                self.assertEquals(ti.state, State.NONE)
+                self.assertEqual(ti.state, State.NONE)
 
     def test_soft_fail(self):
         sensor = self._make_sensor(False, soft_fail=True)
@@ -128,9 +134,9 @@ def test_soft_fail(self):
 
         self._run(sensor)
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 2)
+        self.assertEqual(len(tis), 2)
         for ti in tis:
-            self.assertEquals(ti.state, State.SKIPPED)
+            self.assertEqual(ti.state, State.SKIPPED)
 
     def test_soft_fail_with_retries(self):
         sensor = self._make_sensor(
@@ -144,17 +150,359 @@ def test_soft_fail_with_retries(self):
         with self.assertRaises(AirflowSensorTimeout):
             self._run(sensor)
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 2)
+        self.assertEqual(len(tis), 2)
         for ti in tis:
             if ti.task_id == SENSOR_OP:
-                self.assertEquals(ti.state, State.UP_FOR_RETRY)
+                self.assertEqual(ti.state, State.UP_FOR_RETRY)
             if ti.task_id == DUMMY_OP:
-                self.assertEquals(ti.state, State.NONE)
+                self.assertEqual(ti.state, State.NONE)
 
         sleep(0.001)
         # after retry DAG run is skipped
         self._run(sensor)
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 2)
+        self.assertEqual(len(tis), 2)
         for ti in tis:
-            self.assertEquals(ti.state, State.SKIPPED)
+            self.assertEqual(ti.state, State.SKIPPED)
+
+    def test_ok_with_reschedule(self):
+        sensor = self._make_sensor(
+            return_value=None,
+            poke_interval=10,
+            timeout=25,
+            mode='reschedule')
+        sensor.poke = Mock(side_effect=[False, False, True])
+        dr = self._make_dag_run()
+
+        # first poke returns False and task is re-scheduled
+        date1 = timezone.utcnow()
+        with freeze_time(date1):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                # verify task is re-scheduled, i.e. state set to NONE
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+                # verify task start date is the initial one
+                self.assertEqual(ti.start_date, date1)
+                # verify one row in task_reschedule table
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 1)
+                self.assertEqual(task_reschedules[0].start_date, date1)
+                self.assertEqual(task_reschedules[0].reschedule_date,
+                                 date1 + timedelta(seconds=sensor.poke_interval))
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # second poke returns False and task is re-scheduled
+        date2 = date1 + timedelta(seconds=sensor.poke_interval)
+        with freeze_time(date2):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                # verify task is re-scheduled, i.e. state set to NONE
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+                # verify task start date is the initial one
+                self.assertEqual(ti.start_date, date1)
+                # verify two rows in task_reschedule table
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 2)
+                self.assertEqual(task_reschedules[1].start_date, date2)
+                self.assertEqual(task_reschedules[1].reschedule_date,
+                                 date2 + timedelta(seconds=sensor.poke_interval))
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # third poke returns True and task succeeds
+        date3 = date2 + timedelta(seconds=sensor.poke_interval)
+        with freeze_time(date3):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.SUCCESS)
+                # verify task start date is the initial one
+                self.assertEqual(ti.start_date, date1)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+    def test_fail_with_reschedule(self):
+        sensor = self._make_sensor(
+            return_value=False,
+            poke_interval=10,
+            timeout=5,
+            mode='reschedule')
+        dr = self._make_dag_run()
+
+        # first poke returns False and task is re-scheduled
+        date1 = timezone.utcnow()
+        with freeze_time(date1):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # second poke returns False, timeout occurs
+        date2 = date1 + timedelta(seconds=sensor.poke_interval)
+        with freeze_time(date2):
+            with self.assertRaises(AirflowSensorTimeout):
+                self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.FAILED)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+    def test_soft_fail_with_reschedule(self):
+        sensor = self._make_sensor(
+            return_value=False,
+            poke_interval=10,
+            timeout=5,
+            soft_fail=True,
+            mode='reschedule')
+        dr = self._make_dag_run()
+
+        # first poke returns False and task is re-scheduled
+        date1 = timezone.utcnow()
+        with freeze_time(date1):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # second poke returns False, timeout occurs
+        date2 = date1 + timedelta(seconds=sensor.poke_interval)
+        with freeze_time(date2):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            self.assertEqual(ti.state, State.SKIPPED)
+
+    def test_ok_with_reschedule_and_retry(self):
+        sensor = self._make_sensor(
+            return_value=None,
+            poke_interval=10,
+            timeout=5,
+            retries=1,
+            retry_delay=timedelta(seconds=10),
+            mode='reschedule')
+        sensor.poke = Mock(side_effect=[False, False, False, True])
+        dr = self._make_dag_run()
+
+        # first poke returns False and task is re-scheduled
+        date1 = timezone.utcnow()
+        with freeze_time(date1):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+                # verify one row in task_reschedule table
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 1)
+                self.assertEqual(task_reschedules[0].start_date, date1)
+                self.assertEqual(task_reschedules[0].reschedule_date,
+                                 date1 + timedelta(seconds=sensor.poke_interval))
+                self.assertEqual(task_reschedules[0].try_number, 1)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # second poke fails and task instance is marked up to retry
+        date2 = date1 + timedelta(seconds=sensor.poke_interval)
+        with freeze_time(date2):
+            with self.assertRaises(AirflowSensorTimeout):
+                self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.UP_FOR_RETRY)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # third poke returns False and task is rescheduled again
+        date3 = date2 + timedelta(seconds=sensor.poke_interval) + sensor.retry_delay
+        with freeze_time(date3):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+                # verify one row in task_reschedule table
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 1)
+                self.assertEqual(task_reschedules[0].start_date, date3)
+                self.assertEqual(task_reschedules[0].reschedule_date,
+                                 date3 + timedelta(seconds=sensor.poke_interval))
+                self.assertEqual(task_reschedules[0].try_number, 2)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # fourth poke return True and task succeeds
+        date4 = date3 + timedelta(seconds=sensor.poke_interval)
+        with freeze_time(date4):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.SUCCESS)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+    def test_should_include_ready_to_reschedule_dep(self):
+        sensor = self._make_sensor(True)
+        deps = sensor.deps
+        self.assertTrue(ReadyToRescheduleDep() in deps)
+
+    def test_invalid_mode(self):
+        with self.assertRaises(AirflowException):
+            self._make_sensor(
+                return_value=True,
+                mode='foo')
+
+    def test_ok_with_custom_reschedule_exception(self):
+        sensor = self._make_sensor(
+            return_value=None,
+            mode='reschedule')
+        date1 = timezone.utcnow()
+        date2 = date1 + timedelta(seconds=60)
+        date3 = date1 + timedelta(seconds=120)
+        sensor.poke = Mock(side_effect=[
+            AirflowRescheduleException(date2),
+            AirflowRescheduleException(date3),
+            True,
+        ])
+        dr = self._make_dag_run()
+
+        # first poke returns False and task is re-scheduled
+        with freeze_time(date1):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                # verify task is re-scheduled, i.e. state set to NONE
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+                # verify one row in task_reschedule table
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 1)
+                self.assertEqual(task_reschedules[0].start_date, date1)
+                self.assertEqual(task_reschedules[0].reschedule_date, date2)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # second poke returns False and task is re-scheduled
+        with freeze_time(date2):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                # verify task is re-scheduled, i.e. state set to NONE
+                self.assertEqual(ti.state, State.UP_FOR_RESCHEDULE)
+                # verify two rows in task_reschedule table
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 2)
+                self.assertEqual(task_reschedules[1].start_date, date2)
+                self.assertEqual(task_reschedules[1].reschedule_date, date3)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+        # third poke returns True and task succeeds
+        with freeze_time(date3):
+            self._run(sensor)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                self.assertEqual(ti.state, State.SUCCESS)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+    def test_reschedule_with_test_mode(self):
+        sensor = self._make_sensor(
+            return_value=None,
+            poke_interval=10,
+            timeout=25,
+            mode='reschedule')
+        sensor.poke = Mock(side_effect=[False])
+        dr = self._make_dag_run()
+
+        # poke returns False and AirflowRescheduleException is raised
+        date1 = timezone.utcnow()
+        with freeze_time(date1):
+            for dt in self.dag.date_range(DEFAULT_DATE, end_date=DEFAULT_DATE):
+                TaskInstance(sensor, dt).run(
+                    ignore_ti_state=True,
+                    test_mode=True)
+        tis = dr.get_task_instances()
+        self.assertEqual(len(tis), 2)
+        for ti in tis:
+            if ti.task_id == SENSOR_OP:
+                # in test mode state is not modified
+                self.assertEqual(ti.state, State.NONE)
+                # in test mode no reschedule request is recorded
+                task_reschedules = TaskReschedule.find_for_task_instance(ti)
+                self.assertEqual(len(task_reschedules), 0)
+            if ti.task_id == DUMMY_OP:
+                self.assertEqual(ti.state, State.NONE)
+
+    def test_sensor_with_invalid_poke_interval(self):
+        negative_poke_interval = -10
+        non_number_poke_interval = "abcd"
+        positive_poke_interval = 10
+        with self.assertRaises(AirflowException):
+            self._make_sensor(
+                return_value=None,
+                poke_interval=negative_poke_interval,
+                timeout=25)
+
+        with self.assertRaises(AirflowException):
+            self._make_sensor(
+                return_value=None,
+                poke_interval=non_number_poke_interval,
+                timeout=25)
+
+        self._make_sensor(
+            return_value=None,
+            poke_interval=positive_poke_interval,
+            timeout=25)
+
+    def test_sensor_with_invalid_timeout(self):
+        negative_timeout = -25
+        non_number_timeout = "abcd"
+        positive_timeout = 25
+        with self.assertRaises(AirflowException):
+            self._make_sensor(
+                return_value=None,
+                poke_interval=10,
+                timeout=negative_timeout)
+
+        with self.assertRaises(AirflowException):
+            self._make_sensor(
+                return_value=None,
+                poke_interval=10,
+                timeout=non_number_timeout)
+
+        self._make_sensor(
+            return_value=None,
+            poke_interval=10,
+            timeout=positive_timeout)
diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py
index 403c892c239b2..18fff556bf035 100644
--- a/tests/sensors/test_external_task_sensor.py
+++ b/tests/sensors/test_external_task_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,12 +17,11 @@
 # specific language governing permissions and limitations
 # under the License.
 import unittest
-
 from datetime import timedelta, time
 
 from airflow import DAG, configuration, settings
 from airflow import exceptions
-from airflow.exceptions import AirflowSensorTimeout
+from airflow.exceptions import AirflowException, AirflowSensorTimeout
 from airflow.models import TaskInstance, DagBag
 from airflow.operators.bash_operator import BashOperator
 from airflow.operators.dummy_operator import DummyOperator
@@ -75,6 +74,30 @@ def test_external_task_sensor(self):
             ignore_ti_state=True
         )
 
+    def test_external_dag_sensor(self):
+
+        other_dag = DAG(
+            'other_dag',
+            default_args=self.args,
+            end_date=DEFAULT_DATE,
+            schedule_interval='@once')
+        other_dag.create_dagrun(
+            run_id='test',
+            start_date=DEFAULT_DATE,
+            execution_date=DEFAULT_DATE,
+            state=State.SUCCESS)
+        t = ExternalTaskSensor(
+            task_id='test_external_dag_sensor_check',
+            external_dag_id='other_dag',
+            external_task_id=None,
+            dag=self.dag
+        )
+        t.run(
+            start_date=DEFAULT_DATE,
+            end_date=DEFAULT_DATE,
+            ignore_ti_state=True
+        )
+
     def test_templated_sensor(self):
         dag = DAG(TEST_DAG_ID, self.args)
 
@@ -247,3 +270,54 @@ def test_external_task_sensor_error_delta_and_fn(self):
                 allowed_states=['success'],
                 dag=self.dag
             )
+
+    def test_catch_invalid_allowed_states(self):
+        with self.assertRaises(ValueError):
+            ExternalTaskSensor(
+                task_id='test_external_task_sensor_check',
+                external_dag_id=TEST_DAG_ID,
+                external_task_id=TEST_TASK_ID,
+                allowed_states=['invalid_state'],
+                dag=self.dag
+            )
+
+        with self.assertRaises(ValueError):
+            ExternalTaskSensor(
+                task_id='test_external_task_sensor_check',
+                external_dag_id=TEST_DAG_ID,
+                external_task_id=None,
+                allowed_states=['invalid_state'],
+                dag=self.dag
+            )
+
+    def test_external_task_sensor_waits_for_task_check_existence(self):
+        t = ExternalTaskSensor(
+            task_id='test_external_task_sensor_check',
+            external_dag_id="example_bash_operator",
+            external_task_id="non-existing-task",
+            check_existence=True,
+            dag=self.dag
+        )
+
+        with self.assertRaises(AirflowException):
+            t.run(
+                start_date=DEFAULT_DATE,
+                end_date=DEFAULT_DATE,
+                ignore_ti_state=True
+            )
+
+    def test_external_task_sensor_waits_for_dag_check_existence(self):
+        t = ExternalTaskSensor(
+            task_id='test_external_task_sensor_check',
+            external_dag_id="non-existing-dag",
+            external_task_id=None,
+            check_existence=True,
+            dag=self.dag
+        )
+
+        with self.assertRaises(AirflowException):
+            t.run(
+                start_date=DEFAULT_DATE,
+                end_date=DEFAULT_DATE,
+                ignore_ti_state=True
+            )
diff --git a/tests/sensors/test_hdfs_sensor.py b/tests/sensors/test_hdfs_sensor.py
index b94065d8423ef..26adeaa3ab60b 100644
--- a/tests/sensors/test_hdfs_sensor.py
+++ b/tests/sensors/test_hdfs_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/sensors/test_http_sensor.py b/tests/sensors/test_http_sensor.py
index de9513724415a..7e078be2a7e2c 100644
--- a/tests/sensors/test_http_sensor.py
+++ b/tests/sensors/test_http_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -26,14 +26,7 @@
 from airflow.operators.http_operator import SimpleHttpOperator
 from airflow.sensors.http_sensor import HttpSensor
 from airflow.utils.timezone import datetime
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+from tests.compat import mock
 
 configuration.load_test_config()
 
@@ -140,7 +133,7 @@ class FakeSession(object):
     def __init__(self):
         self.response = requests.Response()
         self.response.status_code = 200
-        self.response._content = 'airbnb/airflow'.encode('ascii', 'ignore')
+        self.response._content = 'apache/airflow'.encode('ascii', 'ignore')
 
     def send(self, request, **kwargs):
         return self.response
@@ -178,7 +171,7 @@ def test_get_response_check(self):
             method='GET',
             endpoint='/search',
             data={"client": "ubuntu", "q": "airflow"},
-            response_check=lambda response: ("airbnb/airflow" in response.text),
+            response_check=lambda response: ("apache/airflow" in response.text),
             headers={},
             dag=self.dag)
         t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
@@ -192,7 +185,7 @@ def test_sensor(self):
             request_params={"client": "ubuntu", "q": "airflow", 'date': '{{ds}}'},
             headers={},
             response_check=lambda response: (
-                "airbnb/airflow/" + DEFAULT_DATE.strftime('%Y-%m-%d')
+                "apache/airflow/" + DEFAULT_DATE.strftime('%Y-%m-%d')
                 in response.text),
             poke_interval=5,
             timeout=15,
diff --git a/tests/sensors/test_sql_sensor.py b/tests/sensors/test_sql_sensor.py
index 960bdd21ed1aa..0b25d58056eda 100644
--- a/tests/sensors/test_sql_sensor.py
+++ b/tests/sensors/test_sql_sensor.py
@@ -7,19 +7,21 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import mock
 import unittest
 
 from airflow import DAG
 from airflow import configuration
+from airflow.exceptions import AirflowException
 from airflow.sensors.sql_sensor import SqlSensor
 from airflow.utils.timezone import datetime
 
@@ -38,20 +40,70 @@ def setUp(self):
         }
         self.dag = DAG(TEST_DAG_ID, default_args=args)
 
-    def test_sql_sensor_mysql(self):
+    def test_unsupported_conn_type(self):
         t = SqlSensor(
+            task_id='sql_sensor_check',
+            conn_id='redis_default',
+            sql="SELECT count(1) FROM INFORMATION_SCHEMA.TABLES",
+            dag=self.dag
+        )
+
+        with self.assertRaises(AirflowException):
+            t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+    @unittest.skipUnless(
+        'mysql' in configuration.conf.get('core', 'sql_alchemy_conn'), "this is a mysql test")
+    def test_sql_sensor_mysql(self):
+        t1 = SqlSensor(
             task_id='sql_sensor_check',
             conn_id='mysql_default',
             sql="SELECT count(1) FROM INFORMATION_SCHEMA.TABLES",
             dag=self.dag
         )
-        t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+        t1.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+        t2 = SqlSensor(
+            task_id='sql_sensor_check',
+            conn_id='mysql_default',
+            sql="SELECT count(%s) FROM INFORMATION_SCHEMA.TABLES",
+            parameters=["table_name"],
+            dag=self.dag
+        )
+        t2.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
 
+    @unittest.skipUnless(
+        'postgresql' in configuration.conf.get('core', 'sql_alchemy_conn'), "this is a postgres test")
     def test_sql_sensor_postgres(self):
-        t = SqlSensor(
+        t1 = SqlSensor(
             task_id='sql_sensor_check',
             conn_id='postgres_default',
             sql="SELECT count(1) FROM INFORMATION_SCHEMA.TABLES",
             dag=self.dag
         )
-        t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+        t1.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+        t2 = SqlSensor(
+            task_id='sql_sensor_check',
+            conn_id='postgres_default',
+            sql="SELECT count(%s) FROM INFORMATION_SCHEMA.TABLES",
+            parameters=["table_name"],
+            dag=self.dag
+        )
+        t2.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
+
+    @mock.patch('airflow.sensors.sql_sensor.BaseHook')
+    def test_sql_sensor_postgres_poke(self, mock_hook):
+        t = SqlSensor(
+            task_id='sql_sensor_check',
+            conn_id='postgres_default',
+            sql="SELECT 1",
+        )
+
+        mock_hook.get_connection('postgres_default').conn_type = "postgres"
+        mock_get_records = mock_hook.get_connection.return_value.get_hook.return_value.get_records
+
+        mock_get_records.return_value = []
+        self.assertFalse(t.poke(None))
+
+        mock_get_records.return_value = [['1']]
+        self.assertTrue(t.poke(None))
diff --git a/tests/sensors/test_timedelta_sensor.py b/tests/sensors/test_timedelta_sensor.py
index 18ee07bfa8823..6951746532f6f 100644
--- a/tests/sensors/test_timedelta_sensor.py
+++ b/tests/sensors/test_timedelta_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/sensors/test_timeout_sensor.py b/tests/sensors/test_timeout_sensor.py
index 7a4e70b71cfdb..91fc2d57a619e 100644
--- a/tests/sensors/test_timeout_sensor.py
+++ b/tests/sensors/test_timeout_sensor.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/task/task_runner/__init__.py b/tests/task/task_runner/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/task/task_runner/__init__.py
+++ b/tests/task/task_runner/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/task/task_runner/test_bash_task_runner.py b/tests/task/task_runner/test_standard_task_runner.py
similarity index 94%
rename from tests/task/task_runner/test_bash_task_runner.py
rename to tests/task/task_runner/test_standard_task_runner.py
index 24c0819fdb070..38a4e41e37ed6 100644
--- a/tests/task/task_runner/test_bash_task_runner.py
+++ b/tests/task/task_runner/test_standard_task_runner.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,7 +25,7 @@
 from airflow import models, settings
 from airflow.jobs import LocalTaskJob
 from airflow.models import TaskInstance as TI
-from airflow.task.task_runner import BashTaskRunner
+from airflow.task.task_runner import StandardTaskRunner
 from airflow.utils import timezone
 from airflow.utils.state import State
 
@@ -61,7 +61,7 @@
 }
 
 
-class TestBashTaskRunner(unittest.TestCase):
+class TestStandardTaskRunner(unittest.TestCase):
     def setUp(self):
         dictConfig(LOGGING_CONFIG)
 
@@ -71,7 +71,7 @@ def test_start_and_terminate(self):
         local_task_job.task_instance.run_as_user = None
         local_task_job.task_instance.command_as_list.return_value = ['sleep', '1000']
 
-        runner = BashTaskRunner(local_task_job)
+        runner = StandardTaskRunner(local_task_job)
         runner.start()
 
         pgid = os.getpgid(runner.process.pid)
@@ -119,7 +119,7 @@ def test_on_kill(self):
         ti = TI(task=task, execution_date=DEFAULT_DATE)
         job1 = LocalTaskJob(task_instance=ti, ignore_ti_state=True)
 
-        runner = BashTaskRunner(job1)
+        runner = StandardTaskRunner(job1)
         runner.start()
 
         # give the task some time to startup
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
new file mode 100644
index 0000000000000..a768f3083b805
--- /dev/null
+++ b/tests/test_configuration.py
@@ -0,0 +1,321 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import contextlib
+import os
+import warnings
+from collections import OrderedDict
+
+import six
+
+from airflow import configuration
+from airflow.configuration import conf, AirflowConfigParser, parameterized_config
+
+if six.PY2:
+    # Need `assertWarns` back-ported from unittest2
+    import unittest2 as unittest
+else:
+    import unittest
+
+
+@contextlib.contextmanager
+def env_vars(**vars):
+    original = {}
+    for key, value in vars.items():
+        original[key] = os.environ.get(key)
+        if value is not None:
+            os.environ[key] = value
+        else:
+            os.environ.pop(key, None)
+    yield
+    for key, value in original.items():
+        if value is not None:
+            os.environ[key] = value
+        else:
+            os.environ.pop(key, None)
+
+
+class ConfTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        os.environ['AIRFLOW__TESTSECTION__TESTKEY'] = 'testvalue'
+        os.environ['AIRFLOW__TESTSECTION__TESTPERCENT'] = 'with%percent'
+        configuration.load_test_config()
+        conf.set('core', 'percent', 'with%%inside')
+
+    @classmethod
+    def tearDownClass(cls):
+        del os.environ['AIRFLOW__TESTSECTION__TESTKEY']
+        del os.environ['AIRFLOW__TESTSECTION__TESTPERCENT']
+
+    def test_airflow_home_default(self):
+        with env_vars(AIRFLOW_HOME=None):
+            self.assertEqual(
+                configuration.get_airflow_home(),
+                configuration.expand_env_var('~/airflow'))
+
+    def test_airflow_home_override(self):
+        with env_vars(AIRFLOW_HOME='/path/to/airflow'):
+            self.assertEqual(
+                configuration.get_airflow_home(),
+                '/path/to/airflow')
+
+    def test_airflow_config_default(self):
+        with env_vars(AIRFLOW_CONFIG=None):
+            self.assertEqual(
+                configuration.get_airflow_config('/home/airflow'),
+                configuration.expand_env_var('/home/airflow/airflow.cfg'))
+
+    def test_airflow_config_override(self):
+        with env_vars(AIRFLOW_CONFIG='/path/to/airflow/airflow.cfg'):
+            self.assertEqual(
+                configuration.get_airflow_config('/home//airflow'),
+                '/path/to/airflow/airflow.cfg')
+
+    def test_env_var_config(self):
+        opt = conf.get('testsection', 'testkey')
+        self.assertEqual(opt, 'testvalue')
+
+        opt = conf.get('testsection', 'testpercent')
+        self.assertEqual(opt, 'with%percent')
+
+        self.assertTrue(conf.has_option('testsection', 'testkey'))
+
+    def test_conf_as_dict(self):
+        cfg_dict = conf.as_dict()
+
+        # test that configs are picked up
+        self.assertEqual(cfg_dict['core']['unit_test_mode'], 'True')
+
+        self.assertEqual(cfg_dict['core']['percent'], 'with%inside')
+
+        # test env vars
+        self.assertEqual(cfg_dict['testsection']['testkey'], '< hidden >')
+
+    def test_conf_as_dict_source(self):
+        # test display_source
+        cfg_dict = conf.as_dict(display_source=True)
+        self.assertEqual(
+            cfg_dict['core']['load_examples'][1], 'airflow.cfg')
+        self.assertEqual(
+            cfg_dict['testsection']['testkey'], ('< hidden >', 'env var'))
+
+    def test_conf_as_dict_sensitive(self):
+        # test display_sensitive
+        cfg_dict = conf.as_dict(display_sensitive=True)
+        self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
+        self.assertEqual(cfg_dict['testsection']['testpercent'], 'with%percent')
+
+        # test display_source and display_sensitive
+        cfg_dict = conf.as_dict(display_sensitive=True, display_source=True)
+        self.assertEqual(
+            cfg_dict['testsection']['testkey'], ('testvalue', 'env var'))
+
+    def test_conf_as_dict_raw(self):
+        # test display_sensitive
+        cfg_dict = conf.as_dict(raw=True, display_sensitive=True)
+        self.assertEqual(cfg_dict['testsection']['testkey'], 'testvalue')
+
+        # Values with '%' in them should be escaped
+        self.assertEqual(cfg_dict['testsection']['testpercent'], 'with%%percent')
+        self.assertEqual(cfg_dict['core']['percent'], 'with%%inside')
+
+    def test_command_precedence(self):
+        TEST_CONFIG = '''[test]
+key1 = hello
+key2_cmd = printf cmd_result
+key3 = airflow
+key4_cmd = printf key4_result
+'''
+        TEST_CONFIG_DEFAULT = '''[test]
+key1 = awesome
+key2 = airflow
+
+[another]
+key6 = value6
+'''
+
+        test_conf = AirflowConfigParser(
+            default_config=parameterized_config(TEST_CONFIG_DEFAULT))
+        test_conf.read_string(TEST_CONFIG)
+        test_conf.as_command_stdout = test_conf.as_command_stdout | {
+            ('test', 'key2'),
+            ('test', 'key4'),
+        }
+        self.assertEqual('hello', test_conf.get('test', 'key1'))
+        self.assertEqual('cmd_result', test_conf.get('test', 'key2'))
+        self.assertEqual('airflow', test_conf.get('test', 'key3'))
+        self.assertEqual('key4_result', test_conf.get('test', 'key4'))
+        self.assertEqual('value6', test_conf.get('another', 'key6'))
+
+        self.assertEqual('hello', test_conf.get('test', 'key1', fallback='fb'))
+        self.assertEqual('value6', test_conf.get('another', 'key6', fallback='fb'))
+        self.assertEqual('fb', test_conf.get('another', 'key7', fallback='fb'))
+        self.assertEqual(True, test_conf.getboolean('another', 'key8_boolean', fallback='True'))
+        self.assertEqual(10, test_conf.getint('another', 'key8_int', fallback='10'))
+        self.assertEqual(1.0, test_conf.getfloat('another', 'key8_float', fallback='1'))
+
+        self.assertTrue(test_conf.has_option('test', 'key1'))
+        self.assertTrue(test_conf.has_option('test', 'key2'))
+        self.assertTrue(test_conf.has_option('test', 'key3'))
+        self.assertTrue(test_conf.has_option('test', 'key4'))
+        self.assertFalse(test_conf.has_option('test', 'key5'))
+        self.assertTrue(test_conf.has_option('another', 'key6'))
+
+        cfg_dict = test_conf.as_dict(display_sensitive=True)
+        self.assertEqual('cmd_result', cfg_dict['test']['key2'])
+        self.assertNotIn('key2_cmd', cfg_dict['test'])
+
+    def test_remove_option(self):
+        TEST_CONFIG = '''[test]
+key1 = hello
+key2 = airflow
+'''
+        TEST_CONFIG_DEFAULT = '''[test]
+key1 = awesome
+key2 = airflow
+'''
+
+        test_conf = AirflowConfigParser(
+            default_config=parameterized_config(TEST_CONFIG_DEFAULT))
+        test_conf.read_string(TEST_CONFIG)
+
+        self.assertEqual('hello', test_conf.get('test', 'key1'))
+        test_conf.remove_option('test', 'key1', remove_default=False)
+        self.assertEqual('awesome', test_conf.get('test', 'key1'))
+
+        test_conf.remove_option('test', 'key2')
+        self.assertFalse(test_conf.has_option('test', 'key2'))
+
+    def test_getsection(self):
+        TEST_CONFIG = '''
+[test]
+key1 = hello
+'''
+        TEST_CONFIG_DEFAULT = '''
+[test]
+key1 = awesome
+key2 = airflow
+
+[testsection]
+key3 = value3
+'''
+        test_conf = AirflowConfigParser(
+            default_config=parameterized_config(TEST_CONFIG_DEFAULT))
+        test_conf.read_string(TEST_CONFIG)
+
+        self.assertEqual(
+            OrderedDict([('key1', 'hello'), ('key2', 'airflow')]),
+            test_conf.getsection('test')
+        )
+        self.assertEqual(
+            OrderedDict([
+                ('key3', 'value3'),
+                ('testkey', 'testvalue'),
+                ('testpercent', 'with%percent')]),
+            test_conf.getsection('testsection')
+        )
+
+    def test_broker_transport_options(self):
+        section_dict = conf.getsection("celery_broker_transport_options")
+        self.assertTrue(isinstance(section_dict['visibility_timeout'], int))
+        self.assertTrue(isinstance(section_dict['_test_only_bool'], bool))
+        self.assertTrue(isinstance(section_dict['_test_only_float'], float))
+        self.assertTrue(isinstance(section_dict['_test_only_string'], six.string_types))
+
+    def test_deprecated_options(self):
+        # Guarantee we have a deprecated setting, so we test the deprecation
+        # lookup even if we remove this explicit fallback
+        conf.deprecated_options['celery'] = {
+            'worker_concurrency': 'celeryd_concurrency',
+        }
+
+        # Remove it so we are sure we use the right setting
+        conf.remove_option('celery', 'worker_concurrency')
+
+        with self.assertWarns(DeprecationWarning):
+            os.environ['AIRFLOW__CELERY__CELERYD_CONCURRENCY'] = '99'
+            self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
+            os.environ.pop('AIRFLOW__CELERY__CELERYD_CONCURRENCY')
+
+        with self.assertWarns(DeprecationWarning):
+            conf.set('celery', 'celeryd_concurrency', '99')
+            self.assertEqual(conf.getint('celery', 'worker_concurrency'), 99)
+            conf.remove_option('celery', 'celeryd_concurrency')
+
+    def test_deprecated_options_cmd(self):
+        # Guarantee we have a deprecated setting, so we test the deprecation
+        # lookup even if we remove this explicit fallback
+        conf.deprecated_options['celery'] = {'result_backend': 'celery_result_backend'}
+        conf.as_command_stdout.add(('celery', 'celery_result_backend'))
+
+        conf.remove_option('celery', 'result_backend')
+        conf.set('celery', 'celery_result_backend_cmd', '/bin/echo 99')
+
+        with self.assertWarns(DeprecationWarning):
+            tmp = None
+            if 'AIRFLOW__CELERY__RESULT_BACKEND' in os.environ:
+                tmp = os.environ.pop('AIRFLOW__CELERY__RESULT_BACKEND')
+            self.assertEqual(conf.getint('celery', 'result_backend'), 99)
+            if tmp:
+                os.environ['AIRFLOW__CELERY__RESULT_BACKEND'] = tmp
+
+    def test_deprecated_values(self):
+        def make_config():
+            test_conf = AirflowConfigParser()
+            # Guarantee we have a deprecated setting, so we test the deprecation
+            # lookup even if we remove this explicit fallback
+            test_conf.deprecated_values = {
+                'core': {
+                    'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
+                },
+            }
+            test_conf.read_dict({
+                'core': {
+                    'executor': 'SequentialExecutor',
+                    'task_runner': 'BashTaskRunner',
+                    'sql_alchemy_conn': 'sqlite://',
+                },
+                'webserver': {
+                    'authenticate': False,
+                },
+            })
+            return test_conf
+
+        with self.assertWarns(FutureWarning):
+            test_conf = make_config()
+            self.assertEqual(test_conf.get('core', 'task_runner'), 'StandardTaskRunner')
+
+        with self.assertWarns(FutureWarning):
+            with env_vars(AIRFLOW__CORE__TASK_RUNNER='BashTaskRunner'):
+                test_conf = make_config()
+
+                self.assertEqual(test_conf.get('core', 'task_runner'), 'StandardTaskRunner')
+
+        with warnings.catch_warnings(record=True) as w:
+            with env_vars(AIRFLOW__CORE__TASK_RUNNER='NotBashTaskRunner'):
+                test_conf = make_config()
+
+                self.assertEqual(test_conf.get('core', 'task_runner'), 'NotBashTaskRunner')
+
+                self.assertListEqual([], w)
diff --git a/tests/impersonation.py b/tests/test_impersonation.py
similarity index 99%
rename from tests/impersonation.py
rename to tests/test_impersonation.py
index f789fcb95f463..8ded423e356a0 100644
--- a/tests/impersonation.py
+++ b/tests/test_impersonation.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/jobs.py b/tests/test_jobs.py
similarity index 77%
rename from tests/jobs.py
rename to tests/test_jobs.py
index f534b6540aa41..e3ae8be181b25 100644
--- a/tests/jobs.py
+++ b/tests/test_jobs.py
@@ -28,51 +28,50 @@
 import multiprocessing
 import os
 import shutil
-import six
 import threading
 import time
 import unittest
 from tempfile import mkdtemp
 
+import psutil
+import six
 import sqlalchemy
+from tests.compat import Mock, patch, MagicMock, PropertyMock
+from parameterized import parameterized
 
+from airflow.utils.db import create_session
 from airflow import AirflowException, settings, models
+from airflow import configuration
 from airflow.bin import cli
+import airflow.example_dags
 from airflow.executors import BaseExecutor, SequentialExecutor
-from airflow.jobs import BackfillJob, SchedulerJob, LocalTaskJob
-from airflow.models import DAG, DagModel, DagBag, DagRun, Pool, TaskInstance as TI
-from airflow.operators.dummy_operator import DummyOperator
+from airflow.jobs import BaseJob, BackfillJob, SchedulerJob, LocalTaskJob
+from airflow.models import DAG, DagModel, DagBag, DagRun, Pool, TaskInstance as TI, \
+    errors
+from airflow.models.slamiss import SlaMiss
 from airflow.operators.bash_operator import BashOperator
+from airflow.operators.dummy_operator import DummyOperator
 from airflow.task.task_runner.base_task_runner import BaseTaskRunner
 from airflow.utils import timezone
-
+from airflow.utils.dag_processing import SimpleDag, SimpleDagBag, list_py_file_paths
 from airflow.utils.dates import days_ago
 from airflow.utils.db import provide_session
+from airflow.utils.net import get_hostname
 from airflow.utils.state import State
 from airflow.utils.timeout import timeout
-from airflow.utils.dag_processing import SimpleDag, SimpleDagBag, list_py_file_paths
-from airflow.utils.net import get_hostname
-
-from mock import Mock, patch, MagicMock, PropertyMock
-from tests.executors.test_executor import TestExecutor
-
+from tests.test_utils.db import clear_db_runs, clear_db_pools, clear_db_dags, \
+    clear_db_sla_miss, clear_db_errors
 from tests.core import TEST_DAG_FOLDER
+from tests.executors.test_executor import TestExecutor
+from tests.compat import mock
 
-from airflow import configuration
 configuration.load_test_config()
 
-
-try:
-    from unittest import mock
-except ImportError:
-    try:
-        import mock
-    except ImportError:
-        mock = None
+logger = logging.getLogger(__name__)
 
 DEV_NULL = '/dev/null'
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
-
+TRY_NUMBER = 1
 # Include the words "airflow" and "dag" in the file contents,
 # tricking airflow into thinking these
 # files contain a DAG (otherwise Airflow will skip them)
@@ -86,9 +85,52 @@
     os.path.dirname(os.path.realpath(__file__)), 'dags')
 
 
+class BaseJobTest(unittest.TestCase):
+    class TestJob(BaseJob):
+        __mapper_args__ = {
+            'polymorphic_identity': 'TestJob'
+        }
+
+        def __init__(self, cb):
+            self.cb = cb
+            super(BaseJobTest.TestJob, self).__init__()
+
+        def _execute(self):
+            return self.cb()
+
+    def test_state_success(self):
+        job = self.TestJob(lambda: True)
+        job.run()
+
+        self.assertEqual(job.state, State.SUCCESS)
+        self.assertIsNotNone(job.end_date)
+
+    def test_state_sysexit(self):
+        import sys
+        job = self.TestJob(lambda: sys.exit(0))
+        job.run()
+
+        self.assertEqual(job.state, State.SUCCESS)
+        self.assertIsNotNone(job.end_date)
+
+    def test_state_failed(self):
+        def abort():
+            raise RuntimeError("fail")
+
+        job = self.TestJob(abort)
+        with self.assertRaises(RuntimeError):
+            job.run()
+
+        self.assertEqual(job.state, State.FAILED)
+        self.assertIsNotNone(job.end_date)
+
+
 class BackfillJobTest(unittest.TestCase):
 
     def setUp(self):
+        clear_db_runs()
+        clear_db_pools()
+
         self.parser = cli.CLIFactory.get_parser()
         self.dagbag = DagBag(include_examples=True)
 
@@ -154,23 +196,21 @@ def test_backfill_multi_dates(self):
     def test_backfill_examples(self):
         """
         Test backfilling example dags
-        """
 
-        # some DAGs really are just examples... but try to make them work!
-        skip_dags = [
-            'example_http_operator',
-            'example_twitter_dag',
-            'example_trigger_target_dag',
-            'example_trigger_controller_dag',  # tested above
-            'test_utils',  # sleeps forever
-            'example_kubernetes_executor',  # requires kubernetes cluster
-            'example_kubernetes_operator'  # requires kubernetes cluster
-        ]
+        Try to backfill some of the example dags. Be carefull, not all dags are suitable
+        for doing this. For example, a dag that sleeps forever, or does not have a
+        schedule won't work here since you simply can't backfill them.
+        """
+        include_dags = {
+            'example_branch_operator',
+            'example_bash_operator',
+            'example_skip_dag',
+            'latest_only'
+        }
 
-        logger = logging.getLogger('BackfillJobTest.test_backfill_examples')
         dags = [
             dag for dag in self.dagbag.dags.values()
-            if 'example_dags' in dag.full_filepath and dag.dag_id not in skip_dags
+            if 'example_dags' in dag.full_filepath and dag.dag_id in include_dags
         ]
 
         for dag in dags:
@@ -178,6 +218,11 @@ def test_backfill_examples(self):
                 start_date=DEFAULT_DATE,
                 end_date=DEFAULT_DATE)
 
+        # Make sure that we have the dags that we want to test available
+        # in the example_dags folder, if this assertion fails, one of the
+        # dags in the include_dags array isn't available anymore
+        self.assertEqual(len(include_dags), len(dags))
+
         for i, dag in enumerate(sorted(dags, key=lambda d: d.dag_id)):
             logger.info('*** Running example DAG #{}: {}'.format(i, dag.dag_id))
             job = BackfillJob(
@@ -214,6 +259,46 @@ def test_backfill_conf(self):
 
         self.assertEqual(conf, dr[0].conf)
 
+    def test_backfill_run_rescheduled(self):
+        dag = DAG(
+            dag_id='test_backfill_run_rescheduled',
+            start_date=DEFAULT_DATE,
+            schedule_interval='@daily')
+
+        with dag:
+            DummyOperator(
+                task_id='test_backfill_run_rescheduled_task-1',
+                dag=dag,
+            )
+
+        dag.clear()
+
+        executor = TestExecutor(do_update=True)
+
+        job = BackfillJob(dag=dag,
+                          executor=executor,
+                          start_date=DEFAULT_DATE,
+                          end_date=DEFAULT_DATE + datetime.timedelta(days=2),
+                          )
+        job.run()
+
+        ti = TI(task=dag.get_task('test_backfill_run_rescheduled_task-1'),
+                execution_date=DEFAULT_DATE)
+        ti.refresh_from_db()
+        ti.set_state(State.UP_FOR_RESCHEDULE)
+
+        job = BackfillJob(dag=dag,
+                          executor=executor,
+                          start_date=DEFAULT_DATE,
+                          end_date=DEFAULT_DATE + datetime.timedelta(days=2),
+                          rerun_failed_tasks=True
+                          )
+        job.run()
+        ti = TI(task=dag.get_task('test_backfill_run_rescheduled_task-1'),
+                execution_date=DEFAULT_DATE)
+        ti.refresh_from_db()
+        self.assertEqual(ti.state, State.SUCCESS)
+
     def test_backfill_rerun_failed_tasks(self):
         dag = DAG(
             dag_id='test_backfill_rerun_failed',
@@ -251,47 +336,47 @@ def test_backfill_rerun_failed_tasks(self):
         ti = TI(task=dag.get_task('test_backfill_rerun_failed_task-1'),
                 execution_date=DEFAULT_DATE)
         ti.refresh_from_db()
-        self.assertEquals(ti.state, State.SUCCESS)
+        self.assertEqual(ti.state, State.SUCCESS)
 
     def test_backfill_rerun_upstream_failed_tasks(self):
-            dag = DAG(
-                dag_id='test_backfill_rerun_upstream_failed',
-                start_date=DEFAULT_DATE,
-                schedule_interval='@daily')
+        dag = DAG(
+            dag_id='test_backfill_rerun_upstream_failed',
+            start_date=DEFAULT_DATE,
+            schedule_interval='@daily')
 
-            with dag:
-                t1 = DummyOperator(task_id='test_backfill_rerun_upstream_failed_task-1',
-                                   dag=dag)
-                t2 = DummyOperator(task_id='test_backfill_rerun_upstream_failed_task-2',
-                                   dag=dag)
-                t1.set_upstream(t2)
+        with dag:
+            t1 = DummyOperator(task_id='test_backfill_rerun_upstream_failed_task-1',
+                               dag=dag)
+            t2 = DummyOperator(task_id='test_backfill_rerun_upstream_failed_task-2',
+                               dag=dag)
+            t1.set_upstream(t2)
 
-            dag.clear()
-            executor = TestExecutor(do_update=True)
+        dag.clear()
+        executor = TestExecutor(do_update=True)
 
-            job = BackfillJob(dag=dag,
-                              executor=executor,
-                              start_date=DEFAULT_DATE,
-                              end_date=DEFAULT_DATE + datetime.timedelta(days=2),
-                              )
-            job.run()
+        job = BackfillJob(dag=dag,
+                          executor=executor,
+                          start_date=DEFAULT_DATE,
+                          end_date=DEFAULT_DATE + datetime.timedelta(days=2),
+                          )
+        job.run()
 
-            ti = TI(task=dag.get_task('test_backfill_rerun_upstream_failed_task-1'),
-                    execution_date=DEFAULT_DATE)
-            ti.refresh_from_db()
-            ti.set_state(State.UPSTREAM_FAILED)
+        ti = TI(task=dag.get_task('test_backfill_rerun_upstream_failed_task-1'),
+                execution_date=DEFAULT_DATE)
+        ti.refresh_from_db()
+        ti.set_state(State.UPSTREAM_FAILED)
 
-            job = BackfillJob(dag=dag,
-                              executor=executor,
-                              start_date=DEFAULT_DATE,
-                              end_date=DEFAULT_DATE + datetime.timedelta(days=2),
-                              rerun_failed_tasks=True
-                              )
-            job.run()
-            ti = TI(task=dag.get_task('test_backfill_rerun_upstream_failed_task-1'),
-                    execution_date=DEFAULT_DATE)
-            ti.refresh_from_db()
-            self.assertEquals(ti.state, State.SUCCESS)
+        job = BackfillJob(dag=dag,
+                          executor=executor,
+                          start_date=DEFAULT_DATE,
+                          end_date=DEFAULT_DATE + datetime.timedelta(days=2),
+                          rerun_failed_tasks=True
+                          )
+        job.run()
+        ti = TI(task=dag.get_task('test_backfill_rerun_upstream_failed_task-1'),
+                execution_date=DEFAULT_DATE)
+        ti.refresh_from_db()
+        self.assertEqual(ti.state, State.SUCCESS)
 
     def test_backfill_rerun_failed_tasks_without_flag(self):
         dag = DAG(
@@ -381,8 +466,6 @@ def test_backfill_ordered_concurrent_execute(self):
     def test_backfill_pooled_tasks(self):
         """
         Test that queued tasks are executed by BackfillJob
-
-        Test for https://github.com/airbnb/airflow/pull/1225
         """
         session = settings.Session()
         pool = Pool(pool='test_backfill_pooled_task_pool', slots=1)
@@ -431,7 +514,7 @@ def test_backfill_depends_on_past(self):
         # ti should have succeeded
         ti = TI(dag.tasks[0], run_date)
         ti.refresh_from_db()
-        self.assertEquals(ti.state, State.SUCCESS)
+        self.assertEqual(ti.state, State.SUCCESS)
 
     def test_run_ignores_all_dependencies(self):
         """
@@ -454,7 +537,7 @@ def test_run_ignores_all_dependencies(self):
             execution_date=DEFAULT_DATE)
 
         ti_dependent0.refresh_from_db()
-        self.assertEquals(ti_dependent0.state, State.FAILED)
+        self.assertEqual(ti_dependent0.state, State.FAILED)
 
         task1_id = 'test_run_dependency_task'
         args1 = ['run',
@@ -468,7 +551,7 @@ def test_run_ignores_all_dependencies(self):
             task=dag.get_task(task1_id),
             execution_date=DEFAULT_DATE + datetime.timedelta(days=1))
         ti_dependency.refresh_from_db()
-        self.assertEquals(ti_dependency.state, State.FAILED)
+        self.assertEqual(ti_dependency.state, State.FAILED)
 
         task2_id = 'test_run_dependent_task'
         args2 = ['run',
@@ -482,7 +565,7 @@ def test_run_ignores_all_dependencies(self):
             task=dag.get_task(task2_id),
             execution_date=DEFAULT_DATE + datetime.timedelta(days=1))
         ti_dependent.refresh_from_db()
-        self.assertEquals(ti_dependent.state, State.SUCCESS)
+        self.assertEqual(ti_dependent.state, State.SUCCESS)
 
     def test_run_naive_taskinstance(self):
         """
@@ -507,7 +590,7 @@ def test_run_naive_taskinstance(self):
             execution_date=NAIVE_DATE)
 
         ti_dependent0.refresh_from_db()
-        self.assertEquals(ti_dependent0.state, State.FAILED)
+        self.assertEqual(ti_dependent0.state, State.FAILED)
 
     def test_cli_backfill_depends_on_past(self):
         """
@@ -538,6 +621,41 @@ def test_cli_backfill_depends_on_past(self):
         self.assertEqual(ti.state, State.SUCCESS)
         dag.clear()
 
+    def test_cli_backfill_depends_on_past_backwards(self):
+        """
+        Test that CLI respects -B argument and raises on interaction with depends_on_past
+        """
+        dag_id = 'test_depends_on_past'
+        start_date = DEFAULT_DATE + datetime.timedelta(days=1)
+        end_date = start_date + datetime.timedelta(days=1)
+        args = [
+            'backfill',
+            dag_id,
+            '-l',
+            '-s',
+            start_date.isoformat(),
+            '-e',
+            end_date.isoformat(),
+            '-I'
+        ]
+        dag = self.dagbag.get_dag(dag_id)
+        dag.clear()
+
+        cli.backfill(self.parser.parse_args(args + ['-I']))
+        ti = TI(dag.get_task('test_dop_task'), end_date)
+        ti.refresh_from_db()
+        # runs fine forwards
+        self.assertEqual(ti.state, State.SUCCESS)
+
+        # raises backwards
+        expected_msg = 'You cannot backfill backwards because one or more tasks depend_on_past: {}'.format(
+            'test_dop_task')
+        self.assertRaisesRegexp(
+            AirflowException,
+            expected_msg,
+            cli.backfill,
+            self.parser.parse_args(args + ['-B']))
+
     def test_cli_receives_delay_arg(self):
         """
         Tests that the --delay argument is passed correctly to the BackfillJob
@@ -859,6 +977,59 @@ def test_backfill_execute_subdag(self):
         subdag.clear()
         dag.clear()
 
+    def test_subdag_clear_parentdag_downstream_clear(self):
+        dag = self.dagbag.get_dag('example_subdag_operator')
+        subdag_op_task = dag.get_task('section-1')
+
+        subdag = subdag_op_task.subdag
+        subdag.schedule_interval = '@daily'
+
+        executor = TestExecutor(do_update=True)
+        job = BackfillJob(dag=subdag,
+                          start_date=DEFAULT_DATE,
+                          end_date=DEFAULT_DATE,
+                          executor=executor,
+                          donot_pickle=True)
+
+        with timeout(seconds=30):
+            job.run()
+
+        ti0 = TI(
+            task=subdag.get_task('section-1-task-1'),
+            execution_date=DEFAULT_DATE)
+        ti0.refresh_from_db()
+        self.assertEqual(ti0.state, State.SUCCESS)
+
+        sdag = subdag.sub_dag(
+            task_regex='section-1-task-1',
+            include_downstream=True,
+            include_upstream=False)
+
+        sdag.clear(
+            start_date=DEFAULT_DATE,
+            end_date=DEFAULT_DATE,
+            include_parentdag=True)
+
+        ti0.refresh_from_db()
+        self.assertEqual(State.NONE, ti0.state)
+
+        ti1 = TI(
+            task=dag.get_task('some-other-task'),
+            execution_date=DEFAULT_DATE)
+        self.assertEqual(State.NONE, ti1.state)
+
+        # Checks that all the Downstream tasks for Parent DAG
+        # have been cleared
+        for task in subdag_op_task.downstream_list:
+            ti = TI(
+                task=dag.get_task(task.task_id),
+                execution_date=DEFAULT_DATE
+            )
+            self.assertEqual(State.NONE, ti.state)
+
+        subdag.clear()
+        dag.clear()
+
     def test_backfill_execute_subdag_with_removed_task(self):
         """
         Ensure that subdag operators execute properly in the case where
@@ -961,8 +1132,31 @@ def test_update_counters(self):
 
         ti_status.failed.clear()
 
+        # test for retry
+        ti.set_state(State.UP_FOR_RETRY, session)
+        ti_status.running[ti.key] = ti
+        job._update_counters(ti_status=ti_status)
+        self.assertTrue(len(ti_status.running) == 0)
+        self.assertTrue(len(ti_status.succeeded) == 0)
+        self.assertTrue(len(ti_status.skipped) == 0)
+        self.assertTrue(len(ti_status.failed) == 0)
+        self.assertTrue(len(ti_status.to_run) == 1)
+
+        ti_status.to_run.clear()
+
         # test for reschedule
-        # test for failed
+        ti.set_state(State.UP_FOR_RESCHEDULE, session)
+        ti_status.running[ti.key] = ti
+        job._update_counters(ti_status=ti_status)
+        self.assertTrue(len(ti_status.running) == 0)
+        self.assertTrue(len(ti_status.succeeded) == 0)
+        self.assertTrue(len(ti_status.skipped) == 0)
+        self.assertTrue(len(ti_status.failed) == 0)
+        self.assertTrue(len(ti_status.to_run) == 1)
+
+        ti_status.to_run.clear()
+
+        # test for none
         ti.set_state(State.NONE, session)
         ti_status.running[ti.key] = ti
         job._update_counters(ti_status=ti_status)
@@ -972,6 +1166,8 @@ def test_update_counters(self):
         self.assertTrue(len(ti_status.failed) == 0)
         self.assertTrue(len(ti_status.to_run) == 1)
 
+        ti_status.to_run.clear()
+
         session.close()
 
     def test_dag_get_run_dates(self):
@@ -984,7 +1180,8 @@ def get_test_dag_for_backfill(schedule_interval=None):
             DummyOperator(
                 task_id='dummy',
                 dag=dag,
-                owner='airflow')
+                owner='airflow',
+            )
             return dag
 
         test_dag = get_test_dag_for_backfill()
@@ -1001,10 +1198,67 @@ def get_test_dag_for_backfill(schedule_interval=None):
                              start_date=DEFAULT_DATE - datetime.timedelta(hours=3),
                              end_date=DEFAULT_DATE,))
 
+    def test_backfill_run_backwards(self):
+        dag = self.dagbag.get_dag("test_start_date_scheduling")
+        dag.clear()
+
+        job = BackfillJob(
+            dag=dag,
+            start_date=DEFAULT_DATE,
+            end_date=DEFAULT_DATE + datetime.timedelta(days=1),
+            run_backwards=True
+        )
+        job.run()
+
+        session = settings.Session()
+        tis = session.query(TI).filter(
+            TI.dag_id == 'test_start_date_scheduling' and TI.task_id == 'dummy'
+        ).order_by(TI.execution_date).all()
+
+        queued_times = [ti.queued_dttm for ti in tis]
+        self.assertTrue(queued_times == sorted(queued_times, reverse=True))
+        self.assertTrue(all([ti.state == State.SUCCESS for ti in tis]))
+
+        dag.clear()
+        session.close()
+
 
 class LocalTaskJobTest(unittest.TestCase):
     def setUp(self):
-        pass
+        clear_db_runs()
+
+    def test_localtaskjob_essential_attr(self):
+        """
+        Check whether essential attributes
+        of LocalTaskJob can be assigned with
+        proper values without intervention
+        """
+        dag = DAG(
+            'test_localtaskjob_essential_attr',
+            start_date=DEFAULT_DATE,
+            default_args={'owner': 'owner1'})
+
+        with dag:
+            op1 = DummyOperator(task_id='op1')
+
+        dag.clear()
+        dr = dag.create_dagrun(run_id="test",
+                               state=State.SUCCESS,
+                               execution_date=DEFAULT_DATE,
+                               start_date=DEFAULT_DATE)
+        ti = dr.get_task_instance(task_id=op1.task_id)
+
+        job1 = LocalTaskJob(task_instance=ti,
+                            ignore_ti_state=True,
+                            executor=SequentialExecutor())
+
+        essential_attr = ["dag_id", "job_type", "start_date", "hostname"]
+
+        check_result_1 = [hasattr(job1, attr) for attr in essential_attr]
+        self.assertTrue(all(check_result_1))
+
+        check_result_2 = [getattr(job1, attr) is not None for attr in essential_attr]
+        self.assertTrue(all(check_result_2))
 
     @patch('os.getpid')
     def test_localtaskjob_heartbeat(self, mock_pid):
@@ -1046,6 +1300,10 @@ def test_localtaskjob_heartbeat(self, mock_pid):
         mock_pid.return_value = 2
         self.assertRaises(AirflowException, job1.heartbeat_callback)
 
+    @unittest.skipIf('mysql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                     "flaky when run on mysql")
+    @unittest.skipIf('postgresql' in configuration.conf.get('core', 'sql_alchemy_conn'),
+                     'flaky when run on postgresql')
     def test_mark_success_no_kill(self):
         """
         Test that ensures that mark_success in the UI doesn't cause
@@ -1127,11 +1385,29 @@ def test_localtaskjob_double_trigger(self):
 class SchedulerJobTest(unittest.TestCase):
 
     def setUp(self):
-        self.dagbag = DagBag()
-        session = settings.Session()
-        session.query(models.DagRun).delete()
-        session.query(models.ImportError).delete()
-        session.commit()
+        clear_db_runs()
+        clear_db_pools()
+        clear_db_dags()
+        clear_db_sla_miss()
+        clear_db_errors()
+
+    @classmethod
+    def setUpClass(cls):
+        cls.dagbag = DagBag()
+
+        def getboolean(section, key):
+            if section.lower() == 'core' and key.lower() == 'load_examples':
+                return False
+            else:
+                return configuration.conf.getboolean(section, key)
+
+        cls.patcher = mock.patch('airflow.jobs.conf.getboolean')
+        mock_getboolean = cls.patcher.start()
+        mock_getboolean.side_effect = getboolean
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.patcher.stop()
 
     @staticmethod
     def run_single_scheduler_loop_with_no_dags(dags_folder):
@@ -1154,6 +1430,21 @@ def run_single_scheduler_loop_with_no_dags(dags_folder):
     def _make_simple_dag_bag(self, dags):
         return SimpleDagBag([SimpleDag(dag) for dag in dags])
 
+    def test_no_orphan_process_will_be_left(self):
+        empty_dir = mkdtemp()
+        current_process = psutil.Process()
+        old_children = current_process.children(recursive=True)
+        scheduler = SchedulerJob(subdir=empty_dir,
+                                 num_runs=1)
+        scheduler.executor = TestExecutor()
+        scheduler.run()
+        shutil.rmtree(empty_dir)
+
+        # Remove potential noise created by previous tests.
+        current_children = set(current_process.children(recursive=True)) - set(
+            old_children)
+        self.assertFalse(current_children)
+
     def test_process_executor_events(self):
         dag_id = "test_process_executor_events"
         dag_id2 = "test_process_executor_events_2"
@@ -1162,7 +1453,7 @@ def test_process_executor_events(self):
         dag = DAG(dag_id=dag_id, start_date=DEFAULT_DATE)
         dag2 = DAG(dag_id=dag_id2, start_date=DEFAULT_DATE)
         task1 = DummyOperator(dag=dag, task_id=task_id_1)
-        task2 = DummyOperator(dag=dag2, task_id=task_id_1)
+        DummyOperator(dag=dag2, task_id=task_id_1)
 
         dagbag1 = self._make_simple_dag_bag([dag])
         dagbag2 = self._make_simple_dag_bag([dag2])
@@ -1224,7 +1515,7 @@ def test_execute_task_instances_is_paused_wont_execute(self):
 
         scheduler._execute_task_instances(dagbag, [State.SCHEDULED])
         ti1.refresh_from_db()
-        self.assertEquals(State.SCHEDULED, ti1.state)
+        self.assertEqual(State.SCHEDULED, ti1.state)
 
     def test_execute_task_instances_no_dagrun_task_will_execute(self):
         """
@@ -1249,7 +1540,7 @@ def test_execute_task_instances_no_dagrun_task_will_execute(self):
 
         scheduler._execute_task_instances(dagbag, [State.SCHEDULED])
         ti1.refresh_from_db()
-        self.assertEquals(State.QUEUED, ti1.state)
+        self.assertEqual(State.QUEUED, ti1.state)
 
     def test_execute_task_instances_backfill_tasks_wont_execute(self):
         """
@@ -1278,7 +1569,7 @@ def test_execute_task_instances_backfill_tasks_wont_execute(self):
 
         scheduler._execute_task_instances(dagbag, [State.SCHEDULED])
         ti1.refresh_from_db()
-        self.assertEquals(State.SCHEDULED, ti1.state)
+        self.assertEqual(State.SCHEDULED, ti1.state)
 
     def test_find_executable_task_instances_backfill_nodagrun(self):
         dag_id = 'SchedulerJobTest.test_find_executable_task_instances_backfill_nodagrun'
@@ -1338,7 +1629,7 @@ def test_find_executable_task_instances_pool(self):
             TI(task2, dr1.execution_date),
             TI(task1, dr2.execution_date),
             TI(task2, dr2.execution_date)
-            ])
+        ])
         for ti in tis:
             ti.state = State.SCHEDULED
             session.merge(ti)
@@ -1449,6 +1740,39 @@ def test_find_executable_task_instances_concurrency(self):
 
         self.assertEqual(0, len(res))
 
+    def test_find_executable_task_instances_concurrency_queued(self):
+        dag_id = 'SchedulerJobTest.test_find_executable_task_instances_concurrency_queued'
+        dag = DAG(dag_id=dag_id, start_date=DEFAULT_DATE, concurrency=3)
+        task1 = DummyOperator(dag=dag, task_id='dummy1')
+        task2 = DummyOperator(dag=dag, task_id='dummy2')
+        task3 = DummyOperator(dag=dag, task_id='dummy3')
+        dagbag = self._make_simple_dag_bag([dag])
+
+        scheduler = SchedulerJob()
+        session = settings.Session()
+        dag_run = scheduler.create_dag_run(dag)
+
+        ti1 = TI(task1, dag_run.execution_date)
+        ti2 = TI(task2, dag_run.execution_date)
+        ti3 = TI(task3, dag_run.execution_date)
+        ti1.state = State.RUNNING
+        ti2.state = State.QUEUED
+        ti3.state = State.SCHEDULED
+
+        session.merge(ti1)
+        session.merge(ti2)
+        session.merge(ti3)
+
+        session.commit()
+
+        res = scheduler._find_executable_task_instances(
+            dagbag,
+            states=[State.SCHEDULED],
+            session=session)
+
+        self.assertEqual(1, len(res))
+        self.assertEqual(res[0].key, ti3.key)
+
     def test_find_executable_task_instances_task_concurrency(self):
         dag_id = 'SchedulerJobTest.test_find_executable_task_instances_task_concurrency'
         task_id_1 = 'dummy'
@@ -1635,7 +1959,7 @@ def test_enqueue_task_instances_with_queued_state(self):
         with patch.object(BaseExecutor, 'queue_command') as mock_queue_command:
             scheduler._enqueue_task_instances_with_queued_state(dagbag, [ti1])
 
-        mock_queue_command.assert_called()
+        assert mock_queue_command.called
 
     def test_execute_task_instances_nothing(self):
         dag_id = 'SchedulerJobTest.test_execute_task_instances_nothing'
@@ -1684,8 +2008,12 @@ def test_execute_task_instances(self):
         session.commit()
 
         self.assertEqual(State.RUNNING, dr1.state)
-        self.assertEqual(2, DAG.get_num_task_instances(dag_id, dag.task_ids,
-            states=[State.RUNNING], session=session))
+        self.assertEqual(
+            2,
+            DAG.get_num_task_instances(
+                dag_id, dag.task_ids, states=[State.RUNNING], session=session
+            )
+        )
 
         # create second dag run
         dr2 = scheduler.create_dag_run(dag)
@@ -1709,8 +2037,12 @@ def test_execute_task_instances(self):
         ti2.refresh_from_db()
         ti3.refresh_from_db()
         ti4.refresh_from_db()
-        self.assertEqual(3, DAG.get_num_task_instances(dag_id, dag.task_ids,
-            states=[State.RUNNING, State.QUEUED], session=session))
+        self.assertEqual(
+            3,
+            DAG.get_num_task_instances(
+                dag_id, dag.task_ids, states=[State.RUNNING, State.QUEUED], session=session
+            )
+        )
         self.assertEqual(State.RUNNING, ti1.state)
         self.assertEqual(State.RUNNING, ti2.state)
         six.assertCountEqual(self, [State.QUEUED, State.SCHEDULED], [ti3.state, ti4.state])
@@ -1754,44 +2086,29 @@ def test_execute_task_instances_limit(self):
             ti.refresh_from_db()
             self.assertEqual(State.QUEUED, ti.state)
 
+    @unittest.skipUnless("INTEGRATION" in os.environ,
+                         "The test is flaky with nondeterministic result")
     def test_change_state_for_tis_without_dagrun(self):
-        dag1 = DAG(
-            dag_id='test_change_state_for_tis_without_dagrun',
-            start_date=DEFAULT_DATE)
+        dag1 = DAG(dag_id='test_change_state_for_tis_without_dagrun', start_date=DEFAULT_DATE)
 
-        DummyOperator(
-            task_id='dummy',
-            dag=dag1,
-            owner='airflow')
-        DummyOperator(
-            task_id='dummy_b',
-            dag=dag1,
-            owner='airflow')
+        DummyOperator(task_id='dummy', dag=dag1, owner='airflow')
 
-        dag2 = DAG(
-            dag_id='test_change_state_for_tis_without_dagrun_dont_change',
-            start_date=DEFAULT_DATE)
+        DummyOperator(task_id='dummy_b', dag=dag1, owner='airflow')
 
-        DummyOperator(
-            task_id='dummy',
-            dag=dag2,
-            owner='airflow')
+        dag2 = DAG(dag_id='test_change_state_for_tis_without_dagrun_dont_change', start_date=DEFAULT_DATE)
 
-        dag3 = DAG(
-            dag_id='test_change_state_for_tis_without_dagrun_no_dagrun',
-            start_date=DEFAULT_DATE)
+        DummyOperator(task_id='dummy', dag=dag2, owner='airflow')
 
-        DummyOperator(
-            task_id='dummy',
-            dag=dag3,
-            owner='airflow')
+        dag3 = DAG(dag_id='test_change_state_for_tis_without_dagrun_no_dagrun', start_date=DEFAULT_DATE)
+
+        DummyOperator(task_id='dummy', dag=dag3, owner='airflow')
 
         session = settings.Session()
         dr1 = dag1.create_dagrun(run_id=DagRun.ID_PREFIX,
-                               state=State.RUNNING,
-                               execution_date=DEFAULT_DATE,
-                               start_date=DEFAULT_DATE,
-                               session=session)
+                                 state=State.RUNNING,
+                                 execution_date=DEFAULT_DATE,
+                                 start_date=DEFAULT_DATE,
+                                 session=session)
 
         dr2 = dag2.create_dagrun(run_id=DagRun.ID_PREFIX,
                                  state=State.RUNNING,
@@ -1835,7 +2152,7 @@ def test_change_state_for_tis_without_dagrun(self):
         self.assertEqual(ti2.state, State.SCHEDULED)
 
         ti3.refresh_from_db(session=session)
-        self.assertEquals(ti3.state, State.NONE)
+        self.assertEqual(ti3.state, State.NONE)
 
         dr1.refresh_from_db(session=session)
         dr1.state = State.FAILED
@@ -1850,7 +2167,7 @@ def test_change_state_for_tis_without_dagrun(self):
             new_state=State.NONE,
             session=session)
         ti1a.refresh_from_db(session=session)
-        self.assertEqual(ti1a.state, State.NONE)
+        self.assertEqual(ti1a.state, State.SCHEDULED)
 
         # don't touch ti1b
         ti1b.refresh_from_db(session=session)
@@ -1860,6 +2177,54 @@ def test_change_state_for_tis_without_dagrun(self):
         ti2.refresh_from_db(session=session)
         self.assertEqual(ti2.state, State.SCHEDULED)
 
+    def test_change_state_for_tasks_failed_to_execute(self):
+        dag = DAG(
+            dag_id='dag_id',
+            start_date=DEFAULT_DATE)
+
+        task = DummyOperator(
+            task_id='task_id',
+            dag=dag,
+            owner='airflow')
+
+        # If there's no left over task in executor.queued_tasks, nothing happens
+        session = settings.Session()
+        scheduler_job = SchedulerJob()
+        mock_logger = mock.MagicMock()
+        test_executor = TestExecutor()
+        scheduler_job.executor = test_executor
+        scheduler_job._logger = mock_logger
+        scheduler_job._change_state_for_tasks_failed_to_execute()
+        mock_logger.info.assert_not_called()
+
+        # Tasks failed to execute with QUEUED state will be set to SCHEDULED state.
+        session.query(TI).delete()
+        session.commit()
+        key = 'dag_id', 'task_id', DEFAULT_DATE, 1
+        test_executor.queued_tasks[key] = 'value'
+        ti = TI(task, DEFAULT_DATE)
+        ti.state = State.QUEUED
+        session.merge(ti)
+        session.commit()
+
+        scheduler_job._change_state_for_tasks_failed_to_execute()
+
+        ti.refresh_from_db()
+        self.assertEqual(State.SCHEDULED, ti.state)
+
+        # Tasks failed to execute with RUNNING state will not be set to SCHEDULED state.
+        session.query(TI).delete()
+        session.commit()
+        ti.state = State.RUNNING
+
+        session.merge(ti)
+        session.commit()
+
+        scheduler_job._change_state_for_tasks_failed_to_execute()
+
+        ti.refresh_from_db()
+        self.assertEqual(State.RUNNING, ti.state)
+
     def test_execute_helper_reset_orphaned_tasks(self):
         session = settings.Session()
         dag = DAG(
@@ -1888,13 +2253,13 @@ def test_execute_helper_reset_orphaned_tasks(self):
         session.commit()
 
         processor = mock.MagicMock()
-        processor.get_last_finish_time.return_value = None
 
         scheduler = SchedulerJob(num_runs=0, run_duration=0)
         executor = TestExecutor()
         scheduler.executor = executor
+        scheduler.processor_agent = processor
 
-        scheduler._execute_helper(processor_manager=processor)
+        scheduler._execute_helper()
 
         ti = dr.get_task_instance(task_id=op1.task_id, session=session)
         self.assertEqual(ti.state, State.NONE)
@@ -1902,6 +2267,51 @@ def test_execute_helper_reset_orphaned_tasks(self):
         ti2 = dr2.get_task_instance(task_id=op1.task_id, session=session)
         self.assertEqual(ti2.state, State.SCHEDULED)
 
+    @parameterized.expand([
+        [State.UP_FOR_RETRY, State.FAILED],
+        [State.QUEUED, State.NONE],
+        [State.SCHEDULED, State.NONE],
+        [State.UP_FOR_RESCHEDULE, State.NONE],
+    ])
+    def test_execute_helper_should_change_state_for_tis_without_dagrun(
+            self, initial_task_state, expected_task_state):
+        session = settings.Session()
+        dag = DAG(
+            'test_execute_helper_should_change_state_for_tis_without_dagrun',
+            start_date=DEFAULT_DATE,
+            default_args={'owner': 'owner1'})
+
+        with dag:
+            op1 = DummyOperator(task_id='op1')
+
+        # Create DAG run with FAILED state
+        dag.clear()
+        dr = dag.create_dagrun(run_id=DagRun.ID_PREFIX,
+                               state=State.FAILED,
+                               execution_date=DEFAULT_DATE,
+                               start_date=DEFAULT_DATE,
+                               session=session)
+        ti = dr.get_task_instance(task_id=op1.task_id, session=session)
+        ti.state = initial_task_state
+        session.commit()
+
+        # Create scheduler and mock calls to processor. Run duration is set
+        # to a high value to ensure loop is entered. Poll interval is 0 to
+        # avoid sleep. Done flag is set to true to exist the loop immediately.
+        scheduler = SchedulerJob(num_runs=0, processor_poll_interval=0)
+        executor = TestExecutor()
+        executor.queued_tasks
+        scheduler.executor = executor
+        processor = mock.MagicMock()
+        processor.harvest_simple_dags.return_value = [dag]
+        processor.done = True
+        scheduler.processor_agent = processor
+
+        scheduler._execute_helper()
+
+        ti = dr.get_task_instance(task_id=op1.task_id, session=session)
+        self.assertEqual(ti.state, expected_task_state)
+
     @provide_session
     def evaluate_dagrun(
             self,
@@ -2001,13 +2411,33 @@ def test_dagrun_root_fail_unfinished(self):
 
         # Mark the successful task as never having run since we want to see if the
         # dagrun will be in a running state despite haveing an unfinished task.
-        session = settings.Session()
-        ti = dr.get_task_instance('test_dagrun_unfinished', session=session)
-        ti.state = State.NONE
-        session.commit()
+        with create_session() as session:
+            ti = dr.get_task_instance('test_dagrun_unfinished', session=session)
+            ti.state = State.NONE
+            session.commit()
         dr_state = dr.update_state()
         self.assertEqual(dr_state, State.RUNNING)
 
+    def test_dagrun_root_after_dagrun_unfinished(self):
+        """
+        DagRuns with one successful and one future root task -> SUCCESS
+
+        Noted: the DagRun state could be still in running state during CI.
+        """
+        dag_id = 'test_dagrun_states_root_future'
+        dag = self.dagbag.get_dag(dag_id)
+        dag.clear()
+        scheduler = SchedulerJob(dag_id, num_runs=2)
+        # we can't use dag.run or evaluate_dagrun because it uses BackfillJob
+        # instead of SchedulerJob and BackfillJobs are allowed to not respect start dates
+        scheduler.run()
+
+        first_run = DagRun.find(dag_id=dag_id, execution_date=DEFAULT_DATE)[0]
+        ti_ids = [(ti.task_id, ti.state) for ti in first_run.get_task_instances()]
+
+        self.assertEqual(ti_ids, [('current', State.SUCCESS)])
+        self.assertIn(first_run.state, [State.SUCCESS, State.RUNNING])
+
     def test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date(self):
         """
         DagRun is marked a success if ignore_first_depends_on_past=True
@@ -2046,44 +2476,68 @@ def test_scheduler_start_date(self):
         """
         Test that the scheduler respects start_dates, even when DAGS have run
         """
+        with create_session() as session:
+            dag_id = 'test_start_date_scheduling'
+            dag = self.dagbag.get_dag(dag_id)
+            dag.clear()
+            self.assertTrue(dag.start_date > datetime.datetime.utcnow())
 
-        dag_id = 'test_start_date_scheduling'
-        dag = self.dagbag.get_dag(dag_id)
-        dag.clear()
-        self.assertTrue(dag.start_date > DEFAULT_DATE)
+            scheduler = SchedulerJob(dag_id,
+                                     subdir=os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py'),
+                                     num_runs=1)
+            scheduler.run()
 
-        scheduler = SchedulerJob(dag_id,
-                                 num_runs=2)
-        scheduler.run()
+            # zero tasks ran
+            self.assertEqual(
+                len(session.query(TI).filter(TI.dag_id == dag_id).all()), 0)
+            session.commit()
 
-        # zero tasks ran
-        session = settings.Session()
-        self.assertEqual(
-            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 0)
+            # previously, running this backfill would kick off the Scheduler
+            # because it would take the most recent run and start from there
+            # That behavior still exists, but now it will only do so if after the
+            # start date
+            backfill = BackfillJob(
+                dag=dag,
+                start_date=DEFAULT_DATE,
+                end_date=DEFAULT_DATE)
+            backfill.run()
 
-        # previously, running this backfill would kick off the Scheduler
-        # because it would take the most recent run and start from there
-        # That behavior still exists, but now it will only do so if after the
-        # start date
-        backfill = BackfillJob(
-            dag=dag,
-            start_date=DEFAULT_DATE,
-            end_date=DEFAULT_DATE)
-        backfill.run()
+            # one task ran
+            self.assertEqual(
+                len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
+            session.commit()
 
-        # one task ran
-        session = settings.Session()
-        self.assertEqual(
-            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
+            scheduler = SchedulerJob(dag_id,
+                                     subdir=os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py'),
+                                     num_runs=1)
+            scheduler.run()
 
+            # still one task
+            self.assertEqual(
+                len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
+            session.commit()
+
+    def test_scheduler_task_start_date(self):
+        """
+        Test that the scheduler respects task start dates that are different
+        from DAG start dates
+        """
+        dag_id = 'test_task_start_date_scheduling'
+        dag = self.dagbag.get_dag(dag_id)
+        dag.clear()
         scheduler = SchedulerJob(dag_id,
+                                 subdir=os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py'),
                                  num_runs=2)
         scheduler.run()
 
-        # still one task
         session = settings.Session()
-        self.assertEqual(
-            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
+        tiq = session.query(TI).filter(TI.dag_id == dag_id)
+        ti1s = tiq.filter(TI.task_id == 'dummy1').all()
+        ti2s = tiq.filter(TI.task_id == 'dummy2').all()
+        self.assertEqual(len(ti1s), 0)
+        self.assertEqual(len(ti2s), 2)
+        for t in ti2s:
+            self.assertEqual(t.state, State.SUCCESS)
 
     def test_scheduler_multiprocessing(self):
         """
@@ -2095,7 +2549,8 @@ def test_scheduler_multiprocessing(self):
             dag.clear()
 
         scheduler = SchedulerJob(dag_ids=dag_ids,
-                                 num_runs=2)
+                                 subdir=os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py'),
+                                 num_runs=1)
         scheduler.run()
 
         # zero tasks ran
@@ -2121,7 +2576,14 @@ def test_scheduler_dagrun_once(self):
         dr = scheduler.create_dag_run(dag)
         self.assertIsNone(dr)
 
-    def test_scheduler_process_task_instances(self):
+    @parameterized.expand([
+        [State.NONE, None, None],
+        [State.UP_FOR_RETRY, timezone.utcnow() - datetime.timedelta(minutes=30),
+            timezone.utcnow() - datetime.timedelta(minutes=15)],
+        [State.UP_FOR_RESCHEDULE, timezone.utcnow() - datetime.timedelta(minutes=30),
+            timezone.utcnow() - datetime.timedelta(minutes=15)],
+    ])
+    def test_scheduler_process_task_instances(self, state, start_date, end_date):
         """
         Test if _process_task_instances puts the right task instances into the
         queue.
@@ -2134,29 +2596,34 @@ def test_scheduler_process_task_instances(self):
             dag=dag,
             owner='airflow')
 
-        session = settings.Session()
-        orm_dag = DagModel(dag_id=dag.dag_id)
-        session.merge(orm_dag)
-        session.commit()
-        session.close()
+        with create_session() as session:
+            orm_dag = DagModel(dag_id=dag.dag_id)
+            session.merge(orm_dag)
 
         scheduler = SchedulerJob()
         dag.clear()
         dr = scheduler.create_dag_run(dag)
         self.assertIsNotNone(dr)
 
+        with create_session() as session:
+            tis = dr.get_task_instances(session=session)
+            for ti in tis:
+                ti.state = state
+                ti.start_date = start_date
+                ti.end_date = end_date
+
         queue = Mock()
         scheduler._process_task_instances(dag, queue=queue)
 
         queue.append.assert_called_with(
-            (dag.dag_id, dag_task1.task_id, DEFAULT_DATE)
+            (dag.dag_id, dag_task1.task_id, DEFAULT_DATE, TRY_NUMBER)
         )
 
     def test_scheduler_do_not_schedule_removed_task(self):
         dag = DAG(
             dag_id='test_scheduler_do_not_schedule_removed_task',
             start_date=DEFAULT_DATE)
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2186,7 +2653,7 @@ def test_scheduler_do_not_schedule_too_early(self):
         dag = DAG(
             dag_id='test_scheduler_do_not_schedule_too_early',
             start_date=timezone.datetime(2200, 1, 1))
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2208,11 +2675,25 @@ def test_scheduler_do_not_schedule_too_early(self):
 
         queue.put.assert_not_called()
 
+    def test_scheduler_do_not_schedule_without_tasks(self):
+        dag = DAG(
+            dag_id='test_scheduler_do_not_schedule_without_tasks',
+            start_date=DEFAULT_DATE)
+
+        with create_session() as session:
+            orm_dag = DagModel(dag_id=dag.dag_id)
+            session.merge(orm_dag)
+            scheduler = SchedulerJob()
+            dag.clear(session=session)
+            dag.start_date = None
+            dr = scheduler.create_dag_run(dag, session=session)
+            self.assertIsNone(dr)
+
     def test_scheduler_do_not_run_finished(self):
         dag = DAG(
             dag_id='test_scheduler_do_not_run_finished',
             start_date=DEFAULT_DATE)
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2248,7 +2729,7 @@ def test_scheduler_add_new_task(self):
             dag_id='test_scheduler_add_new_task',
             start_date=DEFAULT_DATE)
 
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2266,9 +2747,9 @@ def test_scheduler_add_new_task(self):
         self.assertIsNotNone(dr)
 
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 1)
+        self.assertEqual(len(tis), 1)
 
-        dag_task2 = DummyOperator(
+        DummyOperator(
             task_id='dummy2',
             dag=dag,
             owner='airflow')
@@ -2277,7 +2758,7 @@ def test_scheduler_add_new_task(self):
         scheduler._process_task_instances(dag, queue=queue)
 
         tis = dr.get_task_instances()
-        self.assertEquals(len(tis), 2)
+        self.assertEqual(len(tis), 2)
 
     def test_scheduler_verify_max_active_runs(self):
         """
@@ -2288,7 +2769,7 @@ def test_scheduler_verify_max_active_runs(self):
             start_date=DEFAULT_DATE)
         dag.max_active_runs = 1
 
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2317,7 +2798,7 @@ def test_scheduler_fail_dagrun_timeout(self):
             start_date=DEFAULT_DATE)
         dag.dagrun_timeout = datetime.timedelta(seconds=60)
 
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2340,12 +2821,15 @@ def test_scheduler_fail_dagrun_timeout(self):
         self.assertIsNotNone(dr2)
 
         dr.refresh_from_db(session=session)
-        self.assertEquals(dr.state, State.FAILED)
+        self.assertEqual(dr.state, State.FAILED)
 
     def test_scheduler_verify_max_active_runs_and_dagrun_timeout(self):
         """
-        Test if a a dagrun will not be scheduled if max_dag_runs has been reached and dagrun_timeout is not reached
-        Test if a a dagrun will be scheduled if max_dag_runs has been reached but dagrun_timeout is also reached
+        Test if a a dagrun will not be scheduled if max_dag_runs
+        has been reached and dagrun_timeout is not reached
+
+        Test if a a dagrun will be scheduled if max_dag_runs has
+        been reached but dagrun_timeout is also reached
         """
         dag = DAG(
             dag_id='test_scheduler_verify_max_active_runs_and_dagrun_timeout',
@@ -2353,7 +2837,7 @@ def test_scheduler_verify_max_active_runs_and_dagrun_timeout(self):
         dag.max_active_runs = 1
         dag.dagrun_timeout = datetime.timedelta(seconds=60)
 
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2417,7 +2901,7 @@ def test_scheduler_max_active_runs_respected_after_clear(self):
         scheduler._process_task_instances(dag, queue=queue)
 
         queue.append.assert_called_with(
-            (dag.dag_id, dag_task1.task_id, DEFAULT_DATE)
+            (dag.dag_id, dag_task1.task_id, DEFAULT_DATE, TRY_NUMBER)
         )
 
     @patch.object(TI, 'pool_full')
@@ -2451,12 +2935,12 @@ def test_scheduler_verify_pool_full(self, mock_pool_full):
         # Create 2 dagruns, which will create 2 task instances.
         dr = scheduler.create_dag_run(dag)
         self.assertIsNotNone(dr)
-        self.assertEquals(dr.execution_date, DEFAULT_DATE)
+        self.assertEqual(dr.execution_date, DEFAULT_DATE)
         dr = scheduler.create_dag_run(dag)
         self.assertIsNotNone(dr)
         queue = []
         scheduler._process_task_instances(dag, queue=queue)
-        self.assertEquals(len(queue), 2)
+        self.assertEqual(len(queue), 2)
         dagbag = self._make_simple_dag_bag([dag])
 
         # Recreated part of the scheduler here, to kick off tasks -> executor
@@ -2475,7 +2959,7 @@ def test_scheduler_verify_pool_full(self, mock_pool_full):
                                           (State.SCHEDULED,
                                            State.UP_FOR_RETRY))
 
-        self.assertEquals(len(scheduler.executor.queued_tasks), 1)
+        self.assertEqual(len(scheduler.executor.queued_tasks), 1)
 
     def test_scheduler_auto_align(self):
         """
@@ -2489,7 +2973,7 @@ def test_scheduler_auto_align(self):
             start_date=timezone.datetime(2016, 1, 1, 10, 10, 0),
             schedule_interval="4 5 * * *"
         )
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2504,14 +2988,14 @@ def test_scheduler_auto_align(self):
 
         dr = scheduler.create_dag_run(dag)
         self.assertIsNotNone(dr)
-        self.assertEquals(dr.execution_date, timezone.datetime(2016, 1, 2, 5, 4))
+        self.assertEqual(dr.execution_date, timezone.datetime(2016, 1, 2, 5, 4))
 
         dag = DAG(
             dag_id='test_scheduler_auto_align_2',
             start_date=timezone.datetime(2016, 1, 1, 10, 10, 0),
             schedule_interval="10 10 * * *"
         )
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2526,7 +3010,7 @@ def test_scheduler_auto_align(self):
 
         dr = scheduler.create_dag_run(dag)
         self.assertIsNotNone(dr)
-        self.assertEquals(dr.execution_date, timezone.datetime(2016, 1, 1, 10, 10))
+        self.assertEqual(dr.execution_date, timezone.datetime(2016, 1, 1, 10, 10))
 
     def test_scheduler_reschedule(self):
         """
@@ -2542,7 +3026,7 @@ def test_scheduler_reschedule(self):
         dag = DAG(
             dag_id='test_scheduler_reschedule',
             start_date=DEFAULT_DATE)
-        dag_task1 = DummyOperator(
+        DummyOperator(
             task_id='dummy',
             dag=dag,
             owner='airflow')
@@ -2572,13 +3056,83 @@ def do_schedule(function, function2):
             scheduler.run()
 
         do_schedule()
-        self.assertEquals(1, len(executor.queued_tasks))
+        self.assertEqual(1, len(executor.queued_tasks))
         executor.queued_tasks.clear()
 
         do_schedule()
-        self.assertEquals(2, len(executor.queued_tasks))
+        self.assertEqual(2, len(executor.queued_tasks))
 
     def test_scheduler_sla_miss_callback(self):
+        """
+        Test that the scheduler calls the sla miss callback
+        """
+        session = settings.Session()
+
+        sla_callback = MagicMock()
+
+        # Create dag with a start of 1 day ago, but an sla of 0
+        # so we'll already have an sla_miss on the books.
+        test_start_date = days_ago(1)
+        dag = DAG(dag_id='test_sla_miss',
+                  sla_miss_callback=sla_callback,
+                  default_args={'start_date': test_start_date,
+                                'sla': datetime.timedelta()})
+
+        task = DummyOperator(task_id='dummy',
+                             dag=dag,
+                             owner='airflow')
+
+        session.merge(models.TaskInstance(task=task,
+                                          execution_date=test_start_date,
+                                          state='success'))
+
+        session.merge(SlaMiss(task_id='dummy',
+                              dag_id='test_sla_miss',
+                              execution_date=test_start_date))
+
+        scheduler = SchedulerJob(dag_id='test_sla_miss',
+                                 num_runs=1)
+        scheduler.manage_slas(dag=dag, session=session)
+
+        assert sla_callback.called
+
+    def test_scheduler_sla_miss_callback_invalid_sla(self):
+        """
+        Test that the scheduler does not call the sla miss callback when
+        given an invalid sla
+        """
+        session = settings.Session()
+
+        sla_callback = MagicMock()
+
+        # Create dag with a start of 1 day ago, but an sla of 0
+        # so we'll already have an sla_miss on the books.
+        # Pass anything besides a timedelta object to the sla argument.
+        test_start_date = days_ago(1)
+        dag = DAG(dag_id='test_sla_miss',
+                  sla_miss_callback=sla_callback,
+                  default_args={'start_date': test_start_date,
+                                'sla': None})
+
+        task = DummyOperator(task_id='dummy',
+                             dag=dag,
+                             owner='airflow')
+
+        session.merge(models.TaskInstance(task=task,
+                                          execution_date=test_start_date,
+                                          state='success'))
+
+        session.merge(SlaMiss(task_id='dummy',
+                              dag_id='test_sla_miss',
+                              execution_date=test_start_date))
+
+        scheduler = SchedulerJob(dag_id='test_sla_miss',
+                                 num_runs=1)
+        scheduler.manage_slas(dag=dag, session=session)
+
+        sla_callback.assert_not_called()
+
+    def test_scheduler_sla_miss_callback_sent_notification(self):
         """
         Test that the scheduler does not call the sla_miss_callback when a notification has already been sent
         """
@@ -2587,7 +3141,8 @@ def test_scheduler_sla_miss_callback(self):
         # Mock the callback function so we can verify that it was not called
         sla_callback = MagicMock()
 
-        # Create dag with a start of 2 days ago, but an sla of 1 day ago so we'll already have an sla_miss on the books
+        # Create dag with a start of 2 days ago, but an sla of 1 day
+        # ago so we'll already have an sla_miss on the books
         test_start_date = days_ago(2)
         dag = DAG(dag_id='test_sla_miss',
                   sla_miss_callback=sla_callback,
@@ -2604,11 +3159,11 @@ def test_scheduler_sla_miss_callback(self):
                                           state='success'))
 
         # Create an SlaMiss where notification was sent, but email was not
-        session.merge(models.SlaMiss(task_id='dummy',
-                                     dag_id='test_sla_miss',
-                                     execution_date=test_start_date,
-                                     email_sent=False,
-                                     notification_sent=True))
+        session.merge(SlaMiss(task_id='dummy',
+                              dag_id='test_sla_miss',
+                              execution_date=test_start_date,
+                              email_sent=False,
+                              notification_sent=True))
 
         # Now call manage_slas and see if the sla_miss callback gets called
         scheduler = SchedulerJob(dag_id='test_sla_miss',
@@ -2641,9 +3196,9 @@ def test_scheduler_sla_miss_callback_exception(self):
                                           state='Success'))
 
         # Create an SlaMiss where notification was sent, but email was not
-        session.merge(models.SlaMiss(task_id='dummy',
-                                     dag_id='test_sla_miss',
-                                     execution_date=test_start_date))
+        session.merge(SlaMiss(task_id='dummy',
+                              dag_id='test_sla_miss',
+                              execution_date=test_start_date))
 
         # Now call manage_slas and see if the sla_miss callback gets called
         scheduler = SchedulerJob(dag_id='test_sla_miss')
@@ -2651,7 +3206,7 @@ def test_scheduler_sla_miss_callback_exception(self):
         with mock.patch('airflow.jobs.SchedulerJob.log',
                         new_callable=PropertyMock) as mock_log:
             scheduler.manage_slas(dag=dag, session=session)
-            sla_callback.assert_called()
+            assert sla_callback.called
             mock_log().exception.assert_called_with(
                 'Could not call sla_miss_callback for DAG %s',
                 'test_sla_miss')
@@ -2683,9 +3238,9 @@ def test_scheduler_sla_miss_email_exception(self, mock_send_email):
                                           state='Success'))
 
         # Create an SlaMiss where notification was sent, but email was not
-        session.merge(models.SlaMiss(task_id='dummy',
-                                     dag_id='test_sla_miss',
-                                     execution_date=test_start_date))
+        session.merge(SlaMiss(task_id='dummy',
+                              dag_id='test_sla_miss',
+                              execution_date=test_start_date))
 
         scheduler = SchedulerJob(dag_id='test_sla_miss',
                                  num_runs=1)
@@ -2743,7 +3298,7 @@ def do_schedule(function, function2):
             scheduler.run()
 
         do_schedule()
-        self.assertEquals(1, len(executor.queued_tasks))
+        self.assertEqual(1, len(executor.queued_tasks))
 
         def run_with_error(task):
             try:
@@ -2752,7 +3307,8 @@ def run_with_error(task):
                 pass
 
         ti_tuple = six.next(six.itervalues(executor.queued_tasks))
-        (command, priority, queue, ti) = ti_tuple
+        (command, priority, queue, simple_ti) = ti_tuple
+        ti = simple_ti.construct_task_instance()
         ti.task = dag_task1
 
         self.assertEqual(ti.try_number, 1)
@@ -2770,13 +3326,24 @@ def run_with_error(task):
         do_schedule()
         self.assertTrue(executor.has_task(ti))
         ti.refresh_from_db()
+        # removing self.assertEqual(ti.state, State.SCHEDULED)
+        # as scheduler will move state from SCHEDULED to QUEUED
+
+        # now the executor has cleared and it should be allowed the re-queue,
+        # but tasks stay in the executor.queued_tasks after executor.heartbeat()
+        # will be set back to SCHEDULED state
+        executor.queued_tasks.clear()
+        do_schedule()
+        ti.refresh_from_db()
+
         self.assertEqual(ti.state, State.SCHEDULED)
 
-        # now the executor has cleared and it should be allowed the re-queue
+        # To verify that task does get re-queued.
         executor.queued_tasks.clear()
+        executor.do_update = True
         do_schedule()
         ti.refresh_from_db()
-        self.assertEqual(ti.state, State.QUEUED)
+        self.assertIn(ti.state, [State.RUNNING, State.SUCCESS])
 
     @unittest.skipUnless("INTEGRATION" in os.environ, "Can only run end to end")
     def test_retry_handling_job(self):
@@ -2794,8 +3361,8 @@ def test_retry_handling_job(self):
         scheduler.run()
 
         session = settings.Session()
-        ti = session.query(TI).filter(TI.dag_id==dag.dag_id,
-                                      TI.task_id==dag_task1.task_id).first()
+        ti = session.query(TI).filter(TI.dag_id == dag.dag_id,
+                                      TI.task_id == dag_task1.task_id).first()
 
         # make sure the counter has increased
         self.assertEqual(ti.try_number, 2)
@@ -2821,7 +3388,9 @@ def test_scheduler_run_duration(self):
         logging.info("Test ran in %.2fs, expected %.2fs",
                      run_duration,
                      expected_run_duration)
-        self.assertLess(run_duration - expected_run_duration, 5.0)
+        # 5s to wait for child process to exit, 1s dummy sleep
+        # in scheduler loop to prevent excessive logs and 1s for last loop to finish.
+        self.assertLess(run_duration - expected_run_duration, 6.0)
 
     def test_dag_with_system_exit(self):
         """
@@ -2845,9 +3414,9 @@ def test_dag_with_system_exit(self):
                                  subdir=dag_directory,
                                  num_runs=1)
         scheduler.run()
-        session = settings.Session()
-        self.assertEqual(
-            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
+        with create_session() as session:
+            self.assertEqual(
+                len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
 
     def test_dag_get_active_runs(self):
         """
@@ -2855,7 +3424,8 @@ def test_dag_get_active_runs(self):
         """
 
         now = timezone.utcnow()
-        six_hours_ago_to_the_hour = (now - datetime.timedelta(hours=6)).replace(minute=0, second=0, microsecond=0)
+        six_hours_ago_to_the_hour = \
+            (now - datetime.timedelta(hours=6)).replace(minute=0, second=0, microsecond=0)
 
         START_DATE = six_hours_ago_to_the_hour
         DAG_NAME1 = 'get_active_runs_test'
@@ -2898,7 +3468,7 @@ def test_dag_get_active_runs(self):
 
         try:
             running_date = running_dates[0]
-        except:
+        except Exception:
             running_date = 'Except'
 
         self.assertEqual(execution_date, running_date, 'Running Date must match Execution Date')
@@ -2981,10 +3551,9 @@ def setup_dag(dag_id, schedule_interval, start_date, catchup):
         dr = scheduler.create_dag_run(dag4)
         self.assertIsNotNone(dr)
 
-
     def test_add_unparseable_file_before_sched_start_creates_import_error(self):
+        dags_folder = mkdtemp()
         try:
-            dags_folder = mkdtemp()
             unparseable_filename = os.path.join(dags_folder, TEMP_DAG_FILENAME)
             with open(unparseable_filename, 'w') as unparseable_file:
                 unparseable_file.writelines(UNPARSEABLE_DAG_FILE_CONTENTS)
@@ -2992,8 +3561,8 @@ def test_add_unparseable_file_before_sched_start_creates_import_error(self):
         finally:
             shutil.rmtree(dags_folder)
 
-        session = settings.Session()
-        import_errors = session.query(models.ImportError).all()
+        with create_session() as session:
+            import_errors = session.query(errors.ImportError).all()
 
         self.assertEqual(len(import_errors), 1)
         import_error = import_errors[0]
@@ -3003,8 +3572,8 @@ def test_add_unparseable_file_before_sched_start_creates_import_error(self):
                          "invalid syntax ({}, line 1)".format(TEMP_DAG_FILENAME))
 
     def test_add_unparseable_file_after_sched_start_creates_import_error(self):
+        dags_folder = mkdtemp()
         try:
-            dags_folder = mkdtemp()
             unparseable_filename = os.path.join(dags_folder, TEMP_DAG_FILENAME)
             self.run_single_scheduler_loop_with_no_dags(dags_folder)
 
@@ -3014,8 +3583,8 @@ def test_add_unparseable_file_after_sched_start_creates_import_error(self):
         finally:
             shutil.rmtree(dags_folder)
 
-        session = settings.Session()
-        import_errors = session.query(models.ImportError).all()
+        with create_session() as session:
+            import_errors = session.query(errors.ImportError).all()
 
         self.assertEqual(len(import_errors), 1)
         import_error = import_errors[0]
@@ -3035,8 +3604,8 @@ def test_no_import_errors_with_parseable_dag(self):
         finally:
             shutil.rmtree(dags_folder)
 
-        session = settings.Session()
-        import_errors = session.query(models.ImportError).all()
+        with create_session() as session:
+            import_errors = session.query(errors.ImportError).all()
 
         self.assertEqual(len(import_errors), 0)
 
@@ -3061,7 +3630,7 @@ def test_new_import_error_replaces_old(self):
             shutil.rmtree(dags_folder)
 
         session = settings.Session()
-        import_errors = session.query(models.ImportError).all()
+        import_errors = session.query(errors.ImportError).all()
 
         self.assertEqual(len(import_errors), 1)
         import_error = import_errors[0]
@@ -3089,7 +3658,7 @@ def test_remove_error_clears_import_error(self):
             shutil.rmtree(dags_folder)
 
         session = settings.Session()
-        import_errors = session.query(models.ImportError).all()
+        import_errors = session.query(errors.ImportError).all()
 
         self.assertEqual(len(import_errors), 0)
 
@@ -3108,8 +3677,8 @@ def test_remove_file_clears_import_error(self):
         # Rerun the scheduler once the dag file has been removed
         self.run_single_scheduler_loop_with_no_dags(dags_folder)
 
-        session = settings.Session()
-        import_errors = session.query(models.ImportError).all()
+        with create_session() as session:
+            import_errors = session.query(errors.ImportError).all()
 
         self.assertEqual(len(import_errors), 0)
 
@@ -3118,16 +3687,33 @@ def test_list_py_file_paths(self):
         [JIRA-1357] Test the 'list_py_file_paths' function used by the
         scheduler to list and load DAGs.
         """
-        detected_files = []
-        expected_files = []
+        detected_files = set()
+        expected_files = set()
+        # No_dags is empty, _invalid_ is ignored by .airflowignore
+        ignored_files = [
+            'no_dags.py',
+            'test_invalid_cron.py',
+            'test_zip_invalid_cron.zip',
+        ]
         for file_name in os.listdir(TEST_DAGS_FOLDER):
             if file_name.endswith('.py') or file_name.endswith('.zip'):
-                if file_name not in ['no_dags.py']:
-                    expected_files.append(
+                if file_name not in ignored_files:
+                    expected_files.add(
                         '{}/{}'.format(TEST_DAGS_FOLDER, file_name))
-        for file_path in list_py_file_paths(TEST_DAGS_FOLDER):
-            detected_files.append(file_path)
-        self.assertEqual(sorted(detected_files), sorted(expected_files))
+        for file_path in list_py_file_paths(TEST_DAGS_FOLDER, include_examples=False):
+            detected_files.add(file_path)
+        self.assertEqual(detected_files, expected_files)
+
+        example_dag_folder = airflow.example_dags.__path__[0]
+        for root, dirs, files in os.walk(example_dag_folder):
+            for file_name in files:
+                if file_name.endswith('.py') or file_name.endswith('.zip'):
+                    if file_name not in ['__init__.py']:
+                        expected_files.add(os.path.join(root, file_name))
+        detected_files.clear()
+        for file_path in list_py_file_paths(TEST_DAGS_FOLDER, include_examples=True):
+            detected_files.add(file_path)
+        self.assertEqual(detected_files, expected_files)
 
     def test_reset_orphaned_tasks_nothing(self):
         """Try with nothing. """
@@ -3155,7 +3741,7 @@ def test_reset_orphaned_tasks_external_triggered_dag(self):
         session.commit()
 
         reset_tis = scheduler.reset_state_for_orphaned_tasks(session=session)
-        self.assertEquals(1, len(reset_tis))
+        self.assertEqual(1, len(reset_tis))
 
     def test_reset_orphaned_tasks_backfill_dag(self):
         dag_id = 'test_reset_orphaned_tasks_backfill_dag'
@@ -3176,7 +3762,7 @@ def test_reset_orphaned_tasks_backfill_dag(self):
         session.commit()
 
         self.assertTrue(dr1.is_backfill)
-        self.assertEquals(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
+        self.assertEqual(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
 
     def test_reset_orphaned_tasks_specified_dagrun(self):
         """Try to reset when we specify a dagrun and ensure nothing else is."""
@@ -3204,11 +3790,11 @@ def test_reset_orphaned_tasks_specified_dagrun(self):
         session.commit()
 
         reset_tis = scheduler.reset_state_for_orphaned_tasks(filter_by_dag_run=dr2, session=session)
-        self.assertEquals(1, len(reset_tis))
+        self.assertEqual(1, len(reset_tis))
         ti1.refresh_from_db(session=session)
         ti2.refresh_from_db(session=session)
-        self.assertEquals(State.SCHEDULED, ti1.state)
-        self.assertEquals(State.NONE, ti2.state)
+        self.assertEqual(State.SCHEDULED, ti1.state)
+        self.assertEqual(State.NONE, ti2.state)
 
     def test_reset_orphaned_tasks_nonexistent_dagrun(self):
         """Make sure a task in an orphaned state is not reset if it has no dagrun. """
@@ -3229,7 +3815,7 @@ def test_reset_orphaned_tasks_nonexistent_dagrun(self):
         session.merge(ti)
         session.commit()
 
-        self.assertEquals(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
+        self.assertEqual(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
 
     def test_reset_orphaned_tasks_no_orphans(self):
         dag_id = 'test_reset_orphaned_tasks_no_orphans'
@@ -3248,9 +3834,9 @@ def test_reset_orphaned_tasks_no_orphans(self):
         session.merge(tis[0])
         session.commit()
 
-        self.assertEquals(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
+        self.assertEqual(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
         tis[0].refresh_from_db()
-        self.assertEquals(State.RUNNING, tis[0].state)
+        self.assertEqual(State.RUNNING, tis[0].state)
 
     def test_reset_orphaned_tasks_non_running_dagruns(self):
         """Ensure orphaned tasks with non-running dagruns are not reset."""
@@ -3265,13 +3851,13 @@ def test_reset_orphaned_tasks_non_running_dagruns(self):
         dr1 = scheduler.create_dag_run(dag)
         dr1.state = State.SUCCESS
         tis = dr1.get_task_instances(session=session)
-        self.assertEquals(1, len(tis))
+        self.assertEqual(1, len(tis))
         tis[0].state = State.SCHEDULED
         session.merge(dr1)
         session.merge(tis[0])
         session.commit()
 
-        self.assertEquals(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
+        self.assertEqual(0, len(scheduler.reset_state_for_orphaned_tasks(session=session)))
 
     def test_reset_orphaned_tasks_with_orphans(self):
         """Create dagruns and esnure only ones with correct states are reset."""
diff --git a/tests/test_logging_config.py b/tests/test_logging_config.py
index d6f1194f75d67..29d101f8be2e3 100644
--- a/tests/test_logging_config.py
+++ b/tests/test_logging_config.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,15 +17,21 @@
 # specific language governing permissions and limitations
 # under the License.
 import os
-import shutil
+import six
 import sys
 import tempfile
-import unittest
-from mock import patch, mock
 
 from airflow import configuration as conf
 from airflow.configuration import mkdir_p
 from airflow.exceptions import AirflowConfigException
+from tests.compat import mock, patch
+
+
+if six.PY2:
+    # Need `assertWarns` back-ported from unittest2
+    import unittest2 as unittest
+else:
+    import unittest
 
 SETTINGS_FILE_VALID = """
 LOGGING_CONFIG = {
@@ -41,14 +47,24 @@
             'class': 'logging.StreamHandler',
             'formatter': 'airflow.task',
             'stream': 'ext://sys.stdout'
-        }
+        },
+        'task': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'airflow.task',
+            'stream': 'ext://sys.stdout'
+        },
     },
     'loggers': {
         'airflow': {
             'handlers': ['console'],
             'level': 'INFO',
             'propagate': False
-        }
+        },
+        'airflow.task': {
+            'handlers': ['task'],
+            'level': 'INFO',
+            'propagate': False,
+        },
     }
 }
 """
@@ -132,7 +148,7 @@ def __enter__(self):
         return self.settings_file
 
     def __exit__(self, *exc_info):
-        #shutil.rmtree(self.settings_root)
+        # shutil.rmtree(self.settings_root)
         # Reset config
         conf.set('core', 'logging_config_class', '')
         sys.path.remove(self.settings_root)
@@ -200,18 +216,21 @@ def test_loading_no_local_settings(self):
     # When the key is not available in the configuration
     def test_when_the_config_key_does_not_exists(self):
         from airflow import logging_config
+        conf_get = conf.get
 
         def side_effect(*args):
             if args[1] == 'logging_config_class':
                 raise AirflowConfigException
             else:
-                return "bla_bla_from_test"
+                return conf_get(*args)
 
         logging_config.conf.get = mock.Mock(side_effect=side_effect)
 
         with patch.object(logging_config.log, 'debug') as mock_debug:
             logging_config.configure_logging()
-            mock_debug.assert_any_call('Could not find key logging_config_class in config')
+            mock_debug.assert_any_call(
+                'Could not find key logging_config_class in config'
+            )
 
     # Just default
     def test_loading_local_settings_without_logging_config(self):
@@ -222,6 +241,33 @@ def test_loading_local_settings_without_logging_config(self):
                 'Unable to load custom logging, using default config instead'
             )
 
+    def test_1_9_config(self):
+        from airflow.logging_config import configure_logging
+        conf.set('core', 'task_log_reader', 'file.task')
+        try:
+            with self.assertWarnsRegex(DeprecationWarning, r'file.task'):
+                configure_logging()
+                self.assertEqual(conf.get('core', 'task_log_reader'), 'task')
+        finally:
+            conf.remove_option('core', 'task_log_reader', remove_default=False)
+
+    def test_loading_remote_logging_with_wasb_handler(self):
+        """Test if logging can be configured successfully for Azure Blob Storage"""
+        import logging
+        from airflow.config_templates import airflow_local_settings
+        from airflow.logging_config import configure_logging
+        from airflow.utils.log.wasb_task_handler import WasbTaskHandler
+
+        conf.set('core', 'remote_logging', 'True')
+        conf.set('core', 'remote_log_conn_id', 'some_wasb')
+        conf.set('core', 'remote_base_log_folder', 'wasb://some-folder')
+
+        six.moves.reload_module(airflow_local_settings)
+        configure_logging()
+
+        logger = logging.getLogger('airflow.task')
+        self.assertIsInstance(logger.handlers[0], WasbTaskHandler)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/models.py b/tests/test_models.py
similarity index 69%
rename from tests/models.py
rename to tests/test_models.py
index d38681741daa1..b04c012ff9663 100644
--- a/tests/models.py
+++ b/tests/test_models.py
@@ -23,40 +23,49 @@
 from __future__ import unicode_literals
 
 import datetime
+import inspect
 import logging
 import os
-import pendulum
-import unittest
-import time
-import six
 import re
-import urllib
 import textwrap
-import inspect
+import time
+import unittest
+import urllib
+import uuid
+import shutil
+from tempfile import NamedTemporaryFile, mkdtemp
+
+import pendulum
+import six
+from cryptography.fernet import Fernet
+from freezegun import freeze_time
+from mock import ANY, mock_open, patch
+from parameterized import parameterized
 
-from airflow import configuration, models, settings, AirflowException
+from airflow import AirflowException, configuration, models, settings
+from airflow.contrib.sensors.python_sensor import PythonSensor
 from airflow.exceptions import AirflowDagCycleException, AirflowSkipException
 from airflow.jobs import BackfillJob
-from airflow.models import DAG, TaskInstance as TI
-from airflow.models import DagRun
+from airflow.models import DAG, TaskInstance as TI, DagBag
+from airflow.models import DagModel, DagRun
 from airflow.models import State as ST
-from airflow.models import DagModel, DagRun, DagStat
-from airflow.models import clear_task_instances
 from airflow.models import XCom
-from airflow.models import Connection
-from airflow.jobs import LocalTaskJob
-from airflow.operators.dummy_operator import DummyOperator
+from airflow.models import Variable
+from airflow.models import clear_task_instances
+from airflow.models.taskfail import TaskFail
+from airflow.models.taskreschedule import TaskReschedule
 from airflow.operators.bash_operator import BashOperator
+from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.python_operator import PythonOperator
 from airflow.operators.python_operator import ShortCircuitOperator
+from airflow.operators.subdag_operator import SubDagOperator
 from airflow.ti_deps.deps.trigger_rule_dep import TriggerRuleDep
 from airflow.utils import timezone
-from airflow.utils.weight_rule import WeightRule
+from airflow.utils.dag_processing import SimpleTaskInstance
+from airflow.utils.db import create_session
 from airflow.utils.state import State
 from airflow.utils.trigger_rule import TriggerRule
-from mock import patch, ANY
-from parameterized import parameterized
-from tempfile import NamedTemporaryFile
+from airflow.utils.weight_rule import WeightRule
 
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
 TEST_DAGS_FOLDER = os.path.join(
@@ -65,7 +74,7 @@
 
 class DagTest(unittest.TestCase):
 
-    def test_parms_not_passed_is_empty_dict(self):
+    def test_params_not_passed_is_empty_dict(self):
         """
         Test that when 'params' is _not_ passed to a new Dag, that the params
         attribute is set to an empty dictionary.
@@ -97,7 +106,7 @@ def test_dag_as_context_manager(self):
         """
         Test DAG as a context manager.
         When used as a context manager, Operators are automatically added to
-        the DAG (unless they specifiy a different DAG)
+        the DAG (unless they specify a different DAG)
         """
         dag = DAG(
             'dag',
@@ -139,6 +148,17 @@ def test_dag_as_context_manager(self):
         self.assertEqual(dag.dag_id, 'creating_dag_in_cm')
         self.assertEqual(dag.tasks[0].task_id, 'op6')
 
+        with dag:
+            with dag:
+                op7 = DummyOperator(task_id='op7')
+            op8 = DummyOperator(task_id='op8')
+        op9 = DummyOperator(task_id='op8')
+        op9.dag = dag2
+
+        self.assertEqual(op7.dag, dag)
+        self.assertEqual(op8.dag, dag)
+        self.assertEqual(op9.dag, dag2)
+
     def test_dag_topological_sort(self):
         dag = DAG(
             'dag',
@@ -212,7 +232,13 @@ def test_dag_topological_sort(self):
             start_date=DEFAULT_DATE,
             default_args={'owner': 'owner1'})
 
-        self.assertEquals(tuple(), dag.topological_sort())
+        self.assertEqual(tuple(), dag.topological_sort())
+
+    def test_dag_naive_default_args_start_date(self):
+        dag = DAG('DAG', default_args={'start_date': datetime.datetime(2018, 1, 1)})
+        self.assertEqual(dag.timezone, settings.TIMEZONE)
+        dag = DAG('DAG', start_date=datetime.datetime(2018, 1, 1))
+        self.assertEqual(dag.timezone, settings.TIMEZONE)
 
     def test_dag_none_default_args_start_date(self):
         """
@@ -251,7 +277,7 @@ def test_dag_task_priority_weight_total(self):
                 correct_weight = ((depth - (task_depth + 1)) * width + 1) * weight
 
                 calculated_weight = task.priority_weight_total
-                self.assertEquals(calculated_weight, correct_weight)
+                self.assertEqual(calculated_weight, correct_weight)
 
         # Same test as above except use 'upstream' for weight calculation
         weight = 3
@@ -274,10 +300,10 @@ def test_dag_task_priority_weight_total(self):
                 match = pattern.match(task.task_id)
                 task_depth = int(match.group(1))
                 # the sum of each stages after this task + itself
-                correct_weight = ((task_depth) * width + 1) * weight
+                correct_weight = (task_depth * width + 1) * weight
 
                 calculated_weight = task.priority_weight_total
-                self.assertEquals(calculated_weight, correct_weight)
+                self.assertEqual(calculated_weight, correct_weight)
 
         # Same test as above except use 'absolute' for weight calculation
         weight = 10
@@ -303,7 +329,7 @@ def test_dag_task_priority_weight_total(self):
                 correct_weight = weight
 
                 calculated_weight = task.priority_weight_total
-                self.assertEquals(calculated_weight, correct_weight)
+                self.assertEqual(calculated_weight, correct_weight)
 
         # Test if we enter an invalid weight rule
         with DAG('dag', start_date=DEFAULT_DATE,
@@ -311,7 +337,6 @@ def test_dag_task_priority_weight_total(self):
             with self.assertRaises(AirflowException):
                 DummyOperator(task_id='should_fail', weight_rule='no rule')
 
-
     def test_get_num_task_instances(self):
         test_dag_id = 'test_get_num_task_instances_dag'
         test_task_id = 'task_1'
@@ -334,20 +359,41 @@ def test_get_num_task_instances(self):
         session.merge(ti4)
         session.commit()
 
-        self.assertEqual(0, DAG.get_num_task_instances(test_dag_id, ['fakename'],
-            session=session))
-        self.assertEqual(4, DAG.get_num_task_instances(test_dag_id, [test_task_id],
-            session=session))
-        self.assertEqual(4, DAG.get_num_task_instances(test_dag_id,
-            ['fakename', test_task_id], session=session))
-        self.assertEqual(1, DAG.get_num_task_instances(test_dag_id, [test_task_id],
-            states=[None], session=session))
-        self.assertEqual(2, DAG.get_num_task_instances(test_dag_id, [test_task_id],
-            states=[State.RUNNING], session=session))
-        self.assertEqual(3, DAG.get_num_task_instances(test_dag_id, [test_task_id],
-            states=[None, State.RUNNING], session=session))
-        self.assertEqual(4, DAG.get_num_task_instances(test_dag_id, [test_task_id],
-            states=[None, State.QUEUED, State.RUNNING], session=session))
+        self.assertEqual(
+            0,
+            DAG.get_num_task_instances(test_dag_id, ['fakename'], session=session)
+        )
+        self.assertEqual(
+            4,
+            DAG.get_num_task_instances(test_dag_id, [test_task_id], session=session)
+        )
+        self.assertEqual(
+            4,
+            DAG.get_num_task_instances(
+                test_dag_id, ['fakename', test_task_id], session=session)
+        )
+        self.assertEqual(
+            1,
+            DAG.get_num_task_instances(
+                test_dag_id, [test_task_id], states=[None], session=session)
+        )
+        self.assertEqual(
+            2,
+            DAG.get_num_task_instances(
+                test_dag_id, [test_task_id], states=[State.RUNNING], session=session)
+        )
+        self.assertEqual(
+            3,
+            DAG.get_num_task_instances(
+                test_dag_id, [test_task_id],
+                states=[None, State.RUNNING], session=session)
+        )
+        self.assertEqual(
+            4,
+            DAG.get_num_task_instances(
+                test_dag_id, [test_task_id],
+                states=[None, State.QUEUED, State.RUNNING], session=session)
+        )
         session.close()
 
     def test_render_template_field(self):
@@ -362,13 +408,129 @@ def test_render_template_field(self):
         result = task.render_template('', '{{ foo }}', dict(foo='bar'))
         self.assertEqual(result, 'bar')
 
+    def test_render_template_list_field(self):
+        """Tests if render_template from a list field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        self.assertListEqual(
+            task.render_template('', ['{{ foo }}_1', '{{ foo }}_2'], {'foo': 'bar'}),
+            ['bar_1', 'bar_2']
+        )
+
+    def test_render_template_tuple_field(self):
+        """Tests if render_template from a tuple field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        # tuple is replaced by a list
+        self.assertListEqual(
+            task.render_template('', ('{{ foo }}_1', '{{ foo }}_2'), {'foo': 'bar'}),
+            ['bar_1', 'bar_2']
+        )
+
+    def test_render_template_dict_field(self):
+        """Tests if render_template from a dict field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        self.assertDictEqual(
+            task.render_template('', {'key1': '{{ foo }}_1', 'key2': '{{ foo }}_2'}, {'foo': 'bar'}),
+            {'key1': 'bar_1', 'key2': 'bar_2'}
+        )
+
+    def test_render_template_dict_field_with_templated_keys(self):
+        """Tests if render_template from a dict field works as expected:
+        dictionary keys are not templated"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        self.assertDictEqual(
+            task.render_template('', {'key_{{ foo }}_1': 1, 'key_2': '{{ foo }}_2'}, {'foo': 'bar'}),
+            {'key_{{ foo }}_1': 1, 'key_2': 'bar_2'}
+        )
+
+    def test_render_template_date_field(self):
+        """Tests if render_template from a date field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        self.assertEqual(
+            task.render_template('', datetime.date(2018, 12, 6), {'foo': 'bar'}),
+            datetime.date(2018, 12, 6)
+        )
+
+    def test_render_template_datetime_field(self):
+        """Tests if render_template from a datetime field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        self.assertEqual(
+            task.render_template('', datetime.datetime(2018, 12, 6, 10, 55), {'foo': 'bar'}),
+            datetime.datetime(2018, 12, 6, 10, 55)
+        )
+
+    def test_render_template_UUID_field(self):
+        """Tests if render_template from a UUID field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        random_uuid = uuid.uuid4()
+        self.assertIs(
+            task.render_template('', random_uuid, {'foo': 'bar'}),
+            random_uuid
+        )
+
+    def test_render_template_object_field(self):
+        """Tests if render_template from an object field works"""
+
+        dag = DAG('test-dag',
+                  start_date=DEFAULT_DATE)
+
+        with dag:
+            task = DummyOperator(task_id='op1')
+
+        test_object = object()
+        self.assertIs(
+            task.render_template('', test_object, {'foo': 'bar'}),
+            test_object
+        )
+
     def test_render_template_field_macro(self):
         """ Tests if render_template from a field works,
             if a custom filter was defined"""
 
         dag = DAG('test-dag',
                   start_date=DEFAULT_DATE,
-                  user_defined_macros = dict(foo='bar'))
+                  user_defined_macros=dict(foo='bar'))
 
         with dag:
             task = DummyOperator(task_id='op1')
@@ -407,11 +569,11 @@ def test_render_template_field_filter(self):
             if a custom filter was defined"""
 
         def jinja_udf(name):
-            return 'Hello %s' %name
+            return 'Hello %s' % name
 
         dag = DAG('test-dag',
                   start_date=DEFAULT_DATE,
-                  user_defined_filters = dict(hello=jinja_udf))
+                  user_defined_filters=dict(hello=jinja_udf))
 
         with dag:
             task = DummyOperator(task_id='op1')
@@ -419,6 +581,51 @@ def jinja_udf(name):
         result = task.render_template('', "{{ 'world' | hello}}", dict())
         self.assertEqual(result, 'Hello world')
 
+    def test_resolve_template_files_value(self):
+
+        with NamedTemporaryFile(suffix='.template') as f:
+            f.write('{{ ds }}'.encode('utf8'))
+            f.flush()
+            template_dir = os.path.dirname(f.name)
+            template_file = os.path.basename(f.name)
+
+            dag = DAG('test-dag',
+                      start_date=DEFAULT_DATE,
+                      template_searchpath=template_dir)
+
+            with dag:
+                task = DummyOperator(task_id='op1')
+
+            task.test_field = template_file
+            task.template_fields = ('test_field',)
+            task.template_ext = ('.template',)
+            task.resolve_template_files()
+
+        self.assertEqual(task.test_field, '{{ ds }}')
+
+    def test_resolve_template_files_list(self):
+
+        with NamedTemporaryFile(suffix='.template') as f:
+            f = NamedTemporaryFile(suffix='.template')
+            f.write('{{ ds }}'.encode('utf8'))
+            f.flush()
+            template_dir = os.path.dirname(f.name)
+            template_file = os.path.basename(f.name)
+
+            dag = DAG('test-dag',
+                      start_date=DEFAULT_DATE,
+                      template_searchpath=template_dir)
+
+            with dag:
+                task = DummyOperator(task_id='op1')
+
+            task.test_field = [template_file, 'some_string']
+            task.template_fields = ('test_field',)
+            task.template_ext = ('.template',)
+            task.resolve_template_files()
+
+        self.assertEqual(task.test_field, ['{{ ds }}', 'some_string'])
+
     def test_cycle(self):
         # test empty
         dag = DAG(
@@ -545,55 +752,159 @@ def test_cycle(self):
         with self.assertRaises(AirflowDagCycleException):
             dag.test_cycle()
 
+    def test_following_previous_schedule(self):
+        """
+        Make sure DST transitions are properly observed
+        """
+        local_tz = pendulum.timezone('Europe/Zurich')
+        start = local_tz.convert(datetime.datetime(2018, 10, 28, 2, 55),
+                                 dst_rule=pendulum.PRE_TRANSITION)
+        self.assertEqual(start.isoformat(), "2018-10-28T02:55:00+02:00",
+                         "Pre-condition: start date is in DST")
+
+        utc = timezone.convert_to_utc(start)
 
-class DagStatTest(unittest.TestCase):
-    def test_dagstats_crud(self):
-        DagStat.create(dag_id='test_dagstats_crud')
+        dag = DAG('tz_dag', start_date=start, schedule_interval='*/5 * * * *')
+        _next = dag.following_schedule(utc)
+        next_local = local_tz.convert(_next)
 
-        session = settings.Session()
-        qry = session.query(DagStat).filter(DagStat.dag_id == 'test_dagstats_crud')
-        self.assertEqual(len(qry.all()), len(State.dag_states))
+        self.assertEqual(_next.isoformat(), "2018-10-28T01:00:00+00:00")
+        self.assertEqual(next_local.isoformat(), "2018-10-28T02:00:00+01:00")
+
+        prev = dag.previous_schedule(utc)
+        prev_local = local_tz.convert(prev)
+
+        self.assertEqual(prev_local.isoformat(), "2018-10-28T02:50:00+02:00")
+
+        prev = dag.previous_schedule(_next)
+        prev_local = local_tz.convert(prev)
+
+        self.assertEqual(prev_local.isoformat(), "2018-10-28T02:55:00+02:00")
+        self.assertEqual(prev, utc)
+
+    def test_following_previous_schedule_daily_dag_CEST_to_CET(self):
+        """
+        Make sure DST transitions are properly observed
+        """
+        local_tz = pendulum.timezone('Europe/Zurich')
+        start = local_tz.convert(datetime.datetime(2018, 10, 27, 3),
+                                 dst_rule=pendulum.PRE_TRANSITION)
 
-        DagStat.set_dirty(dag_id='test_dagstats_crud')
-        res = qry.all()
+        utc = timezone.convert_to_utc(start)
 
-        for stat in res:
-            self.assertTrue(stat.dirty)
+        dag = DAG('tz_dag', start_date=start, schedule_interval='0 3 * * *')
 
-        # create missing
-        DagStat.set_dirty(dag_id='test_dagstats_crud_2')
-        qry2 = session.query(DagStat).filter(DagStat.dag_id == 'test_dagstats_crud_2')
-        self.assertEqual(len(qry2.all()), len(State.dag_states))
+        prev = dag.previous_schedule(utc)
+        prev_local = local_tz.convert(prev)
 
+        self.assertEqual(prev_local.isoformat(), "2018-10-26T03:00:00+02:00")
+        self.assertEqual(prev.isoformat(), "2018-10-26T01:00:00+00:00")
+
+        _next = dag.following_schedule(utc)
+        next_local = local_tz.convert(_next)
+
+        self.assertEqual(next_local.isoformat(), "2018-10-28T03:00:00+01:00")
+        self.assertEqual(_next.isoformat(), "2018-10-28T02:00:00+00:00")
+
+        prev = dag.previous_schedule(_next)
+        prev_local = local_tz.convert(prev)
+
+        self.assertEqual(prev_local.isoformat(), "2018-10-27T03:00:00+02:00")
+        self.assertEqual(prev.isoformat(), "2018-10-27T01:00:00+00:00")
+
+    def test_following_previous_schedule_daily_dag_CET_to_CEST(self):
+        """
+        Make sure DST transitions are properly observed
+        """
+        local_tz = pendulum.timezone('Europe/Zurich')
+        start = local_tz.convert(datetime.datetime(2018, 3, 25, 2),
+                                 dst_rule=pendulum.PRE_TRANSITION)
+
+        utc = timezone.convert_to_utc(start)
+
+        dag = DAG('tz_dag', start_date=start, schedule_interval='0 3 * * *')
+
+        prev = dag.previous_schedule(utc)
+        prev_local = local_tz.convert(prev)
+
+        self.assertEqual(prev_local.isoformat(), "2018-03-24T03:00:00+01:00")
+        self.assertEqual(prev.isoformat(), "2018-03-24T02:00:00+00:00")
+
+        _next = dag.following_schedule(utc)
+        next_local = local_tz.convert(_next)
+
+        self.assertEqual(next_local.isoformat(), "2018-03-25T03:00:00+02:00")
+        self.assertEqual(_next.isoformat(), "2018-03-25T01:00:00+00:00")
+
+        prev = dag.previous_schedule(_next)
+        prev_local = local_tz.convert(prev)
+
+        self.assertEqual(prev_local.isoformat(), "2018-03-24T03:00:00+01:00")
+        self.assertEqual(prev.isoformat(), "2018-03-24T02:00:00+00:00")
+
+    @patch('airflow.models.timezone.utcnow')
+    def test_sync_to_db(self, mock_now):
         dag = DAG(
-            'test_dagstats_crud',
+            'dag',
             start_date=DEFAULT_DATE,
-            default_args={'owner': 'owner1'})
-
+        )
         with dag:
-            op1 = DummyOperator(task_id='A')
-
-        now = timezone.utcnow()
-        dr = dag.create_dagrun(
-            run_id='manual__' + now.isoformat(),
-            execution_date=now,
-            start_date=now,
-            state=State.FAILED,
-            external_trigger=False,
+            DummyOperator(task_id='task', owner='owner1')
+            SubDagOperator(
+                task_id='subtask',
+                owner='owner2',
+                subdag=DAG(
+                    'dag.subtask',
+                    start_date=DEFAULT_DATE,
+                )
+            )
+        now = datetime.datetime.utcnow().replace(tzinfo=pendulum.timezone('UTC'))
+        mock_now.return_value = now
+        session = settings.Session()
+        dag.sync_to_db(session=session)
+
+        orm_dag = session.query(DagModel).filter(DagModel.dag_id == 'dag').one()
+        self.assertEqual(set(orm_dag.owners.split(', ')), {'owner1', 'owner2'})
+        self.assertEqual(orm_dag.last_scheduler_run, now)
+        self.assertTrue(orm_dag.is_active)
+        self.assertIsNone(orm_dag.default_view)
+        self.assertEqual(orm_dag.get_default_view(),
+                         configuration.conf.get('webserver', 'dag_default_view').lower())
+        self.assertEqual(orm_dag.safe_dag_id, 'dag')
+
+        orm_subdag = session.query(DagModel).filter(
+            DagModel.dag_id == 'dag.subtask').one()
+        self.assertEqual(set(orm_subdag.owners.split(', ')), {'owner1', 'owner2'})
+        self.assertEqual(orm_subdag.last_scheduler_run, now)
+        self.assertTrue(orm_subdag.is_active)
+        self.assertEqual(orm_subdag.safe_dag_id, 'dag__dot__subtask')
+
+    @patch('airflow.models.timezone.utcnow')
+    def test_sync_to_db_default_view(self, mock_now):
+        dag = DAG(
+            'dag',
+            start_date=DEFAULT_DATE,
+            default_view="graph",
         )
+        with dag:
+            DummyOperator(task_id='task', owner='owner1')
+            SubDagOperator(
+                task_id='subtask',
+                owner='owner2',
+                subdag=DAG(
+                    'dag.subtask',
+                    start_date=DEFAULT_DATE,
+                )
+            )
+        now = datetime.datetime.utcnow().replace(tzinfo=pendulum.timezone('UTC'))
+        mock_now.return_value = now
+        session = settings.Session()
+        dag.sync_to_db(session=session)
 
-        DagStat.update(dag_ids=['test_dagstats_crud'])
-        res = qry.all()
-        for stat in res:
-            if stat.state == State.FAILED:
-                self.assertEqual(stat.count, 1)
-            else:
-                self.assertEqual(stat.count, 0)
+        orm_dag = session.query(DagModel).filter(DagModel.dag_id == 'dag').one()
+        self.assertIsNotNone(orm_dag.default_view)
+        self.assertEqual(orm_dag.get_default_view(), "graph")
 
-        DagStat.update()
-        res = qry2.all()
-        for stat in res:
-            self.assertFalse(stat.dirty)
 
 class DagRunTest(unittest.TestCase):
 
@@ -647,7 +958,7 @@ def test_clear_task_instances_for_backfill_dagrun(self):
             DagRun.dag_id == dag_id,
             DagRun.execution_date == now
         ).first()
-        self.assertEquals(dr0.state, State.RUNNING)
+        self.assertEqual(dr0.state, State.RUNNING)
 
     def test_id_for_date(self):
         run_id = models.DagRun.id_for_date(
@@ -684,10 +995,14 @@ def test_dagrun_find(self):
 
         session.commit()
 
-        self.assertEqual(1, len(models.DagRun.find(dag_id=dag_id1, external_trigger=True)))
-        self.assertEqual(0, len(models.DagRun.find(dag_id=dag_id1, external_trigger=False)))
-        self.assertEqual(0, len(models.DagRun.find(dag_id=dag_id2, external_trigger=True)))
-        self.assertEqual(1, len(models.DagRun.find(dag_id=dag_id2, external_trigger=False)))
+        self.assertEqual(1,
+                         len(models.DagRun.find(dag_id=dag_id1, external_trigger=True)))
+        self.assertEqual(0,
+                         len(models.DagRun.find(dag_id=dag_id1, external_trigger=False)))
+        self.assertEqual(0,
+                         len(models.DagRun.find(dag_id=dag_id2, external_trigger=True)))
+        self.assertEqual(1,
+                         len(models.DagRun.find(dag_id=dag_id2, external_trigger=False)))
 
     def test_dagrun_success_when_all_skipped(self):
         """
@@ -801,13 +1116,32 @@ def test_dagrun_deadlock(self):
         dr.update_state()
         self.assertEqual(dr.state, State.FAILED)
 
-    def test_dagrun_no_deadlock(self):
+    def test_dagrun_no_deadlock_with_shutdown(self):
+        session = settings.Session()
+        dag = DAG('test_dagrun_no_deadlock_with_shutdown',
+                  start_date=DEFAULT_DATE)
+        with dag:
+            op1 = DummyOperator(task_id='upstream_task')
+            op2 = DummyOperator(task_id='downstream_task')
+            op2.set_upstream(op1)
+
+        dr = dag.create_dagrun(run_id='test_dagrun_no_deadlock_with_shutdown',
+                               state=State.RUNNING,
+                               execution_date=DEFAULT_DATE,
+                               start_date=DEFAULT_DATE)
+        upstream_ti = dr.get_task_instance(task_id='upstream_task')
+        upstream_ti.set_state(State.SHUTDOWN, session=session)
+
+        dr.update_state()
+        self.assertEqual(dr.state, State.RUNNING)
+
+    def test_dagrun_no_deadlock_with_depends_on_past(self):
         session = settings.Session()
         dag = DAG('test_dagrun_no_deadlock',
                   start_date=DEFAULT_DATE)
         with dag:
-            op1 = DummyOperator(task_id='dop', depends_on_past=True)
-            op2 = DummyOperator(task_id='tc', task_concurrency=1)
+            DummyOperator(task_id='dop', depends_on_past=True)
+            DummyOperator(task_id='tc', task_concurrency=1)
 
         dag.clear()
         dr = dag.create_dagrun(run_id='test_dagrun_no_deadlock_1',
@@ -819,9 +1153,9 @@ def test_dagrun_no_deadlock(self):
                                 execution_date=DEFAULT_DATE + datetime.timedelta(days=1),
                                 start_date=DEFAULT_DATE + datetime.timedelta(days=1))
         ti1_op1 = dr.get_task_instance(task_id='dop')
-        ti2_op1 = dr2.get_task_instance(task_id='dop')
+        dr2.get_task_instance(task_id='dop')
         ti2_op1 = dr.get_task_instance(task_id='tc')
-        ti2_op2 = dr.get_task_instance(task_id='tc')
+        dr.get_task_instance(task_id='tc')
         ti1_op1.set_state(state=State.RUNNING, session=session)
         dr.update_state()
         dr2.update_state()
@@ -896,6 +1230,124 @@ def on_failure_callable(context):
         updated_dag_state = dag_run.update_state()
         self.assertEqual(State.FAILED, updated_dag_state)
 
+    def test_dagrun_set_state_end_date(self):
+        session = settings.Session()
+
+        dag = DAG(
+            'test_dagrun_set_state_end_date',
+            start_date=DEFAULT_DATE,
+            default_args={'owner': 'owner1'})
+
+        dag.clear()
+
+        now = timezone.utcnow()
+        dr = dag.create_dagrun(run_id='test_dagrun_set_state_end_date',
+                               state=State.RUNNING,
+                               execution_date=now,
+                               start_date=now)
+
+        # Initial end_date should be NULL
+        # State.SUCCESS and State.FAILED are all ending state and should set end_date
+        # State.RUNNING set end_date back to NULL
+        session.add(dr)
+        session.commit()
+        self.assertIsNone(dr.end_date)
+
+        dr.set_state(State.SUCCESS)
+        session.merge(dr)
+        session.commit()
+
+        dr_database = session.query(DagRun).filter(
+            DagRun.run_id == 'test_dagrun_set_state_end_date'
+        ).one()
+        self.assertIsNotNone(dr_database.end_date)
+        self.assertEqual(dr.end_date, dr_database.end_date)
+
+        dr.set_state(State.RUNNING)
+        session.merge(dr)
+        session.commit()
+
+        dr_database = session.query(DagRun).filter(
+            DagRun.run_id == 'test_dagrun_set_state_end_date'
+        ).one()
+
+        self.assertIsNone(dr_database.end_date)
+
+        dr.set_state(State.FAILED)
+        session.merge(dr)
+        session.commit()
+        dr_database = session.query(DagRun).filter(
+            DagRun.run_id == 'test_dagrun_set_state_end_date'
+        ).one()
+
+        self.assertIsNotNone(dr_database.end_date)
+        self.assertEqual(dr.end_date, dr_database.end_date)
+
+    def test_dagrun_update_state_end_date(self):
+        session = settings.Session()
+
+        dag = DAG(
+            'test_dagrun_update_state_end_date',
+            start_date=DEFAULT_DATE,
+            default_args={'owner': 'owner1'})
+
+        # A -> B
+        with dag:
+            op1 = DummyOperator(task_id='A')
+            op2 = DummyOperator(task_id='B')
+            op1.set_upstream(op2)
+
+        dag.clear()
+
+        now = timezone.utcnow()
+        dr = dag.create_dagrun(run_id='test_dagrun_update_state_end_date',
+                               state=State.RUNNING,
+                               execution_date=now,
+                               start_date=now)
+
+        # Initial end_date should be NULL
+        # State.SUCCESS and State.FAILED are all ending state and should set end_date
+        # State.RUNNING set end_date back to NULL
+        session.merge(dr)
+        session.commit()
+        self.assertIsNone(dr.end_date)
+
+        ti_op1 = dr.get_task_instance(task_id=op1.task_id)
+        ti_op1.set_state(state=State.SUCCESS, session=session)
+        ti_op2 = dr.get_task_instance(task_id=op2.task_id)
+        ti_op2.set_state(state=State.SUCCESS, session=session)
+
+        dr.update_state()
+
+        dr_database = session.query(DagRun).filter(
+            DagRun.run_id == 'test_dagrun_update_state_end_date'
+        ).one()
+        self.assertIsNotNone(dr_database.end_date)
+        self.assertEqual(dr.end_date, dr_database.end_date)
+
+        ti_op1.set_state(state=State.RUNNING, session=session)
+        ti_op2.set_state(state=State.RUNNING, session=session)
+        dr.update_state()
+
+        dr_database = session.query(DagRun).filter(
+            DagRun.run_id == 'test_dagrun_update_state_end_date'
+        ).one()
+
+        self.assertEqual(dr._state, State.RUNNING)
+        self.assertIsNone(dr.end_date)
+        self.assertIsNone(dr_database.end_date)
+
+        ti_op1.set_state(state=State.FAILED, session=session)
+        ti_op2.set_state(state=State.FAILED, session=session)
+        dr.update_state()
+
+        dr_database = session.query(DagRun).filter(
+            DagRun.run_id == 'test_dagrun_update_state_end_date'
+        ).one()
+
+        self.assertIsNotNone(dr_database.end_date)
+        self.assertEqual(dr.end_date, dr_database.end_date)
+
     def test_get_task_instance_on_empty_dagrun(self):
         """
         Make sure that a proper value is returned when a dagrun has no task instances
@@ -904,7 +1356,7 @@ def test_get_task_instance_on_empty_dagrun(self):
             dag_id='test_get_task_instance_on_empty_dagrun',
             start_date=timezone.datetime(2017, 1, 1)
         )
-        dag_task1 = ShortCircuitOperator(
+        ShortCircuitOperator(
             task_id='test_short_circuit_false',
             dag=dag,
             python_callable=lambda: False)
@@ -934,10 +1386,8 @@ def test_get_latest_runs(self):
         dag = DAG(
             dag_id='test_latest_runs_1',
             start_date=DEFAULT_DATE)
-        dag_1_run_1 = self.create_dag_run(dag,
-                execution_date=timezone.datetime(2015, 1, 1))
-        dag_1_run_2 = self.create_dag_run(dag,
-                execution_date=timezone.datetime(2015, 1, 2))
+        self.create_dag_run(dag, execution_date=timezone.datetime(2015, 1, 1))
+        self.create_dag_run(dag, execution_date=timezone.datetime(2015, 1, 2))
         dagruns = models.DagRun.get_latest_runs(session)
         session.close()
         for dagrun in dagruns:
@@ -946,11 +1396,20 @@ def test_get_latest_runs(self):
 
     def test_is_backfill(self):
         dag = DAG(dag_id='test_is_backfill', start_date=DEFAULT_DATE)
+
         dagrun = self.create_dag_run(dag, execution_date=DEFAULT_DATE)
         dagrun.run_id = BackfillJob.ID_PREFIX + '_sfddsffds'
-        dagrun2 = self.create_dag_run(dag, execution_date=DEFAULT_DATE + datetime.timedelta(days=1))
+
+        dagrun2 = self.create_dag_run(
+            dag, execution_date=DEFAULT_DATE + datetime.timedelta(days=1))
+
+        dagrun3 = self.create_dag_run(
+            dag, execution_date=DEFAULT_DATE + datetime.timedelta(days=2))
+        dagrun3.run_id = None
+
         self.assertTrue(dagrun.is_backfill)
         self.assertFalse(dagrun2.is_backfill)
+        self.assertFalse(dagrun3.is_backfill)
 
     def test_removed_task_instances_can_be_restored(self):
         def with_all_tasks_removed(dag):
@@ -961,28 +1420,36 @@ def with_all_tasks_removed(dag):
 
         dagrun = self.create_dag_run(dag)
         flaky_ti = dagrun.get_task_instances()[0]
-        self.assertEquals('flaky_task', flaky_ti.task_id)
-        self.assertEquals(State.NONE, flaky_ti.state)
+        self.assertEqual('flaky_task', flaky_ti.task_id)
+        self.assertEqual(State.NONE, flaky_ti.state)
 
         dagrun.dag = with_all_tasks_removed(dag)
 
         dagrun.verify_integrity()
         flaky_ti.refresh_from_db()
-        self.assertEquals(State.REMOVED, flaky_ti.state)
+        self.assertEqual(State.NONE, flaky_ti.state)
 
         dagrun.dag.add_task(DummyOperator(task_id='flaky_task', owner='test'))
 
         dagrun.verify_integrity()
         flaky_ti.refresh_from_db()
-        self.assertEquals(State.NONE, flaky_ti.state)
+        self.assertEqual(State.NONE, flaky_ti.state)
+
 
 class DagBagTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.empty_dir = mkdtemp()
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.empty_dir)
 
     def test_get_existing_dag(self):
         """
         test that were're able to parse some example DAGs and retrieve them
         """
-        dagbag = models.DagBag(include_examples=True)
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=True)
 
         some_expected_dag_ids = ["example_bash_operator",
                                  "example_branch_operator"]
@@ -999,11 +1466,54 @@ def test_get_non_existing_dag(self):
         """
         test that retrieving a non existing dag id returns None without crashing
         """
-        dagbag = models.DagBag(include_examples=True)
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
 
         non_existing_dag_id = "non_existing_dag_id"
         self.assertIsNone(dagbag.get_dag(non_existing_dag_id))
 
+    def test_dont_load_example(self):
+        """
+        test that the example are not loaded
+        """
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
+
+        self.assertEqual(dagbag.size(), 0)
+
+    def test_safe_mode_heuristic_match(self):
+        """With safe mode enabled, a file matching the discovery heuristics
+        should be discovered.
+        """
+        with NamedTemporaryFile(dir=self.empty_dir, suffix=".py") as fp:
+            fp.write("# airflow".encode())
+            fp.write("# DAG".encode())
+            fp.flush()
+            dagbag = models.DagBag(
+                dag_folder=self.empty_dir, include_examples=False, safe_mode=True)
+            self.assertEqual(len(dagbag.dagbag_stats), 1)
+            self.assertEqual(
+                dagbag.dagbag_stats[0].file,
+                "/{}".format(os.path.basename(fp.name)))
+
+    def test_safe_mode_heuristic_mismatch(self):
+        """With safe mode enabled, a file not matching the discovery heuristics
+        should not be discovered.
+        """
+        with NamedTemporaryFile(dir=self.empty_dir, suffix=".py"):
+            dagbag = models.DagBag(
+                dag_folder=self.empty_dir, include_examples=False, safe_mode=True)
+            self.assertEqual(len(dagbag.dagbag_stats), 0)
+
+    def test_safe_mode_disabled(self):
+        """With safe mode disabled, an empty python file should be discovered.
+        """
+        with NamedTemporaryFile(dir=self.empty_dir, suffix=".py") as fp:
+            dagbag = models.DagBag(
+                dag_folder=self.empty_dir, include_examples=False, safe_mode=False)
+            self.assertEqual(len(dagbag.dagbag_stats), 1)
+            self.assertEqual(
+                dagbag.dagbag_stats[0].file,
+                "/{}".format(os.path.basename(fp.name)))
+
     def test_process_file_that_contains_multi_bytes_char(self):
         """
         test that we're able to parse file that contains multi-byte char
@@ -1012,7 +1522,7 @@ def test_process_file_that_contains_multi_bytes_char(self):
         f.write('\u3042'.encode('utf8'))  # write multi-byte char (hiragana)
         f.flush()
 
-        dagbag = models.DagBag(include_examples=True)
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
         self.assertEqual([], dagbag.process_file(f.name))
 
     def test_zip_skip_log(self):
@@ -1034,11 +1544,24 @@ def test_zip(self):
         """
         test the loading of a DAG within a zip file that includes dependencies
         """
-        dagbag = models.DagBag()
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
         dagbag.process_file(os.path.join(TEST_DAGS_FOLDER, "test_zip.zip"))
         self.assertTrue(dagbag.get_dag("test_zip_dag"))
 
-    @patch.object(DagModel,'get_current')
+    def test_process_file_cron_validity_check(self):
+        """
+        test if an invalid cron expression
+        as schedule interval can be identified
+        """
+        invalid_dag_files = ["test_invalid_cron.py", "test_zip_invalid_cron.zip"]
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
+
+        self.assertEqual(len(dagbag.import_errors), 0)
+        for d in invalid_dag_files:
+            dagbag.process_file(os.path.join(TEST_DAGS_FOLDER, d))
+        self.assertEqual(len(dagbag.import_errors), len(invalid_dag_files))
+
+    @patch.object(DagModel, 'get_current')
     def test_get_dag_without_refresh(self, mock_dagmodel):
         """
         Test that, once a DAG is loaded, it doesn't get refreshed again if it
@@ -1052,15 +1575,16 @@ def test_get_dag_without_refresh(self, mock_dagmodel):
 
         class TestDagBag(models.DagBag):
             process_file_calls = 0
+
             def process_file(self, filepath, only_if_updated=True, safe_mode=True):
                 if 'example_bash_operator.py' == os.path.basename(filepath):
                     TestDagBag.process_file_calls += 1
                 super(TestDagBag, self).process_file(filepath, only_if_updated, safe_mode)
 
         dagbag = TestDagBag(include_examples=True)
-        processed_files = dagbag.process_file_calls
+        dagbag.process_file_calls
 
-        # Should not call process_file agani, since it's already loaded during init.
+        # Should not call process_file again, since it's already loaded during init.
         self.assertEqual(1, dagbag.process_file_calls)
         self.assertIsNotNone(dagbag.get_dag(dag_id))
         self.assertEqual(1, dagbag.process_file_calls)
@@ -1094,9 +1618,9 @@ def process_dag(self, create_dag):
         f.write(source.encode('utf8'))
         f.flush()
 
-        dagbag = models.DagBag(include_examples=False)
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
         found_dags = dagbag.process_file(f.name)
-        return (dagbag, found_dags, f.name)
+        return dagbag, found_dags, f.name
 
     def validate_dags(self, expected_parent_dag, actual_found_dags, actual_dagbag,
                       should_be_found=True):
@@ -1107,12 +1631,12 @@ def validate_dags(self, expected_parent_dag, actual_found_dags, actual_dagbag,
 
         for dag_id in expected_dag_ids:
             actual_dagbag.log.info('validating %s' % dag_id)
-            self.assertEquals(
+            self.assertEqual(
                 dag_id in actual_found_dag_ids, should_be_found,
                 'dag "%s" should %shave been found after processing dag "%s"' %
                 (dag_id, '' if should_be_found else 'not ', expected_parent_dag.dag_id)
             )
-            self.assertEquals(
+            self.assertEqual(
                 dag_id in actual_dagbag.dags, should_be_found,
                 'dag "%s" should %sbe in dagbag.dags after processing dag "%s"' %
                 (dag_id, '' if should_be_found else 'not ', expected_parent_dag.dag_id)
@@ -1271,6 +1795,7 @@ def test_skip_cycle_dags(self):
         Don't crash when loading an invalid (contains a cycle) DAG file.
         Don't load the dag into the DagBag either
         """
+
         # Define Dag to load
         def basic_cycle():
             from airflow.models import DAG
@@ -1404,44 +1929,72 @@ def test_process_file_with_none(self):
         """
         test that process_file can handle Nones
         """
-        dagbag = models.DagBag(include_examples=True)
+        dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
 
         self.assertEqual([], dagbag.process_file(None))
 
     @patch.object(TI, 'handle_failure')
-    def test_kill_zombies(self, mock_ti):
+    def test_kill_zombies(self, mock_ti_handle_failure):
         """
         Test that kill zombies call TIs failure handler with proper context
         """
         dagbag = models.DagBag()
-        session = settings.Session
-        dag = dagbag.get_dag('example_branch_operator')
-        task = dag.get_task(task_id='run_this_first')
+        with create_session() as session:
+            session.query(TI).delete()
+            dag = dagbag.get_dag('example_branch_operator')
+            task = dag.get_task(task_id='run_this_first')
 
-        ti = TI(task, datetime.datetime.now() - datetime.timedelta(1), 'running')
-        lj = LocalTaskJob(ti)
-        lj.state = State.SHUTDOWN
+            ti = TI(task, DEFAULT_DATE, State.RUNNING)
 
-        session.add(lj)
-        session.commit()
+            session.add(ti)
+            session.commit()
 
-        ti.job_id = lj.id
+            zombies = [SimpleTaskInstance(ti)]
+            dagbag.kill_zombies(zombies)
+            mock_ti_handle_failure \
+                .assert_called_with(ANY,
+                                    configuration.getboolean('core',
+                                                             'unit_test_mode'),
+                                    ANY)
 
-        session.add(ti)
-        session.commit()
+    def test_deactivate_unknown_dags(self):
+        """
+        Test that dag_ids not passed into deactivate_unknown_dags
+        are deactivated when function is invoked
+        """
+        dagbag = DagBag(include_examples=True)
+        dag_id = "test_deactivate_unknown_dags"
+        expected_active_dags = dagbag.dags.keys()
+
+        model_before = DagModel(dag_id=dag_id, is_active=True)
+        with create_session() as session:
+            session.merge(model_before)
+
+        models.DAG.deactivate_unknown_dags(expected_active_dags)
+
+        after_model = DagModel.get_dagmodel(dag_id)
+        self.assertTrue(model_before.is_active)
+        self.assertFalse(after_model.is_active)
+
+        # clean up
+        with create_session() as session:
+            session.query(DagModel).filter(DagModel.dag_id == 'test_deactivate_unknown_dags').delete()
 
-        dagbag.kill_zombies()
-        mock_ti.assert_called_with(ANY,
-                                   configuration.getboolean('core', 'unit_test_mode'),
-                                   ANY)
 
 class TaskInstanceTest(unittest.TestCase):
 
+    def tearDown(self):
+        with create_session() as session:
+            session.query(TaskFail).delete()
+            session.query(TaskReschedule).delete()
+            session.query(models.TaskInstance).delete()
+
     def test_set_task_dates(self):
         """
         Test that tasks properly take start/end dates from DAGs
         """
-        dag = DAG('dag', start_date=DEFAULT_DATE, end_date=DEFAULT_DATE + datetime.timedelta(days=10))
+        dag = DAG('dag', start_date=DEFAULT_DATE,
+                  end_date=DEFAULT_DATE + datetime.timedelta(days=10))
 
         op1 = DummyOperator(task_id='op_1', owner='test')
 
@@ -1482,7 +2035,7 @@ def test_timezone_awareness(self):
         op_no_dag = DummyOperator(task_id='op_no_dag')
         ti = TI(task=op_no_dag, execution_date=NAIVE_DATETIME)
 
-        self.assertEquals(ti.execution_date, DEFAULT_DATE)
+        self.assertEqual(ti.execution_date, DEFAULT_DATE)
 
         # check with dag without localized execution_date
         dag = DAG('dag', start_date=DEFAULT_DATE)
@@ -1490,14 +2043,24 @@ def test_timezone_awareness(self):
         dag.add_task(op1)
         ti = TI(task=op1, execution_date=NAIVE_DATETIME)
 
-        self.assertEquals(ti.execution_date, DEFAULT_DATE)
+        self.assertEqual(ti.execution_date, DEFAULT_DATE)
 
         # with dag and localized execution_date
         tz = pendulum.timezone("Europe/Amsterdam")
         execution_date = timezone.datetime(2016, 1, 1, 1, 0, 0, tzinfo=tz)
         utc_date = timezone.convert_to_utc(execution_date)
         ti = TI(task=op1, execution_date=execution_date)
-        self.assertEquals(ti.execution_date, utc_date)
+        self.assertEqual(ti.execution_date, utc_date)
+
+    def test_task_naive_datetime(self):
+        NAIVE_DATETIME = DEFAULT_DATE.replace(tzinfo=None)
+
+        op_no_dag = DummyOperator(task_id='test_task_naive_datetime',
+                                  start_date=NAIVE_DATETIME,
+                                  end_date=NAIVE_DATETIME)
+
+        self.assertTrue(op_no_dag.start_date.tzinfo)
+        self.assertTrue(op_no_dag.end_date.tzinfo)
 
     def test_set_dag(self):
         """
@@ -1597,7 +2160,6 @@ def test_requeue_over_concurrency(self, mock_concurrency_reached):
         ti.run()
         self.assertEqual(ti.state, models.State.NONE)
 
-
     @patch.object(TI, 'pool_full')
     def test_run_pooling_task(self, mock_pool_full):
         """
@@ -1791,11 +2353,107 @@ def test_next_retry_datetime(self):
 
         ti.try_number = 9
         dt = ti.next_retry_datetime()
-        self.assertEqual(dt, ti.end_date+max_delay)
+        self.assertEqual(dt, ti.end_date + max_delay)
 
         ti.try_number = 50
         dt = ti.next_retry_datetime()
-        self.assertEqual(dt, ti.end_date+max_delay)
+        self.assertEqual(dt, ti.end_date + max_delay)
+
+    @patch.object(TI, 'pool_full')
+    def test_reschedule_handling(self, mock_pool_full):
+        """
+        Test that task reschedules are handled properly
+        """
+        # Mock the pool with a pool with slots open since the pool doesn't actually exist
+        mock_pool_full.return_value = False
+
+        # Return values of the python sensor callable, modified during tests
+        done = False
+        fail = False
+
+        def callable():
+            if fail:
+                raise AirflowException()
+            return done
+
+        dag = models.DAG(dag_id='test_reschedule_handling')
+        task = PythonSensor(
+            task_id='test_reschedule_handling_sensor',
+            poke_interval=0,
+            mode='reschedule',
+            python_callable=callable,
+            retries=1,
+            retry_delay=datetime.timedelta(seconds=0),
+            dag=dag,
+            owner='airflow',
+            start_date=timezone.datetime(2016, 2, 1, 0, 0, 0))
+
+        ti = TI(task=task, execution_date=timezone.utcnow())
+        self.assertEqual(ti._try_number, 0)
+        self.assertEqual(ti.try_number, 1)
+
+        def run_ti_and_assert(run_date, expected_start_date, expected_end_date, expected_duration,
+                              expected_state, expected_try_number, expected_task_reschedule_count):
+            with freeze_time(run_date):
+                try:
+                    ti.run()
+                except AirflowException:
+                    if not fail:
+                        raise
+            ti.refresh_from_db()
+            self.assertEqual(ti.state, expected_state)
+            self.assertEqual(ti._try_number, expected_try_number)
+            self.assertEqual(ti.try_number, expected_try_number + 1)
+            self.assertEqual(ti.start_date, expected_start_date)
+            self.assertEqual(ti.end_date, expected_end_date)
+            self.assertEqual(ti.duration, expected_duration)
+            trs = TaskReschedule.find_for_task_instance(ti)
+            self.assertEqual(len(trs), expected_task_reschedule_count)
+
+        date1 = timezone.utcnow()
+        date2 = date1 + datetime.timedelta(minutes=1)
+        date3 = date2 + datetime.timedelta(minutes=1)
+        date4 = date3 + datetime.timedelta(minutes=1)
+
+        # Run with multiple reschedules.
+        # During reschedule the try number remains the same, but each reschedule is recorded.
+        # The start date is expected to remain the inital date, hence the duration increases.
+        # When finished the try number is incremented and there is no reschedule expected
+        # for this try.
+
+        done, fail = False, False
+        run_ti_and_assert(date1, date1, date1, 0, State.UP_FOR_RESCHEDULE, 0, 1)
+
+        done, fail = False, False
+        run_ti_and_assert(date2, date1, date2, 60, State.UP_FOR_RESCHEDULE, 0, 2)
+
+        done, fail = False, False
+        run_ti_and_assert(date3, date1, date3, 120, State.UP_FOR_RESCHEDULE, 0, 3)
+
+        done, fail = True, False
+        run_ti_and_assert(date4, date1, date4, 180, State.SUCCESS, 1, 0)
+
+        # Clear the task instance.
+        dag.clear()
+        ti.refresh_from_db()
+        self.assertEqual(ti.state, State.NONE)
+        self.assertEqual(ti._try_number, 1)
+
+        # Run again after clearing with reschedules and a retry.
+        # The retry increments the try number, and for that try no reschedule is expected.
+        # After the retry the start date is reset, hence the duration is also reset.
+
+        done, fail = False, False
+        run_ti_and_assert(date1, date1, date1, 0, State.UP_FOR_RESCHEDULE, 1, 1)
+
+        done, fail = False, True
+        run_ti_and_assert(date2, date1, date2, 60, State.UP_FOR_RETRY, 2, 0)
+
+        done, fail = False, False
+        run_ti_and_assert(date3, date3, date3, 0, State.UP_FOR_RESCHEDULE, 2, 1)
+
+        done, fail = True, False
+        run_ti_and_assert(date4, date3, date4, 60, State.SUCCESS, 3, 0)
 
     def test_depends_on_past(self):
         dagbag = models.DagBag()
@@ -2037,7 +2695,7 @@ def test_check_and_change_state_before_execution(self):
     def test_check_and_change_state_before_execution_dep_not_met(self):
         dag = models.DAG(dag_id='test_check_and_change_state_before_execution')
         task = DummyOperator(task_id='task', dag=dag, start_date=DEFAULT_DATE)
-        task2= DummyOperator(task_id='task2', dag=dag, start_date=DEFAULT_DATE)
+        task2 = DummyOperator(task_id='task2', dag=dag, start_date=DEFAULT_DATE)
         task >> task2
         ti = TI(
             task=task2, execution_date=timezone.utcnow())
@@ -2076,9 +2734,9 @@ def test_get_num_running_task_instances(self):
         session.add(ti3)
         session.commit()
 
-        self.assertEquals(1, ti1.get_num_running_task_instances(session=session))
-        self.assertEquals(1, ti2.get_num_running_task_instances(session=session))
-        self.assertEquals(1, ti3.get_num_running_task_instances(session=session))
+        self.assertEqual(1, ti1.get_num_running_task_instances(session=session))
+        self.assertEqual(1, ti2.get_num_running_task_instances(session=session))
+        self.assertEqual(1, ti3.get_num_running_task_instances(session=session))
 
     def test_log_url(self):
         now = pendulum.now('Europe/Brussels')
@@ -2092,6 +2750,20 @@ def test_log_url(self):
         self.assertEqual(d['task_id'][0], 'op')
         self.assertEqual(pendulum.parse(d['execution_date'][0]), now)
 
+    @patch('airflow.settings.RBAC', True)
+    def test_log_url_rbac(self):
+        dag = DAG('dag', start_date=DEFAULT_DATE)
+        task = DummyOperator(task_id='op', dag=dag)
+        ti = TI(task=task, execution_date=datetime.datetime(2018, 1, 1))
+
+        expected_url = (
+            'http://localhost:8080/log?'
+            'execution_date=2018-01-01T00%3A00%3A00%2B00%3A00'
+            '&task_id=op'
+            '&dag_id=dag'
+        )
+        self.assertEqual(ti.log_url, expected_url)
+
     def test_mark_success_url(self):
         now = pendulum.now('Europe/Brussels')
         dag = DAG('dag', start_date=DEFAULT_DATE)
@@ -2134,6 +2806,108 @@ def test_overwrite_params_with_dag_run_conf_none(self):
 
         self.assertEqual(False, params["override"])
 
+    def test_success_callbak_no_race_condition(self):
+        class CallbackWrapper(object):
+            def wrap_task_instance(self, ti):
+                self.task_id = ti.task_id
+                self.dag_id = ti.dag_id
+                self.execution_date = ti.execution_date
+                self.task_state_in_callback = ""
+                self.callback_ran = False
+
+            def success_handler(self, context):
+                self.callback_ran = True
+                session = settings.Session()
+                temp_instance = session.query(TI).filter(
+                    TI.task_id == self.task_id).filter(
+                    TI.dag_id == self.dag_id).filter(
+                    TI.execution_date == self.execution_date).one()
+                self.task_state_in_callback = temp_instance.state
+        cw = CallbackWrapper()
+        dag = DAG('test_success_callbak_no_race_condition', start_date=DEFAULT_DATE,
+                  end_date=DEFAULT_DATE + datetime.timedelta(days=10))
+        task = DummyOperator(task_id='op', email='test@test.test',
+                             on_success_callback=cw.success_handler, dag=dag)
+        ti = TI(task=task, execution_date=datetime.datetime.now())
+        ti.state = State.RUNNING
+        session = settings.Session()
+        session.merge(ti)
+        session.commit()
+        cw.wrap_task_instance(ti)
+        ti._run_raw_task()
+        self.assertTrue(cw.callback_ran)
+        self.assertEqual(cw.task_state_in_callback, State.RUNNING)
+        ti.refresh_from_db()
+        self.assertEqual(ti.state, State.SUCCESS)
+
+    @patch('airflow.models.send_email')
+    def test_email_alert(self, mock_send_email):
+        dag = models.DAG(dag_id='test_failure_email')
+        task = BashOperator(
+            task_id='test_email_alert',
+            dag=dag,
+            bash_command='exit 1',
+            start_date=DEFAULT_DATE,
+            email='to')
+
+        ti = TI(task=task, execution_date=datetime.datetime.now())
+
+        try:
+            ti.run()
+        except AirflowException:
+            pass
+
+        (email, title, body), _ = mock_send_email.call_args
+        self.assertEqual(email, 'to')
+        self.assertIn('test_email_alert', title)
+        self.assertIn('test_email_alert', body)
+        self.assertIn('Try 1', body)
+
+    @patch('airflow.models.send_email')
+    def test_email_alert_with_config(self, mock_send_email):
+        dag = models.DAG(dag_id='test_failure_email')
+        task = BashOperator(
+            task_id='test_email_alert_with_config',
+            dag=dag,
+            bash_command='exit 1',
+            start_date=DEFAULT_DATE,
+            email='to')
+
+        ti = TI(
+            task=task, execution_date=datetime.datetime.now())
+
+        configuration.set('email', 'SUBJECT_TEMPLATE', '/subject/path')
+        configuration.set('email', 'HTML_CONTENT_TEMPLATE', '/html_content/path')
+
+        opener = mock_open(read_data='template: {{ti.task_id}}')
+        with patch('airflow.models.open', opener, create=True):
+            try:
+                ti.run()
+            except AirflowException:
+                pass
+
+        (email, title, body), _ = mock_send_email.call_args
+        self.assertEqual(email, 'to')
+        self.assertEqual('template: test_email_alert_with_config', title)
+        self.assertEqual('template: test_email_alert_with_config', body)
+
+    def test_set_duration(self):
+        task = DummyOperator(task_id='op', email='test@test.test')
+        ti = TI(
+            task=task,
+            execution_date=datetime.datetime.now(),
+        )
+        ti.start_date = datetime.datetime(2018, 10, 1, 1)
+        ti.end_date = datetime.datetime(2018, 10, 1, 2)
+        ti.set_duration()
+        self.assertEqual(ti.duration, 3600)
+
+    def test_set_duration_empty_dates(self):
+        task = DummyOperator(task_id='op', email='test@test.test')
+        ti = TI(task=task, execution_date=datetime.datetime.now())
+        ti.set_duration()
+        self.assertIsNone(ti.duration)
+
 
 class ClearTasksTest(unittest.TestCase):
 
@@ -2256,7 +3030,8 @@ def test_dags_clear(self):
         for i in range(num_of_dags):
             dag = DAG('test_dag_clear_' + str(i), start_date=DEFAULT_DATE,
                       end_date=DEFAULT_DATE + datetime.timedelta(days=10))
-            ti = TI(task=DummyOperator(task_id='test_task_clear_' + str(i), owner='test', dag=dag),
+            ti = TI(task=DummyOperator(task_id='test_task_clear_' + str(i), owner='test',
+                                       dag=dag),
                     execution_date=DEFAULT_DATE)
             dags.append(dag)
             tis.append(ti)
@@ -2402,7 +3177,7 @@ def test_xcom_enable_pickle_type(self):
     def test_xcom_disable_pickle_type_fail_on_non_json(self):
         class PickleRce(object):
             def __reduce__(self):
-                return (os.system, ("ls -alt",))
+                return os.system, ("ls -alt",)
 
         configuration.set("core", "xcom_enable_pickling", "False")
 
@@ -2442,49 +3217,61 @@ def test_xcom_get_many(self):
         for result in results:
             self.assertEqual(result.value, json_obj)
 
-class ConnectionTest(unittest.TestCase):
-    @patch.object(configuration, 'get')
-    def test_connection_extra_no_encryption(self, mock_get):
-        """
-        Tests extras on a new connection without encryption. The fernet key
-        is set to a non-base64-encoded string and the extra is stored without
-        encryption.
-        """
-        mock_get.return_value = 'cryptography_not_found_storing_passwords_in_plain_text'
-        test_connection = Connection(extra='testextra')
-        self.assertEqual(test_connection.extra, 'testextra')
-
-    @patch.object(configuration, 'get')
-    def test_connection_extra_with_encryption(self, mock_get):
-        """
-        Tests extras on a new connection with encryption. The fernet key
-        is set to a base64 encoded string and the extra is encrypted.
-        """
-        # 'dGVzdA==' is base64 encoded 'test'
-        mock_get.return_value = 'dGVzdA=='
-        test_connection = Connection(extra='testextra')
-        self.assertEqual(test_connection.extra, 'testextra')
-
-    def test_connection_from_uri_without_extras(self):
-        uri = 'scheme://user:password@host%2flocation:1234/schema'
-        connection = Connection(uri=uri)
-        self.assertEqual(connection.conn_type, 'scheme')
-        self.assertEqual(connection.host, 'host/location')
-        self.assertEqual(connection.schema, 'schema')
-        self.assertEqual(connection.login, 'user')
-        self.assertEqual(connection.password, 'password')
-        self.assertEqual(connection.port, 1234)
-        self.assertIsNone(connection.extra)
-
-    def test_connection_from_uri_with_extras(self):
-        uri = 'scheme://user:password@host%2flocation:1234/schema?'\
-            'extra1=a%20value&extra2=%2fpath%2f'
-        connection = Connection(uri=uri)
-        self.assertEqual(connection.conn_type, 'scheme')
-        self.assertEqual(connection.host, 'host/location')
-        self.assertEqual(connection.schema, 'schema')
-        self.assertEqual(connection.login, 'user')
-        self.assertEqual(connection.password, 'password')
-        self.assertEqual(connection.port, 1234)
-        self.assertDictEqual(connection.extra_dejson, {'extra1': 'a value',
-                                                       'extra2': '/path/'})
+
+class VariableTest(unittest.TestCase):
+    def setUp(self):
+        models._fernet = None
+
+    def tearDown(self):
+        models._fernet = None
+
+    @patch('airflow.models.configuration.conf.get')
+    def test_variable_no_encryption(self, mock_get):
+        """
+        Test variables without encryption
+        """
+        mock_get.return_value = ''
+        Variable.set('key', 'value')
+        session = settings.Session()
+        test_var = session.query(Variable).filter(Variable.key == 'key').one()
+        self.assertFalse(test_var.is_encrypted)
+        self.assertEqual(test_var.val, 'value')
+
+    @patch('airflow.models.configuration.conf.get')
+    def test_variable_with_encryption(self, mock_get):
+        """
+        Test variables with encryption
+        """
+        mock_get.return_value = Fernet.generate_key().decode()
+        Variable.set('key', 'value')
+        session = settings.Session()
+        test_var = session.query(Variable).filter(Variable.key == 'key').one()
+        self.assertTrue(test_var.is_encrypted)
+        self.assertEqual(test_var.val, 'value')
+
+    @patch('airflow.models.configuration.conf.get')
+    def test_var_with_encryption_rotate_fernet_key(self, mock_get):
+        """
+        Tests rotating encrypted variables.
+        """
+        key1 = Fernet.generate_key()
+        key2 = Fernet.generate_key()
+
+        mock_get.return_value = key1.decode()
+        Variable.set('key', 'value')
+        session = settings.Session()
+        test_var = session.query(Variable).filter(Variable.key == 'key').one()
+        self.assertTrue(test_var.is_encrypted)
+        self.assertEqual(test_var.val, 'value')
+        self.assertEqual(Fernet(key1).decrypt(test_var._val.encode()), b'value')
+
+        # Test decrypt of old value with new key
+        mock_get.return_value = ','.join([key2.decode(), key1.decode()])
+        models._fernet = None
+        self.assertEqual(test_var.val, 'value')
+
+        # Test decrypt of new value with new key
+        test_var.rotate_fernet_key()
+        self.assertTrue(test_var.is_encrypted)
+        self.assertEqual(test_var.val, 'value')
+        self.assertEqual(Fernet(key2).decrypt(test_var._val.encode()), b'value')
diff --git a/tests/test_utils/README.md b/tests/test_utils/README.md
index 8a5c90dfd25f4..a93a0aa34c13c 100644
--- a/tests/test_utils/README.md
+++ b/tests/test_utils/README.md
@@ -1 +1,20 @@
-Utilities for use in tests.
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Utilities for use in tests.
diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/test_utils/__init__.py
+++ b/tests/test_utils/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/test_utils/db.py b/tests/test_utils/db.py
new file mode 100644
index 0000000000000..0a2f3961c2852
--- /dev/null
+++ b/tests/test_utils/db.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from airflow.models import DagRun, TaskInstance, Pool, DagModel, errors
+from airflow.models.slamiss import SlaMiss
+from airflow.utils.db import create_session
+
+
+def clear_db_runs():
+    with create_session() as session:
+        session.query(DagRun).delete()
+        session.query(TaskInstance).delete()
+
+
+def clear_db_dags():
+    with create_session() as session:
+        session.query(DagModel).delete()
+
+
+def clear_db_sla_miss():
+    with create_session() as session:
+        session.query(SlaMiss).delete()
+
+
+def clear_db_errors():
+    with create_session() as session:
+        session.query(errors.ImportError).delete()
+
+
+def clear_db_pools():
+    with create_session() as session:
+        session.query(Pool).delete()
diff --git a/tests/test_utils/fake_datetime.py b/tests/test_utils/fake_datetime.py
index 42bb01df01811..8182d83e9a086 100644
--- a/tests/test_utils/fake_datetime.py
+++ b/tests/test_utils/fake_datetime.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/test_utils/reset_warning_registry.py b/tests/test_utils/reset_warning_registry.py
index 3ca78eda67cb4..65f08a2300468 100644
--- a/tests/test_utils/reset_warning_registry.py
+++ b/tests/test_utils/reset_warning_registry.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/__init__.py b/tests/ti_deps/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/ti_deps/__init__.py
+++ b/tests/ti_deps/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/contexts/__init__.py b/tests/ti_deps/contexts/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/ti_deps/contexts/__init__.py
+++ b/tests/ti_deps/contexts/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/__init__.py b/tests/ti_deps/deps/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/ti_deps/deps/__init__.py
+++ b/tests/ti_deps/deps/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/fake_models.py b/tests/ti_deps/deps/fake_models.py
index da4731710b0b0..f7b94846ee5f2 100644
--- a/tests/ti_deps/deps/fake_models.py
+++ b/tests/ti_deps/deps/fake_models.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -26,7 +26,7 @@ def __init__(self, **kwds):
         self.__dict__.update(kwds)
 
     def pool_full(self):
-        # Allow users of this fake to set pool_filled in the contructor to make this
+        # Allow users of this fake to set pool_filled in the constructor to make this
         # return True
         try:
             return self.pool_filled
diff --git a/tests/ti_deps/deps/test_dag_ti_slots_available_dep.py b/tests/ti_deps/deps/test_dag_ti_slots_available_dep.py
index 0ea1503a21afa..cfac5d3051835 100644
--- a/tests/ti_deps/deps/test_dag_ti_slots_available_dep.py
+++ b/tests/ti_deps/deps/test_dag_ti_slots_available_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_dag_unpaused_dep.py b/tests/ti_deps/deps/test_dag_unpaused_dep.py
index c1f4148d29cd2..8a6df5c1bf319 100644
--- a/tests/ti_deps/deps/test_dag_unpaused_dep.py
+++ b/tests/ti_deps/deps/test_dag_unpaused_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_dagrun_exists_dep.py b/tests/ti_deps/deps/test_dagrun_exists_dep.py
index 62aff97cdce89..85d649aaba15d 100644
--- a/tests/ti_deps/deps/test_dagrun_exists_dep.py
+++ b/tests/ti_deps/deps/test_dagrun_exists_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_not_in_retry_period_dep.py b/tests/ti_deps/deps/test_not_in_retry_period_dep.py
index afd9e01a8ff40..ca0726dc0c388 100644
--- a/tests/ti_deps/deps/test_not_in_retry_period_dep.py
+++ b/tests/ti_deps/deps/test_not_in_retry_period_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_not_running_dep.py b/tests/ti_deps/deps/test_not_running_dep.py
index 37696186b7288..1bc4261bb86bc 100644
--- a/tests/ti_deps/deps/test_not_running_dep.py
+++ b/tests/ti_deps/deps/test_not_running_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_not_skipped_dep.py b/tests/ti_deps/deps/test_not_skipped_dep.py
index 07d0bffdc2acb..6e1d832940a19 100644
--- a/tests/ti_deps/deps/test_not_skipped_dep.py
+++ b/tests/ti_deps/deps/test_not_skipped_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py b/tests/ti_deps/deps/test_prev_dagrun_dep.py
index a51dcd091b0e9..e5d8cdf2b15fb 100644
--- a/tests/ti_deps/deps/test_prev_dagrun_dep.py
+++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
new file mode 100644
index 0000000000000..4bfcd6f12e3de
--- /dev/null
+++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from datetime import timedelta
+from mock import Mock, patch
+
+from airflow.models import TaskInstance, DAG
+from airflow.models.taskreschedule import TaskReschedule
+from airflow.ti_deps.dep_context import DepContext
+from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep
+from airflow.utils.state import State
+from airflow.utils.timezone import utcnow
+
+
+class NotInReschedulePeriodDepTest(unittest.TestCase):
+
+    def _get_task_instance(self, state):
+        dag = DAG('test_dag')
+        task = Mock(dag=dag)
+        ti = TaskInstance(task=task, state=state, execution_date=None)
+        return ti
+
+    def _get_task_reschedule(self, reschedule_date):
+        task = Mock(dag_id='test_dag', task_id='test_task')
+        tr = TaskReschedule(task=task, execution_date=None, try_number=None,
+                            start_date=reschedule_date, end_date=reschedule_date,
+                            reschedule_date=reschedule_date)
+        return tr
+
+    def test_should_pass_if_ignore_in_reschedule_period_is_set(self):
+        ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+        dep_context = DepContext(ignore_in_reschedule_period=True)
+        self.assertTrue(ReadyToRescheduleDep().is_met(ti=ti, dep_context=dep_context))
+
+    def test_should_pass_if_not_in_none_state(self):
+        ti = self._get_task_instance(State.UP_FOR_RETRY)
+        self.assertTrue(ReadyToRescheduleDep().is_met(ti=ti))
+
+    @patch('airflow.models.taskreschedule.TaskReschedule.find_for_task_instance', return_value=[])
+    def test_should_pass_if_no_reschedule_record_exists(self, find_for_task_instance):
+        ti = self._get_task_instance(State.NONE)
+        self.assertTrue(ReadyToRescheduleDep().is_met(ti=ti))
+
+    @patch('airflow.models.taskreschedule.TaskReschedule.find_for_task_instance')
+    def test_should_pass_after_reschedule_date_one(self, find_for_task_instance):
+        find_for_task_instance.return_value = [
+            self._get_task_reschedule(utcnow() - timedelta(minutes=1)),
+        ]
+        ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+        self.assertTrue(ReadyToRescheduleDep().is_met(ti=ti))
+
+    @patch('airflow.models.taskreschedule.TaskReschedule.find_for_task_instance')
+    def test_should_pass_after_reschedule_date_multiple(self, find_for_task_instance):
+        find_for_task_instance.return_value = [
+            self._get_task_reschedule(utcnow() - timedelta(minutes=21)),
+            self._get_task_reschedule(utcnow() - timedelta(minutes=11)),
+            self._get_task_reschedule(utcnow() - timedelta(minutes=1)),
+        ]
+        ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+        self.assertTrue(ReadyToRescheduleDep().is_met(ti=ti))
+
+    @patch('airflow.models.taskreschedule.TaskReschedule.find_for_task_instance')
+    def test_should_fail_before_reschedule_date_one(self, find_for_task_instance):
+        find_for_task_instance.return_value = [
+            self._get_task_reschedule(utcnow() + timedelta(minutes=1)),
+        ]
+        ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+        self.assertFalse(ReadyToRescheduleDep().is_met(ti=ti))
+
+    @patch('airflow.models.taskreschedule.TaskReschedule.find_for_task_instance')
+    def test_should_fail_before_reschedule_date_multiple(self, find_for_task_instance):
+        find_for_task_instance.return_value = [
+            self._get_task_reschedule(utcnow() - timedelta(minutes=19)),
+            self._get_task_reschedule(utcnow() - timedelta(minutes=9)),
+            self._get_task_reschedule(utcnow() + timedelta(minutes=1)),
+        ]
+        ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+        self.assertFalse(ReadyToRescheduleDep().is_met(ti=ti))
diff --git a/tests/ti_deps/deps/test_runnable_exec_date_dep.py b/tests/ti_deps/deps/test_runnable_exec_date_dep.py
index 643985ce4c0cb..16057c8108b27 100644
--- a/tests/ti_deps/deps/test_runnable_exec_date_dep.py
+++ b/tests/ti_deps/deps/test_runnable_exec_date_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,6 +25,7 @@
 from airflow.ti_deps.deps.runnable_exec_date_dep import RunnableExecDateDep
 from airflow.utils.timezone import datetime
 
+
 class RunnableExecDateDepTest(unittest.TestCase):
 
     def _get_task_instance(self, execution_date, dag_end_date=None, task_end_date=None):
diff --git a/tests/ti_deps/deps/test_task_concurrency.py b/tests/ti_deps/deps/test_task_concurrency.py
index fd1b28da33967..ad3c0d9ea0dad 100644
--- a/tests/ti_deps/deps/test_task_concurrency.py
+++ b/tests/ti_deps/deps/test_task_concurrency.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,6 @@
 from airflow.models import DAG, BaseOperator
 from airflow.ti_deps.dep_context import DepContext
 from airflow.ti_deps.deps.task_concurrency_dep import TaskConcurrencyDep
-from airflow.utils.state import State
 
 
 class TaskConcurrencyDepTest(unittest.TestCase):
@@ -53,4 +52,3 @@ def test_reached_concurrency(self):
         self.assertTrue(TaskConcurrencyDep().is_met(ti=ti, dep_context=dep_context))
         ti.get_num_running_task_instances = lambda x: 2
         self.assertFalse(TaskConcurrencyDep().is_met(ti=ti, dep_context=dep_context))
-
diff --git a/tests/ti_deps/deps/test_trigger_rule_dep.py b/tests/ti_deps/deps/test_trigger_rule_dep.py
index f4a96f5bd0658..c149026944450 100644
--- a/tests/ti_deps/deps/test_trigger_rule_dep.py
+++ b/tests/ti_deps/deps/test_trigger_rule_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -23,6 +23,7 @@
 from airflow.models import BaseOperator, TaskInstance
 from airflow.utils.trigger_rule import TriggerRule
 from airflow.ti_deps.deps.trigger_rule_dep import TriggerRuleDep
+from airflow.utils.db import create_session
 from airflow.utils.state import State
 
 
@@ -34,7 +35,7 @@ def _get_task_instance(self, trigger_rule=TriggerRule.ALL_SUCCESS,
                             start_date=datetime(2015, 1, 1))
         if upstream_task_ids:
             task._upstream_task_ids.update(upstream_task_ids)
-        return TaskInstance(task=task, state=state, execution_date=None)
+        return TaskInstance(task=task, state=state, execution_date=task.start_date)
 
     def test_no_upstream_tasks(self):
         """
@@ -163,6 +164,44 @@ def test_all_success_tr_failure(self):
         self.assertEqual(len(dep_statuses), 1)
         self.assertFalse(dep_statuses[0].passed)
 
+    def test_none_failed_tr_success(self):
+        """
+        All success including skip trigger rule success
+        """
+        ti = self._get_task_instance(TriggerRule.NONE_FAILED,
+                                     upstream_task_ids=["FakeTaskID",
+                                                        "OtherFakeTaskID"])
+        dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
+            ti=ti,
+            successes=1,
+            skipped=1,
+            failed=0,
+            upstream_failed=0,
+            done=2,
+            flag_upstream_failed=False,
+            session="Fake Session"))
+        self.assertEqual(len(dep_statuses), 0)
+
+    def test_none_failed_tr_failure(self):
+        """
+        All success including skip trigger rule failure
+        """
+        ti = self._get_task_instance(TriggerRule.NONE_FAILED,
+                                     upstream_task_ids=["FakeTaskID",
+                                                        "OtherFakeTaskID",
+                                                        "FailedFakeTaskID"])
+        dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
+            ti=ti,
+            successes=1,
+            skipped=1,
+            failed=1,
+            upstream_failed=0,
+            done=3,
+            flag_upstream_failed=False,
+            session="Fake Session"))
+        self.assertEqual(len(dep_statuses), 1)
+        self.assertFalse(dep_statuses[0].passed)
+
     def test_all_failed_tr_success(self):
         """
         All-failed trigger rule success
@@ -237,6 +276,73 @@ def test_all_done_tr_failure(self):
         self.assertEqual(len(dep_statuses), 1)
         self.assertFalse(dep_statuses[0].passed)
 
+    def test_none_skipped_tr_success(self):
+        """
+        None-skipped trigger rule success
+        """
+
+        ti = self._get_task_instance(TriggerRule.NONE_SKIPPED,
+                                     upstream_task_ids=["FakeTaskID",
+                                                        "OtherFakeTaskID",
+                                                        "FailedFakeTaskID"])
+        with create_session() as session:
+            dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
+                ti=ti,
+                successes=2,
+                skipped=0,
+                failed=1,
+                upstream_failed=0,
+                done=3,
+                flag_upstream_failed=False,
+                session=session))
+            self.assertEqual(len(dep_statuses), 0)
+
+            # with `flag_upstream_failed` set to True
+            dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
+                ti=ti,
+                successes=0,
+                skipped=0,
+                failed=3,
+                upstream_failed=0,
+                done=3,
+                flag_upstream_failed=True,
+                session=session))
+            self.assertEqual(len(dep_statuses), 0)
+
+    def test_none_skipped_tr_failure(self):
+        """
+        None-skipped trigger rule failure
+        """
+        ti = self._get_task_instance(TriggerRule.NONE_SKIPPED,
+                                     upstream_task_ids=["FakeTaskID",
+                                                        "SkippedTaskID"])
+
+        with create_session() as session:
+            dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
+                ti=ti,
+                successes=1,
+                skipped=1,
+                failed=0,
+                upstream_failed=0,
+                done=2,
+                flag_upstream_failed=False,
+                session=session))
+            self.assertEqual(len(dep_statuses), 1)
+            self.assertFalse(dep_statuses[0].passed)
+
+            # with `flag_upstream_failed` set to True
+            dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
+                ti=ti,
+                successes=1,
+                skipped=1,
+                failed=0,
+                upstream_failed=0,
+                done=2,
+                flag_upstream_failed=True,
+                session=session))
+            self.assertEqual(len(dep_statuses), 1)
+            self.assertFalse(dep_statuses[0].passed)
+
     def test_unknown_tr(self):
         """
         Unknown trigger rules should cause this dep to fail
diff --git a/tests/ti_deps/deps/test_valid_state_dep.py b/tests/ti_deps/deps/test_valid_state_dep.py
index 19b18935dbd06..188b074db6526 100644
--- a/tests/ti_deps/deps/test_valid_state_dep.py
+++ b/tests/ti_deps/deps/test_valid_state_dep.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils.py b/tests/utils.py
deleted file mode 100644
index 12d3ff037b264..0000000000000
--- a/tests/utils.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import logging
-import unittest
-
-import airflow.utils.logging
-from airflow import configuration
-from airflow.exceptions import AirflowException
-from airflow.utils.operator_resources import Resources
-
-
-class LogUtilsTest(unittest.TestCase):
-
-    def test_gcs_url_parse(self):
-        """
-        Test GCS url parsing
-        """
-        logging.info(
-            'About to create a GCSLog object without a connection. This will '
-            'log an error but testing will proceed.')
-        glog = airflow.utils.logging.GCSLog()
-
-        self.assertEqual(
-            glog.parse_gcs_url('gs://bucket/path/to/blob'),
-            ('bucket', 'path/to/blob'))
-
-        # invalid URI
-        self.assertRaises(
-            AirflowException,
-            glog.parse_gcs_url,
-            'gs:/bucket/path/to/blob')
-
-        # trailing slash
-        self.assertEqual(
-            glog.parse_gcs_url('gs://bucket/path/to/blob/'),
-            ('bucket', 'path/to/blob'))
-
-        # bucket only
-        self.assertEqual(
-            glog.parse_gcs_url('gs://bucket/'),
-            ('bucket', ''))
-
-class OperatorResourcesTest(unittest.TestCase):
-
-    def setUp(self):
-        configuration.load_test_config()
-
-    def test_all_resources_specified(self):
-        resources = Resources(cpus=1, ram=2, disk=3, gpus=4)
-        self.assertEqual(resources.cpus.qty, 1)
-        self.assertEqual(resources.ram.qty, 2)
-        self.assertEqual(resources.disk.qty, 3)
-        self.assertEqual(resources.gpus.qty, 4)
-
-    def test_some_resources_specified(self):
-        resources = Resources(cpus=0, disk=1)
-        self.assertEqual(resources.cpus.qty, 0)
-        self.assertEqual(resources.ram.qty,
-                         configuration.conf.getint('operators', 'default_ram'))
-        self.assertEqual(resources.disk.qty, 1)
-        self.assertEqual(resources.gpus.qty,
-                         configuration.conf.getint('operators', 'default_gpus'))
-
-    def test_no_resources_specified(self):
-        resources = Resources()
-        self.assertEqual(resources.cpus.qty,
-                         configuration.conf.getint('operators', 'default_cpus'))
-        self.assertEqual(resources.ram.qty,
-                         configuration.conf.getint('operators', 'default_ram'))
-        self.assertEqual(resources.disk.qty,
-                         configuration.conf.getint('operators', 'default_disk'))
-        self.assertEqual(resources.gpus.qty,
-                         configuration.conf.getint('operators', 'default_gpus'))
-
-    def test_negative_resource_qty(self):
-        with self.assertRaises(AirflowException):
-            Resources(cpus=-1)
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/log/__init__.py b/tests/utils/log/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/utils/log/__init__.py
+++ b/tests/utils/log/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/log/elasticmock/__init__.py b/tests/utils/log/elasticmock/__init__.py
index 73c4879847138..6d4ce51fa26ba 100644
--- a/tests/utils/log/elasticmock/__init__.py
+++ b/tests/utils/log/elasticmock/__init__.py
@@ -23,13 +23,14 @@
 # SOFTWARE.
 
 from functools import wraps
+from typing import Dict
 
 from elasticsearch.client import _normalize_hosts
-from mock import patch
+from tests.compat import patch
 
 from .fake_elasticsearch import FakeElasticsearch
 
-ELASTIC_INSTANCES = {}
+ELASTIC_INSTANCES = {}  # type: Dict[str, FakeElasticsearch]
 
 
 def _get_elasticmock(hosts=None, *args, **kwargs):
diff --git a/tests/utils/log/test_es_task_handler.py b/tests/utils/log/test_es_task_handler.py
index 787094ef53860..b6dd2fe1fb051 100644
--- a/tests/utils/log/test_es_task_handler.py
+++ b/tests/utils/log/test_es_task_handler.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -171,10 +171,9 @@ def test_read_raises(self):
             with mock.patch("elasticsearch_dsl.Search.execute") as mock_execute:
                 mock_execute.side_effect = Exception('Failed to read')
                 logs, metadatas = self.es_task_handler.read(self.ti, 1)
-            msg = "Could not read log with log_id: {}".format(self.LOG_ID)
-            mock_exception.assert_called_once()
+            assert mock_exception.call_count == 1
             args, kwargs = mock_exception.call_args
-            self.assertIn(msg, args[0])
+            self.assertIn("Could not read log with log_id:", args[0])
 
         self.assertEqual(1, len(logs))
         self.assertEqual(len(logs), len(metadatas))
diff --git a/tests/utils/log/test_file_processor_handler.py b/tests/utils/log/test_file_processor_handler.py
index 23cf1d4daae0e..42b1a1e27040f 100644
--- a/tests/utils/log/test_file_processor_handler.py
+++ b/tests/utils/log/test_file_processor_handler.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/test_cli_util.py b/tests/utils/test_cli_util.py
index 00dc9f959f1bc..6a411faba7ed5 100644
--- a/tests/utils/test_cli_util.py
+++ b/tests/utils/test_cli_util.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -42,17 +42,17 @@ def test_metrics_build(self):
                     'task_id': 'bar',
                     'execution_date': exec_date}
         for k, v in expected.items():
-            self.assertEquals(v, metrics.get(k))
+            self.assertEqual(v, metrics.get(k))
 
         self.assertTrue(metrics.get('start_datetime') <= datetime.utcnow())
         self.assertTrue(metrics.get('full_command'))
 
         log_dao = metrics.get('log')
         self.assertTrue(log_dao)
-        self.assertEquals(log_dao.dag_id, metrics.get('dag_id'))
-        self.assertEquals(log_dao.task_id, metrics.get('task_id'))
-        self.assertEquals(log_dao.execution_date, metrics.get('execution_date'))
-        self.assertEquals(log_dao.owner, metrics.get('user'))
+        self.assertEqual(log_dao.dag_id, metrics.get('dag_id'))
+        self.assertEqual(log_dao.task_id, metrics.get('task_id'))
+        self.assertEqual(log_dao.execution_date, metrics.get('execution_date'))
+        self.assertEqual(log_dao.owner, metrics.get('user'))
 
     def test_fail_function(self):
         """
diff --git a/tests/utils/test_compression.py b/tests/utils/test_compression.py
index a0886880bf877..5a367094f4d27 100644
--- a/tests/utils/test_compression.py
+++ b/tests/utils/test_compression.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/test_dag_processing.py b/tests/utils/test_dag_processing.py
index 7abe7efe9b35c..ad33b7f64372d 100644
--- a/tests/utils/test_dag_processing.py
+++ b/tests/utils/test_dag_processing.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,11 +17,119 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
+import sys
+import tempfile
 import unittest
+from datetime import timedelta
 
 from mock import MagicMock
 
-from airflow.utils.dag_processing import DagFileProcessorManager
+from airflow import configuration as conf
+from airflow.configuration import mkdir_p
+from airflow.jobs import DagFileProcessor
+from airflow.jobs import LocalTaskJob as LJ
+from airflow.models import DagBag, TaskInstance as TI
+from airflow.utils import timezone
+from airflow.utils.dag_processing import (DagFileProcessorAgent, DagFileProcessorManager,
+                                          SimpleTaskInstance)
+from airflow.utils.db import create_session
+from airflow.utils.state import State
+
+TEST_DAG_FOLDER = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'dags')
+
+DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+
+SETTINGS_FILE_VALID = """
+LOGGING_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'airflow.task': {
+            'format': '[%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'airflow.task',
+            'stream': 'ext://sys.stdout'
+        },
+        'task': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'airflow.task',
+            'stream': 'ext://sys.stdout'
+        },
+    },
+    'loggers': {
+        'airflow': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False
+        },
+        'airflow.task': {
+            'handlers': ['task'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+    }
+}
+"""
+
+SETTINGS_DEFAULT_NAME = 'custom_airflow_local_settings'
+
+
+class settings_context(object):
+    """
+    Sets a settings file and puts it in the Python classpath
+
+    :param content:
+          The content of the settings file
+    """
+
+    def __init__(self, content, dir=None, name='LOGGING_CONFIG'):
+        self.content = content
+        self.settings_root = tempfile.mkdtemp()
+        filename = "{}.py".format(SETTINGS_DEFAULT_NAME)
+
+        if dir:
+            # Replace slashes by dots
+            self.module = dir.replace('/', '.') + '.' + SETTINGS_DEFAULT_NAME + '.' + name
+
+            # Create the directory structure
+            dir_path = os.path.join(self.settings_root, dir)
+            mkdir_p(dir_path)
+
+            # Add the __init__ for the directories
+            # This is required for Python 2.7
+            basedir = self.settings_root
+            for part in dir.split('/'):
+                open(os.path.join(basedir, '__init__.py'), 'w').close()
+                basedir = os.path.join(basedir, part)
+            open(os.path.join(basedir, '__init__.py'), 'w').close()
+
+            self.settings_file = os.path.join(dir_path, filename)
+        else:
+            self.module = SETTINGS_DEFAULT_NAME + '.' + name
+            self.settings_file = os.path.join(self.settings_root, filename)
+
+    def __enter__(self):
+        with open(self.settings_file, 'w') as handle:
+            handle.writelines(self.content)
+        sys.path.append(self.settings_root)
+        conf.set(
+            'core',
+            'logging_config_class',
+            self.module
+        )
+        return self.settings_file
+
+    def __exit__(self, *exc_info):
+        # shutil.rmtree(self.settings_root)
+        # Reset config
+        conf.set('core', 'logging_config_class', '')
+        sys.path.remove(self.settings_root)
 
 
 class TestDagFileProcessorManager(unittest.TestCase):
@@ -29,11 +137,12 @@ def test_set_file_paths_when_processor_file_path_not_in_new_file_paths(self):
         manager = DagFileProcessorManager(
             dag_directory='directory',
             file_paths=['abc.txt'],
-            parallelism=1,
-            process_file_interval=1,
             max_runs=1,
-            min_file_parsing_loop_time=0,
-            processor_factory=MagicMock().return_value)
+            processor_factory=MagicMock().return_value,
+            signal_conn=MagicMock(),
+            stat_queue=MagicMock(),
+            result_queue=MagicMock,
+            async_mode=True)
 
         mock_processor = MagicMock()
         mock_processor.stop.side_effect = AttributeError(
@@ -49,11 +158,12 @@ def test_set_file_paths_when_processor_file_path_is_in_new_file_paths(self):
         manager = DagFileProcessorManager(
             dag_directory='directory',
             file_paths=['abc.txt'],
-            parallelism=1,
-            process_file_interval=1,
             max_runs=1,
-            min_file_parsing_loop_time=0,
-            processor_factory=MagicMock().return_value)
+            processor_factory=MagicMock().return_value,
+            signal_conn=MagicMock(),
+            stat_queue=MagicMock(),
+            result_queue=MagicMock,
+            async_mode=True)
 
         mock_processor = MagicMock()
         mock_processor.stop.side_effect = AttributeError(
@@ -64,3 +174,155 @@ def test_set_file_paths_when_processor_file_path_is_in_new_file_paths(self):
 
         manager.set_file_paths(['abc.txt'])
         self.assertDictEqual(manager._processors, {'abc.txt': mock_processor})
+
+    def test_find_zombies(self):
+        manager = DagFileProcessorManager(
+            dag_directory='directory',
+            file_paths=['abc.txt'],
+            max_runs=1,
+            processor_factory=MagicMock().return_value,
+            signal_conn=MagicMock(),
+            stat_queue=MagicMock(),
+            result_queue=MagicMock,
+            async_mode=True)
+
+        dagbag = DagBag(TEST_DAG_FOLDER)
+        with create_session() as session:
+            session.query(LJ).delete()
+            dag = dagbag.get_dag('example_branch_operator')
+            task = dag.get_task(task_id='run_this_first')
+
+            ti = TI(task, DEFAULT_DATE, State.RUNNING)
+            lj = LJ(ti)
+            lj.state = State.SHUTDOWN
+            lj.id = 1
+            ti.job_id = lj.id
+
+            session.add(lj)
+            session.add(ti)
+            session.commit()
+
+            manager._last_zombie_query_time = timezone.utcnow() - timedelta(
+                seconds=manager._zombie_threshold_secs + 1)
+            zombies = manager._find_zombies()
+            self.assertEqual(1, len(zombies))
+            self.assertIsInstance(zombies[0], SimpleTaskInstance)
+            self.assertEqual(ti.dag_id, zombies[0].dag_id)
+            self.assertEqual(ti.task_id, zombies[0].task_id)
+            self.assertEqual(ti.execution_date, zombies[0].execution_date)
+
+            session.query(TI).delete()
+            session.query(LJ).delete()
+
+
+class TestDagFileProcessorAgent(unittest.TestCase):
+    def test_reload_module(self):
+        """
+        Configure the context to have core.logging_config_class set to a fake logging
+        class path, thus when reloading logging module the airflow.processor_manager
+        logger should not be configured.
+        """
+        with settings_context(SETTINGS_FILE_VALID):
+            # Launch a process through DagFileProcessorAgent, which will try
+            # reload the logging module.
+            def processor_factory(file_path, zombies):
+                return DagFileProcessor(file_path,
+                                        False,
+                                        [],
+                                        zombies)
+
+            test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
+            async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
+
+            log_file_loc = conf.get('core', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION')
+            try:
+                os.remove(log_file_loc)
+            except OSError:
+                pass
+
+            # Starting dag processing with 0 max_runs to avoid redundant operations.
+            processor_agent = DagFileProcessorAgent(test_dag_path,
+                                                    [],
+                                                    0,
+                                                    processor_factory,
+                                                    async_mode)
+            manager_process = \
+                processor_agent._launch_process(processor_agent._dag_directory,
+                                                processor_agent._file_paths,
+                                                processor_agent._max_runs,
+                                                processor_agent._processor_factory,
+                                                processor_agent._child_signal_conn,
+                                                processor_agent._stat_queue,
+                                                processor_agent._result_queue,
+                                                processor_agent._async_mode)
+            if not async_mode:
+                processor_agent.heartbeat()
+
+            manager_process.join()
+
+            # Since we are reloading logging config not creating this file,
+            # we should expect it to be nonexistent.
+            self.assertFalse(os.path.isfile(log_file_loc))
+
+    def test_parse_once(self):
+        def processor_factory(file_path, zombies):
+            return DagFileProcessor(file_path,
+                                    False,
+                                    [],
+                                    zombies)
+
+        test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
+        async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
+        processor_agent = DagFileProcessorAgent(test_dag_path,
+                                                [test_dag_path],
+                                                1,
+                                                processor_factory,
+                                                async_mode)
+        processor_agent.start()
+        parsing_result = []
+        while not processor_agent.done:
+            if not async_mode:
+                processor_agent.heartbeat()
+                processor_agent.wait_until_finished()
+            parsing_result.extend(processor_agent.harvest_simple_dags())
+
+        dag_ids = [result.dag_id for result in parsing_result]
+        self.assertEqual(dag_ids.count('test_start_date_scheduling'), 1)
+
+    def test_launch_process(self):
+        def processor_factory(file_path, zombies):
+            return DagFileProcessor(file_path,
+                                    False,
+                                    [],
+                                    zombies)
+
+        test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
+        async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
+
+        log_file_loc = conf.get('core', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION')
+        try:
+            os.remove(log_file_loc)
+        except OSError:
+            pass
+
+        # Starting dag processing with 0 max_runs to avoid redundant operations.
+        processor_agent = DagFileProcessorAgent(test_dag_path,
+                                                [],
+                                                0,
+                                                processor_factory,
+                                                async_mode)
+        manager_process = \
+            processor_agent._launch_process(processor_agent._dag_directory,
+                                            processor_agent._file_paths,
+                                            processor_agent._max_runs,
+                                            processor_agent._processor_factory,
+                                            processor_agent._child_signal_conn,
+                                            processor_agent._stat_queue,
+                                            processor_agent._result_queue,
+                                            processor_agent._async_mode)
+        if not async_mode:
+            processor_agent.heartbeat()
+
+        manager_process.join()
+
+        self.assertTrue(os.path.isfile(log_file_loc))
diff --git a/tests/utils/test_dates.py b/tests/utils/test_dates.py
index ba762c8d09cfe..85613d079139d 100644
--- a/tests/utils/test_dates.py
+++ b/tests/utils/test_dates.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -33,26 +33,22 @@ def test_days_ago(self):
 
         self.assertTrue(dates.days_ago(0) == today_midnight)
 
-        self.assertTrue(
-            dates.days_ago(100) == today_midnight + timedelta(days=-100))
-
-        self.assertTrue(
-            dates.days_ago(0, hour=3) == today_midnight + timedelta(hours=3))
-        self.assertTrue(
-            dates.days_ago(0, minute=3)
-            == today_midnight + timedelta(minutes=3))
-        self.assertTrue(
-            dates.days_ago(0, second=3)
-            == today_midnight + timedelta(seconds=3))
-        self.assertTrue(
-            dates.days_ago(0, microsecond=3)
-            == today_midnight + timedelta(microseconds=3))
+        self.assertTrue(dates.days_ago(100) == today_midnight + timedelta(days=-100))
+
+        self.assertTrue(dates.days_ago(0, hour=3) == today_midnight + timedelta(hours=3))
+        self.assertTrue(dates.days_ago(0, minute=3) == today_midnight + timedelta(minutes=3))
+        self.assertTrue(dates.days_ago(0, second=3) == today_midnight + timedelta(seconds=3))
+        self.assertTrue(dates.days_ago(0, microsecond=3) == today_midnight + timedelta(microseconds=3))
 
     def test_parse_execution_date(self):
         execution_date_str_wo_ms = '2017-11-02 00:00:00'
         execution_date_str_w_ms = '2017-11-05 16:18:30.989729'
         bad_execution_date_str = '2017-11-06TXX:00:00Z'
 
-        self.assertEqual(timezone.datetime(2017, 11, 2, 0, 0, 0), dates.parse_execution_date(execution_date_str_wo_ms))
-        self.assertEqual(timezone.datetime(2017, 11, 5, 16, 18, 30, 989729), dates.parse_execution_date(execution_date_str_w_ms))
+        self.assertEqual(
+            timezone.datetime(2017, 11, 2, 0, 0, 0),
+            dates.parse_execution_date(execution_date_str_wo_ms))
+        self.assertEqual(
+            timezone.datetime(2017, 11, 5, 16, 18, 30, 989729),
+            dates.parse_execution_date(execution_date_str_w_ms))
         self.assertRaises(ValueError, dates.parse_execution_date, bad_execution_date_str)
diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py
new file mode 100644
index 0000000000000..718e2ae1066fe
--- /dev/null
+++ b/tests/utils/test_db.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from airflow.models import Base as airflow_base
+
+from airflow.settings import engine
+from alembic.autogenerate import compare_metadata
+from alembic.migration import MigrationContext
+from sqlalchemy import MetaData
+
+
+class DbTest(unittest.TestCase):
+
+    def test_database_schema_and_sqlalchemy_model_are_in_sync(self):
+        all_meta_data = MetaData()
+        for (table_name, table) in airflow_base.metadata.tables.items():
+            all_meta_data._add_table(table_name, table.schema, table)
+
+        # create diff between database schema and SQLAlchemy model
+        mc = MigrationContext.configure(engine.connect())
+        diff = compare_metadata(mc, all_meta_data)
+
+        # known diffs to ignore
+        ignores = [
+            # users.password is not part of User model,
+            # otherwise it would show up in (old) UI
+            lambda t: (t[0] == 'remove_column' and
+                       t[2] == 'users' and
+                       t[3].name == 'password'),
+
+            # ignore tables created by celery
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'celery_taskmeta'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'celery_tasksetmeta'),
+
+            # ignore indices created by celery
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'task_id'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'taskset_id'),
+
+            # Ignore all the fab tables
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_permission'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_register_user'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_role'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_permission_view'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_permission_view_role'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_user_role'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_user'),
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'ab_view_menu'),
+
+            # Ignore all the fab indices
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'permission_id'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'name'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'user_id'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'username'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'field_string'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'email'),
+            lambda t: (t[0] == 'remove_index' and
+                       t[1].name == 'permission_view_id'),
+
+            # from test_security unit test
+            lambda t: (t[0] == 'remove_table' and
+                       t[1].name == 'some_model'),
+        ]
+        for ignore in ignores:
+            diff = [d for d in diff if not ignore(d)]
+
+        self.assertFalse(
+            diff,
+            'Database schema and SQLAlchemy model are not in sync: ' + str(diff)
+        )
diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py
index f9b48148bb354..5e6ff5f935714 100644
--- a/tests/utils/test_decorators.py
+++ b/tests/utils/test_decorators.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,7 +18,8 @@
 # under the License.
 
 import unittest
-from airflow.utils.decorators import apply_defaults
+
+from airflow.utils.decorators import apply_defaults, cached_property
 from airflow.exceptions import AirflowException
 
 
@@ -72,3 +73,29 @@ def test_incorrect_default_args(self):
         default_args = {'random_params': True}
         with self.assertRaisesRegexp(AirflowException, 'Argument.*test_param.*required'):
             DummyClass(default_args=default_args)
+
+
+class FixtureClass:
+    @cached_property
+    def value(self):
+        """Fixture docstring"""
+        return 1, object()
+
+
+class FixtureSubClass(FixtureClass):
+    pass
+
+
+class CachedPropertyTest(unittest.TestCase):
+
+    def setUp(self):
+        self.test_obj = FixtureClass()
+        self.test_sub_obj = FixtureSubClass()
+
+    def test_cache_works(self):
+        self.assertIs(self.test_obj.value, self.test_obj.value)
+        self.assertIs(self.test_sub_obj.value, self.test_sub_obj.value)
+
+    def test_docstring(self):
+        self.assertEqual(FixtureClass.value.__doc__, "Fixture docstring")
+        self.assertEqual(FixtureSubClass.value.__doc__, "Fixture docstring")
diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py
new file mode 100644
index 0000000000000..6dae967c4abb9
--- /dev/null
+++ b/tests/utils/test_email.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from airflow.utils.email import get_email_address_list
+
+EMAILS = ['test1@example.com', 'test2@example.com']
+
+
+class EmailTest(unittest.TestCase):
+
+    def test_get_email_address_comma_sep_string(self):
+        emails_string = 'test1@example.com, test2@example.com'
+
+        self.assertEqual(
+            get_email_address_list(emails_string), EMAILS)
+
+    def test_get_email_address_colon_sep_string(self):
+        emails_string = 'test1@example.com; test2@example.com'
+
+        self.assertEqual(
+            get_email_address_list(emails_string), EMAILS)
+
+    def test_get_email_address_list(self):
+        emails_list = ['test1@example.com', 'test2@example.com']
+
+        self.assertEqual(
+            get_email_address_list(emails_list), EMAILS)
diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py
index 1005671e9e84e..9101adae5ef82 100644
--- a/tests/utils/test_helpers.py
+++ b/tests/utils/test_helpers.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,12 +20,18 @@
 import logging
 import multiprocessing
 import os
-import psutil
 import signal
 import time
 import unittest
+from datetime import datetime
 
+import psutil
+import six
+
+from airflow import DAG
 from airflow.utils import helpers
+from airflow.models import TaskInstance
+from airflow.operators.dummy_operator import DummyOperator
 
 
 class TestHelpers(unittest.TestCase):
@@ -57,6 +63,28 @@ def signal_handler(signum, frame):
         while True:
             time.sleep(1)
 
+    def test_render_log_filename(self):
+        try_number = 1
+        dag_id = 'test_render_log_filename_dag'
+        task_id = 'test_render_log_filename_task'
+        execution_date = datetime(2016, 1, 1)
+
+        dag = DAG(dag_id, start_date=execution_date)
+        task = DummyOperator(task_id=task_id, dag=dag)
+        ti = TaskInstance(task=task, execution_date=execution_date)
+
+        filename_template = "{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log"
+
+        ts = ti.get_template_context()['ts']
+        expected_filename = "{dag_id}/{task_id}/{ts}/{try_number}.log".format(dag_id=dag_id,
+                                                                              task_id=task_id,
+                                                                              ts=ts,
+                                                                              try_number=try_number)
+
+        rendered_filename = helpers.render_log_filename(ti, try_number, filename_template)
+
+        self.assertEqual(rendered_filename, expected_filename)
+
     def test_reap_process_group(self):
         """
         Spin up a process that can't be killed by SIGTERM and make sure
@@ -116,6 +144,110 @@ def test_reduce_in_chunks(self):
                                                   2),
                          14)
 
+    def test_is_in(self):
+        obj = ["list", "object"]
+        # Check for existence of a list object within a list
+        self.assertTrue(
+            helpers.is_in(obj, [obj])
+        )
+
+        # Check that an empty list returns false
+        self.assertFalse(
+            helpers.is_in(obj, [])
+        )
+
+        # Check to ensure it handles None types
+        self.assertFalse(
+            helpers.is_in(None, [obj])
+        )
+
+        # Check to ensure true will be returned of multiple objects exist
+        self.assertTrue(
+            helpers.is_in(obj, [obj, obj])
+        )
+
+    def test_is_container(self):
+        self.assertFalse(helpers.is_container("a string is not a container"))
+        self.assertTrue(helpers.is_container(["a", "list", "is", "a", "container"]))
+
+    def test_as_tuple(self):
+        self.assertEqual(
+            helpers.as_tuple("a string is not a container"),
+            ("a string is not a container",)
+        )
+
+        self.assertEqual(
+            helpers.as_tuple(["a", "list", "is", "a", "container"]),
+            ("a", "list", "is", "a", "container")
+        )
+
+
+class HelpersTest(unittest.TestCase):
+    def test_as_tuple_iter(self):
+        test_list = ['test_str']
+        as_tup = helpers.as_tuple(test_list)
+        self.assertTupleEqual(tuple(test_list), as_tup)
+
+    def test_as_tuple_no_iter(self):
+        test_str = 'test_str'
+        as_tup = helpers.as_tuple(test_str)
+        self.assertTupleEqual((test_str,), as_tup)
+
+    def test_is_in(self):
+        from airflow.utils import helpers
+        # `is_in` expects an object, and a list as input
+
+        test_dict = {'test': 1}
+        test_list = ['test', 1, dict()]
+        small_i = 3
+        big_i = 2 ** 31
+        test_str = 'test_str'
+        test_tup = ('test', 'tuple')
+
+        test_container = [test_dict, test_list, small_i, big_i, test_str, test_tup]
+
+        # Test that integers are referenced as the same object
+        self.assertTrue(helpers.is_in(small_i, test_container))
+        self.assertTrue(helpers.is_in(3, test_container))
+
+        # python caches small integers, so i is 3 will be True,
+        # but `big_i is 2 ** 31` is False.
+        self.assertTrue(helpers.is_in(big_i, test_container))
+        self.assertFalse(helpers.is_in(2 ** 31, test_container))
+
+        self.assertTrue(helpers.is_in(test_dict, test_container))
+        self.assertFalse(helpers.is_in({'test': 1}, test_container))
+
+        self.assertTrue(helpers.is_in(test_list, test_container))
+        self.assertFalse(helpers.is_in(['test', 1, dict()], test_container))
+
+        self.assertTrue(helpers.is_in(test_str, test_container))
+        self.assertTrue(helpers.is_in('test_str', test_container))
+        bad_str = 'test_'
+        bad_str += 'str'
+        self.assertFalse(helpers.is_in(bad_str, test_container))
+
+        self.assertTrue(helpers.is_in(test_tup, test_container))
+        self.assertFalse(helpers.is_in(('test', 'tuple'), test_container))
+        bad_tup = ('test', 'tuple', 'hello')
+        self.assertFalse(helpers.is_in(bad_tup[:2], test_container))
+
+    def test_is_container(self):
+        self.assertTrue(helpers.is_container(['test_list']))
+        self.assertFalse(helpers.is_container('test_str_not_iterable'))
+        # Pass an object that is not iter nor a string.
+        self.assertFalse(helpers.is_container(10))
+
+    def test_cross_downstream(self):
+        """Test if all dependencies between tasks are all set correctly."""
+        dag = DAG(dag_id="test_dag", start_date=datetime.now())
+        start_tasks = [DummyOperator(task_id="t{i}".format(i=i), dag=dag) for i in range(1, 4)]
+        end_tasks = [DummyOperator(task_id="t{i}".format(i=i), dag=dag) for i in range(4, 7)]
+        helpers.cross_downstream(from_tasks=start_tasks, to_tasks=end_tasks)
+
+        for start_task in start_tasks:
+            six.assertCountEqual(self, start_task.get_direct_relatives(upstream=False), end_tasks)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/utils/test_json.py b/tests/utils/test_json.py
index 91f53099dff3e..8e95716ee867b 100644
--- a/tests/utils/test_json.py
+++ b/tests/utils/test_json.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index d9623ee113604..72a5793f7e77b 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -169,14 +169,18 @@ def setUp(self):
         self.ti = TaskInstance(task=task, execution_date=DEFAULT_DATE)
 
     def test_python_formatting(self):
-        expected_filename = 'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' % DEFAULT_DATE.isoformat()
+        expected_filename = \
+            'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' \
+            % DEFAULT_DATE.isoformat()
 
         fth = FileTaskHandler('', '{dag_id}/{task_id}/{execution_date}/{try_number}.log')
         rendered_filename = fth._render_filename(self.ti, 42)
         self.assertEqual(expected_filename, rendered_filename)
 
     def test_jinja_rendering(self):
-        expected_filename = 'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' % DEFAULT_DATE.isoformat()
+        expected_filename = \
+            'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' \
+            % DEFAULT_DATE.isoformat()
 
         fth = FileTaskHandler('', '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log')
         rendered_filename = fth._render_filename(self.ti, 42)
diff --git a/tests/utils/test_logging_mixin.py b/tests/utils/test_logging_mixin.py
index 74da742891c3c..fa8c589a9e9fe 100644
--- a/tests/utils/test_logging_mixin.py
+++ b/tests/utils/test_logging_mixin.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -86,8 +86,8 @@ def test_write(self):
 
         self.assertEqual(log._buffer, msg)
 
-        log.write("\n")
-        logger.log.assert_called_once_with(1, msg + "\n")
+        log.write(" \n")
+        logger.log.assert_called_once_with(1, msg)
 
         self.assertEqual(log._buffer, "")
 
@@ -120,4 +120,3 @@ def test_encoding(self):
 
         log = StreamLogWriter(logger, 1)
         self.assertFalse(log.encoding)
-
diff --git a/tests/utils/test_module_loading.py b/tests/utils/test_module_loading.py
index 6c257df1bbcb4..ba1ebcade6a3b 100644
--- a/tests/utils/test_module_loading.py
+++ b/tests/utils/test_module_loading.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/test_net.py b/tests/utils/test_net.py
index f97f6c0821000..86540983be305 100644
--- a/tests/utils/test_net.py
+++ b/tests/utils/test_net.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/utils/test_operator_helpers.py b/tests/utils/test_operator_helpers.py
index 592b4560b62ac..a3586012f1108 100644
--- a/tests/utils/test_operator_helpers.py
+++ b/tests/utils/test_operator_helpers.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,9 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import unittest
 from datetime import datetime
+
 import mock
-import unittest
 
 from airflow.utils import operator_helpers
 
@@ -31,16 +32,18 @@ def setUp(self):
         self.dag_id = 'dag_id'
         self.task_id = 'task_id'
         self.execution_date = '2017-05-21T00:00:00'
+        self.dag_run_id = 'dag_run_id'
         self.context = {
-            'dag': mock.MagicMock(name='dag', dag_id=self.dag_id),
             'dag_run': mock.MagicMock(
                 name='dag_run',
+                run_id=self.dag_run_id,
                 execution_date=datetime.strptime(self.execution_date,
                                                  '%Y-%m-%dT%H:%M:%S'),
             ),
-            'task': mock.MagicMock(name='task', task_id=self.task_id),
             'task_instance': mock.MagicMock(
                 name='task_instance',
+                task_id=self.task_id,
+                dag_id=self.dag_id,
                 execution_date=datetime.strptime(self.execution_date,
                                                  '%Y-%m-%dT%H:%M:%S'),
             ),
@@ -53,10 +56,21 @@ def test_context_to_airflow_vars_all_context(self):
         self.assertDictEqual(
             operator_helpers.context_to_airflow_vars(self.context),
             {
-                'airflow.ctx.dag.dag_id': self.dag_id,
-                'airflow.ctx.dag_run.execution_date': self.execution_date,
-                'airflow.ctx.task.task_id': self.task_id,
-                'airflow.ctx.task_instance.execution_date': self.execution_date,
+                'airflow.ctx.dag_id': self.dag_id,
+                'airflow.ctx.execution_date': self.execution_date,
+                'airflow.ctx.task_id': self.task_id,
+                'airflow.ctx.dag_run_id': self.dag_run_id,
+            }
+        )
+
+        self.assertDictEqual(
+            operator_helpers.context_to_airflow_vars(self.context,
+                                                     in_env_var_format=True),
+            {
+                'AIRFLOW_CTX_DAG_ID': self.dag_id,
+                'AIRFLOW_CTX_EXECUTION_DATE': self.execution_date,
+                'AIRFLOW_CTX_TASK_ID': self.task_id,
+                'AIRFLOW_CTX_DAG_RUN_ID': self.dag_run_id,
             }
         )
 
diff --git a/tests/utils/test_sqlalchemy.py b/tests/utils/test_sqlalchemy.py
new file mode 100644
index 0000000000000..66f00f9427c6b
--- /dev/null
+++ b/tests/utils/test_sqlalchemy.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import datetime
+import unittest
+
+from airflow import settings
+from airflow.models import DAG
+from airflow.settings import Session
+from airflow.utils.state import State
+from airflow.utils.timezone import utcnow
+
+from sqlalchemy.exc import StatementError
+
+
+class TestSqlAlchemyUtils(unittest.TestCase):
+    def setUp(self):
+        session = Session()
+
+        # make sure NOT to run in UTC. Only postgres supports storing
+        # timezone information in the datetime field
+        if session.bind.dialect.name == "postgresql":
+            session.execute("SET timezone='Europe/Amsterdam'")
+
+        self.session = session
+
+    def test_utc_transformations(self):
+        """
+        Test whether what we are storing is what we are retrieving
+        for datetimes
+        """
+        dag_id = 'test_utc_transformations'
+        start_date = utcnow()
+        iso_date = start_date.isoformat()
+        execution_date = start_date + datetime.timedelta(hours=1, days=1)
+
+        dag = DAG(
+            dag_id=dag_id,
+            start_date=start_date,
+        )
+        dag.clear()
+
+        run = dag.create_dagrun(
+            run_id=iso_date,
+            state=State.NONE,
+            execution_date=execution_date,
+            start_date=start_date,
+            session=self.session,
+        )
+
+        self.assertEqual(execution_date, run.execution_date)
+        self.assertEqual(start_date, run.start_date)
+
+        self.assertEqual(execution_date.utcoffset().total_seconds(), 0.0)
+        self.assertEqual(start_date.utcoffset().total_seconds(), 0.0)
+
+        self.assertEqual(iso_date, run.run_id)
+        self.assertEqual(run.start_date.isoformat(), run.run_id)
+
+        dag.clear()
+
+    def test_process_bind_param_naive(self):
+        """
+        Check if naive datetimes are prevented from saving to the db
+        """
+        dag_id = 'test_process_bind_param_naive'
+
+        # naive
+        start_date = datetime.datetime.now()
+        dag = DAG(dag_id=dag_id, start_date=start_date)
+        dag.clear()
+
+        with self.assertRaises((ValueError, StatementError)):
+            dag.create_dagrun(
+                run_id=start_date.isoformat,
+                state=State.NONE,
+                execution_date=start_date,
+                start_date=start_date,
+                session=self.session
+            )
+        dag.clear()
+
+    def tearDown(self):
+        self.session.close()
+        settings.engine.dispose()
diff --git a/tests/utils/test_tests.py b/tests/utils/test_tests.py
new file mode 100644
index 0000000000000..8ef5b5ce22262
--- /dev/null
+++ b/tests/utils/test_tests.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from airflow.utils.tests import assertEqualIgnoreMultipleSpaces
+
+
+class UtilsTestsTest(unittest.TestCase):
+
+    def test_assertEqualIgnoreMultipleSpaces_raises(self):
+        str1 = 'w oo f'
+        str2 = 'meow'
+
+        self.assertRaises(AssertionError, lambda: assertEqualIgnoreMultipleSpaces(self, str1, str2))
+
+    def test_assertEqualIgnoreMultipleSpaces_passes(self):
+        str1 = 'w oo f'
+        str2 = """
+            w
+            oo    f
+        """
+
+        assertEqualIgnoreMultipleSpaces(self, str1, str2)
diff --git a/tests/utils/test_timezone.py b/tests/utils/test_timezone.py
index 0b44bddaaea28..e31895e2df521 100644
--- a/tests/utils/test_timezone.py
+++ b/tests/utils/test_timezone.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,8 +24,8 @@
 from airflow.utils import timezone
 
 CET = pendulum.timezone("Europe/Paris")
-EAT = pendulum.timezone('Africa/Nairobi')      # Africa/Nairobi
-ICT = pendulum.timezone('Asia/Bangkok')      # Asia/Bangkok
+EAT = pendulum.timezone('Africa/Nairobi')  # Africa/Nairobi
+ICT = pendulum.timezone('Asia/Bangkok')  # Asia/Bangkok
 UTC = timezone.utc
 
 
@@ -41,16 +41,16 @@ def test_is_naive(self):
     def test_utcnow(self):
         now = timezone.utcnow()
         self.assertTrue(timezone.is_localized(now))
-        self.assertEquals(now.replace(tzinfo=None), now.astimezone(UTC).replace(tzinfo=None))
+        self.assertEqual(now.replace(tzinfo=None), now.astimezone(UTC).replace(tzinfo=None))
 
     def test_convert_to_utc(self):
         naive = datetime.datetime(2011, 9, 1, 13, 20, 30)
         utc = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=UTC)
-        self.assertEquals(utc, timezone.convert_to_utc(naive))
+        self.assertEqual(utc, timezone.convert_to_utc(naive))
 
         eat = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT)
         utc = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
-        self.assertEquals(utc, timezone.convert_to_utc(eat))
+        self.assertEqual(utc, timezone.convert_to_utc(eat))
 
     def test_make_naive(self):
         self.assertEqual(
@@ -69,4 +69,3 @@ def test_make_aware(self):
             datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT))
         with self.assertRaises(ValueError):
             timezone.make_aware(datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), EAT)
-
diff --git a/tests/utils/test_trigger_rule.py b/tests/utils/test_trigger_rule.py
index be2a9f432b825..1167ac8c36edb 100644
--- a/tests/utils/test_trigger_rule.py
+++ b/tests/utils/test_trigger_rule.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -29,5 +29,7 @@ def test_valid_trigger_rules(self):
         self.assertTrue(TriggerRule.is_valid(TriggerRule.ALL_DONE))
         self.assertTrue(TriggerRule.is_valid(TriggerRule.ONE_SUCCESS))
         self.assertTrue(TriggerRule.is_valid(TriggerRule.ONE_FAILED))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.NONE_FAILED))
+        self.assertTrue(TriggerRule.is_valid(TriggerRule.NONE_SKIPPED))
         self.assertTrue(TriggerRule.is_valid(TriggerRule.DUMMY))
-        self.assertEqual(len(TriggerRule.all_triggers()), 6)
+        self.assertEqual(len(TriggerRule.all_triggers()), 8)
diff --git a/tests/utils/test_weight_rule.py b/tests/utils/test_weight_rule.py
index 27576e2e077aa..c862263387cc3 100644
--- a/tests/utils/test_weight_rule.py
+++ b/tests/utils/test_weight_rule.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/www/__init__.py b/tests/www/__init__.py
index f0f8b68337da6..114d189da14ab 100644
--- a/tests/www/__init__.py
+++ b/tests/www/__init__.py
@@ -7,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
diff --git a/tests/www/api/__init__.py b/tests/www/api/__init__.py
index f0f8b68337da6..114d189da14ab 100644
--- a/tests/www/api/__init__.py
+++ b/tests/www/api/__init__.py
@@ -7,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
diff --git a/tests/www/api/experimental/__init__.py b/tests/www/api/experimental/__init__.py
index f0f8b68337da6..114d189da14ab 100644
--- a/tests/www/api/experimental/__init__.py
+++ b/tests/www/api/experimental/__init__.py
@@ -7,13 +7,12 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
diff --git a/tests/www/api/experimental/test_dag_runs_endpoint.py b/tests/www/api/experimental/test_dag_runs_endpoint.py
new file mode 100644
index 0000000000000..614023bdeef18
--- /dev/null
+++ b/tests/www/api/experimental/test_dag_runs_endpoint.py
@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import unittest
+
+from airflow import configuration
+from airflow.api.common.experimental.trigger_dag import trigger_dag
+from airflow.models import DagRun
+from airflow.settings import Session
+from airflow.www import app as application
+
+
+class TestDagRunsEndpoint(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestDagRunsEndpoint, cls).setUpClass()
+        session = Session()
+        session.query(DagRun).delete()
+        session.commit()
+        session.close()
+
+    def setUp(self):
+        super(TestDagRunsEndpoint, self).setUp()
+        configuration.load_test_config()
+        app = application.create_app(testing=True)
+        self.app = app.test_client()
+
+    def tearDown(self):
+        session = Session()
+        session.query(DagRun).delete()
+        session.commit()
+        session.close()
+        super(TestDagRunsEndpoint, self).tearDown()
+
+    def test_get_dag_runs_success(self):
+        url_template = '/api/experimental/dags/{}/dag_runs'
+        dag_id = 'example_bash_operator'
+        # Create DagRun
+        dag_run = trigger_dag(dag_id=dag_id, run_id='test_get_dag_runs_success')
+
+        response = self.app.get(url_template.format(dag_id))
+        self.assertEqual(200, response.status_code)
+        data = json.loads(response.data.decode('utf-8'))
+
+        self.assertIsInstance(data, list)
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0]['dag_id'], dag_id)
+        self.assertEqual(data[0]['id'], dag_run.id)
+
+    def test_get_dag_runs_success_with_state_parameter(self):
+        url_template = '/api/experimental/dags/{}/dag_runs?state=running'
+        dag_id = 'example_bash_operator'
+        # Create DagRun
+        dag_run = trigger_dag(dag_id=dag_id, run_id='test_get_dag_runs_success')
+
+        response = self.app.get(url_template.format(dag_id))
+        self.assertEqual(200, response.status_code)
+        data = json.loads(response.data.decode('utf-8'))
+
+        self.assertIsInstance(data, list)
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0]['dag_id'], dag_id)
+        self.assertEqual(data[0]['id'], dag_run.id)
+
+    def test_get_dag_runs_success_with_capital_state_parameter(self):
+        url_template = '/api/experimental/dags/{}/dag_runs?state=RUNNING'
+        dag_id = 'example_bash_operator'
+        # Create DagRun
+        dag_run = trigger_dag(dag_id=dag_id, run_id='test_get_dag_runs_success')
+
+        response = self.app.get(url_template.format(dag_id))
+        self.assertEqual(200, response.status_code)
+        data = json.loads(response.data.decode('utf-8'))
+
+        self.assertIsInstance(data, list)
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0]['dag_id'], dag_id)
+        self.assertEqual(data[0]['id'], dag_run.id)
+
+    def test_get_dag_runs_success_with_state_no_result(self):
+        url_template = '/api/experimental/dags/{}/dag_runs?state=dummy'
+        dag_id = 'example_bash_operator'
+        # Create DagRun
+        trigger_dag(dag_id=dag_id, run_id='test_get_dag_runs_success')
+
+        response = self.app.get(url_template.format(dag_id))
+        self.assertEqual(200, response.status_code)
+        data = json.loads(response.data.decode('utf-8'))
+
+        self.assertIsInstance(data, list)
+        self.assertEqual(len(data), 0)
+
+    def test_get_dag_runs_invalid_dag_id(self):
+        url_template = '/api/experimental/dags/{}/dag_runs'
+        dag_id = 'DUMMY_DAG'
+
+        response = self.app.get(url_template.format(dag_id))
+        self.assertEqual(400, response.status_code)
+        data = json.loads(response.data.decode('utf-8'))
+
+        self.assertNotIsInstance(data, list)
+
+    def test_get_dag_runs_no_runs(self):
+        url_template = '/api/experimental/dags/{}/dag_runs'
+        dag_id = 'example_bash_operator'
+
+        response = self.app.get(url_template.format(dag_id))
+        self.assertEqual(200, response.status_code)
+        data = json.loads(response.data.decode('utf-8'))
+
+        self.assertIsInstance(data, list)
+        self.assertEqual(len(data), 0)
diff --git a/tests/www/api/experimental/test_endpoints.py b/tests/www/api/experimental/test_endpoints.py
index a93a19bc8947d..14f0e73cb32ed 100644
--- a/tests/www/api/experimental/test_endpoints.py
+++ b/tests/www/api/experimental/test_endpoints.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -77,6 +77,37 @@ def test_task_info(self):
         self.assertIn('error', response.data.decode('utf-8'))
         self.assertEqual(404, response.status_code)
 
+    def test_get_dag_code(self):
+        url_template = '/api/experimental/dags/{}/code'
+
+        response = self.app.get(
+            url_template.format('example_bash_operator')
+        )
+        self.assertIn('BashOperator(', response.data.decode('utf-8'))
+        self.assertEqual(200, response.status_code)
+
+        response = self.app.get(
+            url_template.format('xyz')
+        )
+        self.assertEqual(404, response.status_code)
+
+    def test_task_paused(self):
+        url_template = '/api/experimental/dags/{}/paused/{}'
+
+        response = self.app.get(
+            url_template.format('example_bash_operator', 'true')
+        )
+        self.assertIn('ok', response.data.decode('utf-8'))
+        self.assertEqual(200, response.status_code)
+
+        url_template = '/api/experimental/dags/{}/paused/{}'
+
+        response = self.app.get(
+            url_template.format('example_bash_operator', 'false')
+        )
+        self.assertIn('ok', response.data.decode('utf-8'))
+        self.assertEqual(200, response.status_code)
+
     def test_trigger_dag(self):
         url_template = '/api/experimental/dags/{}/dag_runs'
         response = self.app.post(
@@ -207,6 +238,49 @@ def test_task_instance_info(self):
         self.assertEqual(400, response.status_code)
         self.assertIn('error', response.data.decode('utf-8'))
 
+    def test_dagrun_status(self):
+        url_template = '/api/experimental/dags/{}/dag_runs/{}'
+        dag_id = 'example_bash_operator'
+        execution_date = utcnow().replace(microsecond=0)
+        datetime_string = quote_plus(execution_date.isoformat())
+        wrong_datetime_string = quote_plus(
+            datetime(1990, 1, 1, 1, 1, 1).isoformat()
+        )
+
+        # Create DagRun
+        trigger_dag(dag_id=dag_id,
+                    run_id='test_task_instance_info_run',
+                    execution_date=execution_date)
+
+        # Test Correct execution
+        response = self.app.get(
+            url_template.format(dag_id, datetime_string)
+        )
+        self.assertEqual(200, response.status_code)
+        self.assertIn('state', response.data.decode('utf-8'))
+        self.assertNotIn('error', response.data.decode('utf-8'))
+
+        # Test error for nonexistent dag
+        response = self.app.get(
+            url_template.format('does_not_exist_dag', datetime_string),
+        )
+        self.assertEqual(404, response.status_code)
+        self.assertIn('error', response.data.decode('utf-8'))
+
+        # Test error for nonexistent dag run (wrong execution_date)
+        response = self.app.get(
+            url_template.format(dag_id, wrong_datetime_string)
+        )
+        self.assertEqual(404, response.status_code)
+        self.assertIn('error', response.data.decode('utf-8'))
+
+        # Test error for bad datetime format
+        response = self.app.get(
+            url_template.format(dag_id, 'not_a_datetime')
+        )
+        self.assertEqual(400, response.status_code)
+        self.assertIn('error', response.data.decode('utf-8'))
+
 
 class TestPoolApiExperimental(unittest.TestCase):
 
diff --git a/tests/www/api/experimental/test_kerberos_endpoints.py b/tests/www/api/experimental/test_kerberos_endpoints.py
index 9179cdecc027a..b305e19b2137a 100644
--- a/tests/www/api/experimental/test_kerberos_endpoints.py
+++ b/tests/www/api/experimental/test_kerberos_endpoints.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,7 +20,6 @@
 import json
 import mock
 import os
-import socket
 import unittest
 
 from datetime import datetime
@@ -38,14 +37,14 @@ def setUp(self):
         configuration.load_test_config()
         try:
             configuration.conf.add_section("api")
-        except:
+        except Exception:
             pass
         configuration.conf.set("api",
                                "auth_backend",
                                "airflow.api.auth.backend.kerberos_auth")
         try:
             configuration.conf.add_section("kerberos")
-        except:
+        except Exception:
             pass
         configuration.conf.set("kerberos",
                                "keytab",
@@ -65,7 +64,7 @@ def test_trigger_dag(self):
 
             response.url = 'http://{}'.format(get_hostname())
 
-            class Request():
+            class Request:
                 headers = {}
 
             response.request = Request()
diff --git a/tests/www/api/experimental/test_password_endpoints.py b/tests/www/api/experimental/test_password_endpoints.py
index 8131ff52f3dd5..20a0608a77880 100644
--- a/tests/www/api/experimental/test_password_endpoints.py
+++ b/tests/www/api/experimental/test_password_endpoints.py
@@ -17,11 +17,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import json
 import unittest
 
-from datetime import datetime
-
 from backports.configparser import DuplicateSectionError
 
 from airflow import models
diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py
index d69041a368af4..22f1e790eb012 100644
--- a/tests/www/test_utils.py
+++ b/tests/www/test_utils.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,17 +17,33 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import unittest
-from xml.dom import minidom
+import functools
+
+from bs4 import BeautifulSoup
+import mock
+import six
+from six.moves.urllib.parse import parse_qs
+
+from airflow.www import app as application
 
 from airflow.www import utils
 
+if six.PY2:
+    # Need `assertRegex` back-ported from unittest2
+    import unittest2 as unittest
+else:
+    import unittest
+
 
 class UtilsTest(unittest.TestCase):
 
     def setUp(self):
         super(UtilsTest, self).setUp()
 
+    def test_empty_variable_should_not_be_hidden(self):
+        self.assertFalse(utils.should_hide_value_for_key(""))
+        self.assertFalse(utils.should_hide_value_for_key(None))
+
     def test_normal_variable_should_not_be_hidden(self):
         self.assertFalse(utils.should_hide_value_for_key("key"))
 
@@ -40,39 +56,43 @@ def test_sensitive_variable_should_be_hidden_ic(self):
     def check_generate_pages_html(self, current_page, total_pages,
                                   window=7, check_middle=False):
         extra_links = 4  # first, prev, next, last
-        html_str = utils.generate_pages(current_page, total_pages)
+        search = "'>\"/><img src=x onerror=alert(1)>"
+        html_str = utils.generate_pages(current_page, total_pages,
+                                        search=search)
+
+        self.assertNotIn(search, html_str,
+                         "The raw search string shouldn't appear in the output")
+        self.assertIn('search=%27%3E%22%2F%3E%3Cimg+src%3Dx+onerror%3Dalert%281%29%3E',
+                      html_str)
+
+        self.assertTrue(
+            callable(html_str.__html__),
+            "Should return something that is HTML-escaping aware"
+        )
 
-        # dom parser has issues with special &laquo; and &raquo;
-        html_str = html_str.replace('&laquo;', '')
-        html_str = html_str.replace('&raquo;', '')
-        dom = minidom.parseString(html_str)
+        dom = BeautifulSoup(html_str, 'html.parser')
         self.assertIsNotNone(dom)
 
-        ulist = dom.getElementsByTagName('ul')[0]
-        ulist_items = ulist.getElementsByTagName('li')
+        ulist = dom.ul
+        ulist_items = ulist.find_all('li')
         self.assertEqual(min(window, total_pages) + extra_links, len(ulist_items))
 
-        def get_text(nodelist):
-            rc = []
-            for node in nodelist:
-                if node.nodeType == node.TEXT_NODE:
-                    rc.append(node.data)
-            return ''.join(rc)
-
         page_items = ulist_items[2:-2]
         mid = int(len(page_items) / 2)
         for i, item in enumerate(page_items):
-            a_node = item.getElementsByTagName('a')[0]
-            href_link = a_node.getAttribute('href')
-            node_text = get_text(a_node.childNodes)
+            a_node = item.a
+            href_link = a_node['href']
+            node_text = a_node.string
             if node_text == str(current_page + 1):
                 if check_middle:
                     self.assertEqual(mid, i)
-                self.assertEqual('javascript:void(0)', a_node.getAttribute('href'))
-                self.assertIn('active', item.getAttribute('class'))
+                self.assertEqual('javascript:void(0)', href_link)
+                self.assertIn('active', item['class'])
             else:
-                link_str = '?page=' + str(int(node_text) - 1)
-                self.assertEqual(link_str, href_link)
+                self.assertRegex(href_link, r'^\?', 'Link is page-relative')
+                query = parse_qs(href_link[1:])
+                self.assertListEqual(query['page'], [str(int(node_text) - 1)])
+                self.assertListEqual(query['search'], [search])
 
     def test_generate_pager_current_start(self):
         self.check_generate_pages_html(current_page=0,
@@ -89,7 +109,7 @@ def test_generate_pager_current_end(self):
 
     def test_params_no_values(self):
         """Should return an empty string if no params are passed"""
-        self.assertEquals('', utils.get_params())
+        self.assertEqual('', utils.get_params())
 
     def test_params_search(self):
         self.assertEqual('search=bash_',
@@ -104,10 +124,172 @@ def test_params_showPaused_false(self):
         self.assertEqual('showPaused=False',
                          utils.get_params(showPaused=False))
 
+    def test_params_none_and_zero(self):
+        qs = utils.get_params(a=0, b=None)
+        # The order won't be consistent, but that doesn't affect behaviour of a browser
+        pairs = list(sorted(qs.split('&')))
+        self.assertListEqual(['a=0', 'b='], pairs)
+
     def test_params_all(self):
-        """Should return params string ordered by param key"""
-        self.assertEqual('page=3&search=bash_&showPaused=False',
-                         utils.get_params(showPaused=False, page=3, search='bash_'))
+        query = utils.get_params(showPaused=False, page=3, search='bash_')
+        self.assertEqual(
+            {'page': ['3'],
+             'search': ['bash_'],
+             'showPaused': ['False']},
+            parse_qs(query)
+        )
+
+    def test_params_escape(self):
+        self.assertEqual('search=%27%3E%22%2F%3E%3Cimg+src%3Dx+onerror%3Dalert%281%29%3E',
+                         utils.get_params(search="'>\"/><img src=x onerror=alert(1)>"))
+
+    # flask_login is loaded by calling flask_login.utils._get_user.
+    @mock.patch("flask_login.utils._get_user")
+    @mock.patch("airflow.settings.Session")
+    def test_action_logging_with_login_user(self, mocked_session, mocked_get_user):
+        fake_username = 'someone'
+        mocked_current_user = mock.MagicMock()
+        mocked_get_user.return_value = mocked_current_user
+        mocked_current_user.user.username = fake_username
+        mocked_session_instance = mock.MagicMock()
+        mocked_session.return_value = mocked_session_instance
+
+        app = application.create_app(testing=True)
+        # Patching here to avoid errors in applicant.create_app
+        with mock.patch("airflow.models.Log") as mocked_log:
+            with app.test_request_context():
+                @utils.action_logging
+                def some_func():
+                    pass
+
+                some_func()
+                mocked_log.assert_called_once()
+                (args, kwargs) = mocked_log.call_args_list[0]
+                self.assertEqual('some_func', kwargs['event'])
+                self.assertEqual(fake_username, kwargs['owner'])
+                mocked_session_instance.add.assert_called_once()
+
+    @mock.patch("flask_login.utils._get_user")
+    @mock.patch("airflow.settings.Session")
+    def test_action_logging_with_invalid_user(self, mocked_session, mocked_get_user):
+        anonymous_username = 'anonymous'
+
+        # When the user returned by flask login_manager._load_user
+        # is invalid.
+        mocked_current_user = mock.MagicMock()
+        mocked_get_user.return_value = mocked_current_user
+        mocked_current_user.user = None
+        mocked_session_instance = mock.MagicMock()
+        mocked_session.return_value = mocked_session_instance
+
+        app = application.create_app(testing=True)
+        # Patching here to avoid errors in applicant.create_app
+        with mock.patch("airflow.models.Log") as mocked_log:
+            with app.test_request_context():
+                @utils.action_logging
+                def some_func():
+                    pass
+
+                some_func()
+                mocked_log.assert_called_once()
+                (args, kwargs) = mocked_log.call_args_list[0]
+                self.assertEqual('some_func', kwargs['event'])
+                self.assertEqual(anonymous_username, kwargs['owner'])
+                mocked_session_instance.add.assert_called_once()
+
+    # flask_login.current_user would be AnonymousUserMixin
+    # when there's no user_id in the flask session.
+    @mock.patch("airflow.settings.Session")
+    def test_action_logging_with_anonymous_user(self, mocked_session):
+        anonymous_username = 'anonymous'
+
+        mocked_session_instance = mock.MagicMock()
+        mocked_session.return_value = mocked_session_instance
+
+        app = application.create_app(testing=True)
+        # Patching here to avoid errors in applicant.create_app
+        with mock.patch("airflow.models.Log") as mocked_log:
+            with app.test_request_context():
+                @utils.action_logging
+                def some_func():
+                    pass
+
+                some_func()
+                mocked_log.assert_called_once()
+                (args, kwargs) = mocked_log.call_args_list[0]
+                self.assertEqual('some_func', kwargs['event'])
+                self.assertEqual(anonymous_username, kwargs['owner'])
+                mocked_session_instance.add.assert_called_once()
+
+    def test_open_maybe_zipped_normal_file(self):
+        with mock.patch(
+                'io.open', mock.mock_open(read_data="data")) as mock_file:
+            utils.open_maybe_zipped('/path/to/some/file.txt')
+            mock_file.assert_called_with('/path/to/some/file.txt', mode='r')
+
+    def test_open_maybe_zipped_normal_file_with_zip_in_name(self):
+        path = '/path/to/fakearchive.zip.other/file.txt'
+        with mock.patch(
+                'io.open', mock.mock_open(read_data="data")) as mock_file:
+            utils.open_maybe_zipped(path)
+            mock_file.assert_called_with(path, mode='r')
+
+    @mock.patch("zipfile.is_zipfile")
+    @mock.patch("zipfile.ZipFile")
+    def test_open_maybe_zipped_archive(self, mocked_ZipFile, mocked_is_zipfile):
+        mocked_is_zipfile.return_value = True
+        instance = mocked_ZipFile.return_value
+        instance.open.return_value = mock.mock_open(read_data="data")
+
+        utils.open_maybe_zipped('/path/to/archive.zip/deep/path/to/file.txt')
+
+        assert mocked_is_zipfile.call_count == 1
+        (args, kwargs) = mocked_is_zipfile.call_args_list[0]
+        self.assertEqual('/path/to/archive.zip', args[0])
+
+        assert mocked_ZipFile.call_count == 1
+        (args, kwargs) = mocked_ZipFile.call_args_list[0]
+        self.assertEqual('/path/to/archive.zip', args[0])
+
+        assert instance.open.call_count == 1
+        (args, kwargs) = instance.open.call_args_list[0]
+        self.assertEqual('deep/path/to/file.txt', args[0])
+
+    def test_get_python_source_from_method(self):
+        class AMockClass(object):
+            def a_method(self):
+                """ A method """
+                pass
+
+        mocked_class = AMockClass()
+
+        result = utils.get_python_source(mocked_class.a_method)
+        self.assertIn('A method', result)
+
+    def test_get_python_source_from_class(self):
+        class AMockClass(object):
+            def __call__(self):
+                """ A __call__ method """
+                pass
+
+        mocked_class = AMockClass()
+
+        result = utils.get_python_source(mocked_class)
+        self.assertIn('A __call__ method', result)
+
+    def test_get_python_source_from_partial_func(self):
+        def a_function(arg_x, arg_y):
+            """ A function with two args """
+            pass
+
+        partial_function = functools.partial(a_function, arg_x=1)
+
+        result = utils.get_python_source(partial_function)
+        self.assertIn('A function with two args', result)
+
+    def test_get_python_source_from_none(self):
+        result = utils.get_python_source(None)
+        self.assertIn('No source code available', result)
 
 
 if __name__ == '__main__':
diff --git a/tests/www/test_validators.py b/tests/www/test_validators.py
index 0fef4e3db202d..e6242632f2cd1 100644
--- a/tests/www/test_validators.py
+++ b/tests/www/test_validators.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -92,6 +92,5 @@ def test_validation_raises_custom_message(self):
         )
 
 
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index f59470ea3de11..29f3d06bae920 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -20,6 +20,7 @@
 import io
 import copy
 import logging.config
+import mock
 import os
 import shutil
 import tempfile
@@ -29,8 +30,11 @@
 
 from urllib.parse import quote_plus
 from werkzeug.test import Client
+from werkzeug.wrappers import BaseResponse
 
-from airflow import models, configuration, settings
+
+import airflow
+from airflow import models, configuration
 from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
 from airflow.models import DAG, DagRun, TaskInstance
 from airflow.operators.dummy_operator import DummyOperator
@@ -66,7 +70,7 @@ def setUp(self):
         self.chart = {
             'label': 'chart',
             'owner': 'airflow',
-            'conn_id': 'airflow_ci',
+            'conn_id': 'airflow_db',
         }
 
     def tearDown(self):
@@ -155,8 +159,6 @@ def test_can_handle_error_on_decrypt(self):
         response = self.app.get('/admin/variable', follow_redirects=True)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(self.session.query(models.Variable).count(), 1)
-        self.assertIn('<span class="label label-danger">Invalid</span>',
-                      response.data.decode('utf-8'))
 
     def test_xss_prevention(self):
         xss = "/admin/airflow/variables/asdf<img%20src=''%20onerror='alert(1);'>"
@@ -385,6 +387,28 @@ def test_get_file_task_log(self):
         self.assertIn('Log by attempts',
                       response.data.decode('utf-8'))
 
+    def test_get_logs_with_metadata_as_download_file(self):
+        url_template = "/admin/airflow/get_logs_with_metadata?dag_id={}&" \
+                       "task_id={}&execution_date={}&" \
+                       "try_number={}&metadata={}&format=file"
+        try_number = 1
+        url = url_template.format(self.DAG_ID,
+                                  self.TASK_ID,
+                                  quote_plus(self.DEFAULT_DATE.isoformat()),
+                                  try_number,
+                                  json.dumps({}))
+        response = self.app.get(url)
+        expected_filename = '{}/{}/{}/{}.log'.format(self.DAG_ID,
+                                                     self.TASK_ID,
+                                                     self.DEFAULT_DATE.isoformat(),
+                                                     try_number)
+
+        content_disposition = response.headers.get('Content-Disposition')
+        self.assertTrue(content_disposition.startswith('attachment'))
+        self.assertTrue(expected_filename in content_disposition)
+        self.assertEqual(200, response.status_code)
+        self.assertIn('Log for testing.', response.data.decode('utf-8'))
+
     def test_get_logs_with_metadata(self):
         url_template = "/admin/airflow/get_logs_with_metadata?dag_id={}&" \
                        "task_id={}&execution_date={}&" \
@@ -450,6 +474,28 @@ def tearDownClass(cls):
         session.close()
         super(TestVarImportView, cls).tearDownClass()
 
+    def test_import_variable_fail(self):
+        with mock.patch('airflow.models.Variable.set') as set_mock:
+            set_mock.side_effect = UnicodeEncodeError
+            content = '{"fail_key": "fail_val"}'
+
+            try:
+                # python 3+
+                bytes_content = io.BytesIO(bytes(content, encoding='utf-8'))
+            except TypeError:
+                # python 2.7
+                bytes_content = io.BytesIO(bytes(content))
+            response = self.app.post(
+                self.IMPORT_ENDPOINT,
+                data={'file': (bytes_content, 'test.json')},
+                follow_redirects=True
+            )
+            self.assertEqual(response.status_code, 200)
+            session = Session()
+            db_dict = {x.key: x.get_val() for x in session.query(models.Variable).all()}
+            session.close()
+            self.assertNotIn('fail_key', db_dict)
+
     def test_import_variables(self):
         content = ('{"str_key": "str_value", "int_key": 60,'
                    '"list_key": [1, 2], "dict_key": {"k_a": 2, "k_b": 3}}')
@@ -465,43 +511,50 @@ def test_import_variables(self):
             follow_redirects=True
         )
         self.assertEqual(response.status_code, 200)
-        body = response.data.decode('utf-8')
-        self.assertIn('str_key', body)
-        self.assertIn('int_key', body)
-        self.assertIn('list_key', body)
-        self.assertIn('dict_key', body)
-        self.assertIn('str_value', body)
-        self.assertIn('60', body)
-        self.assertIn('[1, 2]', body)
-        # As dicts are not ordered, we may get any of the following cases.
-        case_a_dict = '{&#34;k_a&#34;: 2, &#34;k_b&#34;: 3}'
-        case_b_dict = '{&#34;k_b&#34;: 3, &#34;k_a&#34;: 2}'
+        session = Session()
+        # Extract values from Variable
+        db_dict = {x.key: x.get_val() for x in session.query(models.Variable).all()}
+        session.close()
+        self.assertIn('str_key', db_dict)
+        self.assertIn('int_key', db_dict)
+        self.assertIn('list_key', db_dict)
+        self.assertIn('dict_key', db_dict)
+        self.assertEqual('str_value', db_dict['str_key'])
+        self.assertEqual('60', db_dict['int_key'])
+        self.assertEqual('[1, 2]', db_dict['list_key'])
+
+        case_a_dict = '{"k_a": 2, "k_b": 3}'
+        case_b_dict = '{"k_b": 3, "k_a": 2}'
         try:
-            self.assertIn(case_a_dict, body)
+            self.assertEqual(case_a_dict, db_dict['dict_key'])
         except AssertionError:
-            self.assertIn(case_b_dict, body)
+            self.assertEqual(case_b_dict, db_dict['dict_key'])
 
 
 class TestMountPoint(unittest.TestCase):
-    def setUp(self):
-        super(TestMountPoint, self).setUp()
+    @classmethod
+    def setUpClass(cls):
         configuration.load_test_config()
         configuration.conf.set("webserver", "base_url", "http://localhost:8080/test")
-        config = dict()
-        config['WTF_CSRF_METHODS'] = []
         # Clear cached app to remount base_url forcefully
         application.app = None
-        app = application.cached_app(config=config, testing=True)
-        self.client = Client(app)
+        app = application.cached_app(config={'WTF_CSRF_ENABLED': False}, testing=True)
+        cls.client = Client(app, BaseResponse)
+
+    @classmethod
+    def tearDownClass(cls):
+        application.app = None
+        application.appbuilder = None
 
     def test_mount(self):
-        response, _, _ = self.client.get('/', follow_redirects=True)
-        txt = b''.join(response)
-        self.assertEqual(b"Apache Airflow is not at this location", txt)
+        # Test an endpoint that doesn't need auth!
+        resp = self.client.get('/test/health')
+        self.assertEqual(resp.status_code, 200)
+        self.assertIn(b"healthy", resp.data)
 
-        response, _, _ = self.client.get('/test', follow_redirects=True)
-        resp_html = b''.join(response)
-        self.assertIn(b"DAGs", resp_html)
+    def test_not_found(self):
+        resp = self.client.get('/', follow_redirects=True)
+        self.assertEqual(resp.status_code, 404)
 
 
 class ViewWithDateTimeAndNumRunsAndDagRunsFormTester:
@@ -746,5 +799,145 @@ def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(sel
         self.tester.test_with_base_date_and_num_runs_and_execution_date_within()
 
 
+class TestTaskInstanceView(unittest.TestCase):
+    TI_ENDPOINT = '/admin/taskinstance/?flt2_execution_date_greater_than={}'
+
+    def setUp(self):
+        super(TestTaskInstanceView, self).setUp()
+        configuration.load_test_config()
+        app = application.create_app(testing=True)
+        app.config['WTF_CSRF_METHODS'] = []
+        self.app = app.test_client()
+
+    def test_start_date_filter(self):
+        resp = self.app.get(self.TI_ENDPOINT.format('2018-10-09+22:44:31'))
+        # We aren't checking the logic of the date filter itself (that is built
+        # in to flask-admin) but simply that our UTC conversion was run - i.e. it
+        # doesn't blow up!
+        self.assertEqual(resp.status_code, 200)
+
+
+class TestDeleteDag(unittest.TestCase):
+
+    def setUp(self):
+        conf.load_test_config()
+        app = application.create_app(testing=True)
+        app.config['WTF_CSRF_METHODS'] = []
+        self.app = app.test_client()
+
+    def test_delete_dag_button_normal(self):
+        resp = self.app.get('/', follow_redirects=True)
+        self.assertIn('/delete?dag_id=example_bash_operator', resp.data.decode('utf-8'))
+        self.assertIn("return confirmDeleteDag(this, 'example_bash_operator')", resp.data.decode('utf-8'))
+
+    def test_delete_dag_button_for_dag_on_scheduler_only(self):
+        # Test for JIRA AIRFLOW-3233 (PR 4069):
+        # The delete-dag URL should be generated correctly for DAGs
+        # that exist on the scheduler (DB) but not the webserver DagBag
+
+        test_dag_id = "non_existent_dag"
+
+        session = Session()
+        DM = models.DagModel
+        session.query(DM).filter(DM.dag_id == 'example_bash_operator').update({'dag_id': test_dag_id})
+        session.commit()
+
+        resp = self.app.get('/', follow_redirects=True)
+        self.assertIn('/delete?dag_id={}'.format(test_dag_id), resp.data.decode('utf-8'))
+        self.assertIn("return confirmDeleteDag(this, '{}')".format(test_dag_id), resp.data.decode('utf-8'))
+
+        session.query(DM).filter(DM.dag_id == test_dag_id).update({'dag_id': 'example_bash_operator'})
+        session.commit()
+
+
+class TestTriggerDag(unittest.TestCase):
+
+    def setUp(self):
+        conf.load_test_config()
+        app = application.create_app(testing=True)
+        app.config['WTF_CSRF_METHODS'] = []
+        self.app = app.test_client()
+        self.session = Session()
+        models.DagBag().get_dag("example_bash_operator").sync_to_db()
+
+    def test_trigger_dag_button_normal_exist(self):
+        resp = self.app.get('/', follow_redirects=True)
+        self.assertIn('/trigger?dag_id=example_bash_operator', resp.data.decode('utf-8'))
+        self.assertIn("return confirmDeleteDag(this, 'example_bash_operator')", resp.data.decode('utf-8'))
+
+    def test_trigger_dag_button(self):
+
+        test_dag_id = "example_bash_operator"
+
+        DR = models.DagRun
+        self.session.query(DR).delete()
+        self.session.commit()
+
+        self.app.post('/admin/airflow/trigger?dag_id={}'.format(test_dag_id))
+
+        run = self.session.query(DR).filter(DR.dag_id == test_dag_id).first()
+        self.assertIsNotNone(run)
+        self.assertIn("manual__", run.run_id)
+
+
+class HelpersTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.app = application.create_app(testing=True)
+
+        airflow.load_login()
+        # Delay this import until here
+        import airflow.www.views as views
+        cls.views = views
+
+    def test_state_token(self):
+        # It's shouldn't possible to set these odd values anymore, but lets
+        # ensure they are escaped!
+        html = str(self.views.state_token('<script>alert(1)</script>'))
+
+        self.assertIn(
+            '&lt;script&gt;alert(1)&lt;/script&gt;',
+            html,
+        )
+        self.assertNotIn(
+            '<script>alert(1)</script>',
+            html,
+        )
+
+    def test_task_instance_link(self):
+        mock_task = mock.Mock(dag_id='<a&1>', task_id='<b2>', execution_date=datetime(2017, 10, 12))
+        with self.app.test_request_context():
+            html = str(self.views.task_instance_link(
+                v=None, c=None, m=mock_task, p=None
+            ))
+
+        self.assertIn('%3Ca%261%3E', html)
+        self.assertIn('%3Cb2%3E', html)
+        self.assertNotIn('<a&1>', html)
+        self.assertNotIn('<b2>', html)
+
+    def test_dag_link(self):
+        mock_dag = mock.Mock(dag_id='<a&1>', execution_date=datetime(2017, 10, 12))
+        with self.app.test_request_context():
+            html = str(self.views.dag_link(
+                v=None, c=None, m=mock_dag, p=None
+            ))
+
+        self.assertIn('%3Ca%261%3E', html)
+        self.assertNotIn('<a&1>', html)
+
+    def test_dag_run_link(self):
+        mock_run = mock.Mock(dag_id='<a&1>', run_id='<b2>', execution_date=datetime(2017, 10, 12))
+        with self.app.test_request_context():
+            html = str(self.views.dag_run_link(
+                v=None, c=None, m=mock_run, p=None
+            ))
+
+        self.assertIn('%3Ca%261%3E', html)
+        self.assertIn('%3Cb2%3E', html)
+        self.assertNotIn('<a&1>', html)
+        self.assertNotIn('<b2>', html)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/www_rbac/__init__.py b/tests/www_rbac/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/www_rbac/__init__.py
+++ b/tests/www_rbac/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/www_rbac/api/__init__.py b/tests/www_rbac/api/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/www_rbac/api/__init__.py
+++ b/tests/www_rbac/api/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/www_rbac/api/experimental/__init__.py b/tests/www_rbac/api/experimental/__init__.py
index 4067cc78ee9a2..114d189da14ab 100644
--- a/tests/www_rbac/api/experimental/__init__.py
+++ b/tests/www_rbac/api/experimental/__init__.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/www_rbac/api/experimental/test_endpoints.py b/tests/www_rbac/api/experimental/test_endpoints.py
index a84d9cfdb44ae..6f902cc4fd34c 100644
--- a/tests/www_rbac/api/experimental/test_endpoints.py
+++ b/tests/www_rbac/api/experimental/test_endpoints.py
@@ -22,7 +22,9 @@
 import unittest
 from urllib.parse import quote_plus
 
-from airflow import configuration
+
+from airflow import configuration as conf
+from airflow import settings
 from airflow.api.common.experimental.trigger_dag import trigger_dag
 from airflow.models import DagBag, DagRun, Pool, TaskInstance
 from airflow.settings import Session
@@ -30,7 +32,20 @@
 from airflow.www_rbac import app as application
 
 
-class TestApiExperimental(unittest.TestCase):
+class TestBase(unittest.TestCase):
+    def setUp(self):
+        conf.load_test_config()
+        self.app, self.appbuilder = application.create_app(session=Session, testing=True)
+        self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'
+        self.app.config['SECRET_KEY'] = 'secret_key'
+        self.app.config['CSRF_ENABLED'] = False
+        self.app.config['WTF_CSRF_ENABLED'] = False
+        self.client = self.app.test_client()
+        settings.configure_orm()
+        self.session = Session
+
+
+class TestApiExperimental(TestBase):
 
     @classmethod
     def setUpClass(cls):
@@ -43,9 +58,6 @@ def setUpClass(cls):
 
     def setUp(self):
         super(TestApiExperimental, self).setUp()
-        configuration.load_test_config()
-        app, _ = application.create_app(testing=True)
-        self.app = app.test_client()
 
     def tearDown(self):
         session = Session()
@@ -58,29 +70,43 @@ def tearDown(self):
     def test_task_info(self):
         url_template = '/api/experimental/dags/{}/tasks/{}'
 
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('example_bash_operator', 'runme_0')
         )
         self.assertIn('"email"', response.data.decode('utf-8'))
         self.assertNotIn('error', response.data.decode('utf-8'))
         self.assertEqual(200, response.status_code)
 
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('example_bash_operator', 'DNE')
         )
         self.assertIn('error', response.data.decode('utf-8'))
         self.assertEqual(404, response.status_code)
 
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('DNE', 'DNE')
         )
         self.assertIn('error', response.data.decode('utf-8'))
         self.assertEqual(404, response.status_code)
 
+    def test_get_dag_code(self):
+        url_template = '/api/experimental/dags/{}/code'
+
+        response = self.client.get(
+            url_template.format('example_bash_operator')
+        )
+        self.assertIn('BashOperator(', response.data.decode('utf-8'))
+        self.assertEqual(200, response.status_code)
+
+        response = self.client.get(
+            url_template.format('xyz')
+        )
+        self.assertEqual(404, response.status_code)
+
     def test_task_paused(self):
         url_template = '/api/experimental/dags/{}/paused/{}'
 
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('example_bash_operator', 'true')
         )
         self.assertIn('ok', response.data.decode('utf-8'))
@@ -88,7 +114,7 @@ def test_task_paused(self):
 
         url_template = '/api/experimental/dags/{}/paused/{}'
 
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('example_bash_operator', 'false')
         )
         self.assertIn('ok', response.data.decode('utf-8'))
@@ -96,7 +122,7 @@ def test_task_paused(self):
 
     def test_trigger_dag(self):
         url_template = '/api/experimental/dags/{}/dag_runs'
-        response = self.app.post(
+        response = self.client.post(
             url_template.format('example_bash_operator'),
             data=json.dumps({'run_id': 'my_run' + utcnow().isoformat()}),
             content_type="application/json"
@@ -104,7 +130,7 @@ def test_trigger_dag(self):
 
         self.assertEqual(200, response.status_code)
 
-        response = self.app.post(
+        response = self.client.post(
             url_template.format('does_not_exist_dag'),
             data=json.dumps({}),
             content_type="application/json"
@@ -122,7 +148,7 @@ def test_trigger_dag_for_date(self):
         datetime_string = execution_date.isoformat()
 
         # Test Correct execution
-        response = self.app.post(
+        response = self.client.post(
             url_template.format(dag_id),
             data=json.dumps({'execution_date': datetime_string}),
             content_type="application/json"
@@ -137,7 +163,7 @@ def test_trigger_dag_for_date(self):
                         .format(execution_date))
 
         # Test error for nonexistent dag
-        response = self.app.post(
+        response = self.client.post(
             url_template.format('does_not_exist_dag'),
             data=json.dumps({'execution_date': execution_date.isoformat()}),
             content_type="application/json"
@@ -145,7 +171,7 @@ def test_trigger_dag_for_date(self):
         self.assertEqual(404, response.status_code)
 
         # Test error for bad datetime format
-        response = self.app.post(
+        response = self.client.post(
             url_template.format(dag_id),
             data=json.dumps({'execution_date': 'not_a_datetime'}),
             content_type="application/json"
@@ -168,7 +194,7 @@ def test_task_instance_info(self):
                     execution_date=execution_date)
 
         # Test Correct execution
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, datetime_string, task_id)
         )
         self.assertEqual(200, response.status_code)
@@ -176,7 +202,7 @@ def test_task_instance_info(self):
         self.assertNotIn('error', response.data.decode('utf-8'))
 
         # Test error for nonexistent dag
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('does_not_exist_dag', datetime_string,
                                 task_id),
         )
@@ -184,21 +210,21 @@ def test_task_instance_info(self):
         self.assertIn('error', response.data.decode('utf-8'))
 
         # Test error for nonexistent task
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, datetime_string, 'does_not_exist_task')
         )
         self.assertEqual(404, response.status_code)
         self.assertIn('error', response.data.decode('utf-8'))
 
         # Test error for nonexistent dag run (wrong execution_date)
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, wrong_datetime_string, task_id)
         )
         self.assertEqual(404, response.status_code)
         self.assertIn('error', response.data.decode('utf-8'))
 
         # Test error for bad datetime format
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, 'not_a_datetime', task_id)
         )
         self.assertEqual(400, response.status_code)
@@ -219,7 +245,7 @@ def test_dagrun_status(self):
                     execution_date=execution_date)
 
         # Test Correct execution
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, datetime_string)
         )
         self.assertEqual(200, response.status_code)
@@ -227,27 +253,28 @@ def test_dagrun_status(self):
         self.assertNotIn('error', response.data.decode('utf-8'))
 
         # Test error for nonexistent dag
-        response = self.app.get(
+        response = self.client.get(
             url_template.format('does_not_exist_dag', datetime_string),
         )
         self.assertEqual(404, response.status_code)
         self.assertIn('error', response.data.decode('utf-8'))
 
         # Test error for nonexistent dag run (wrong execution_date)
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, wrong_datetime_string)
         )
         self.assertEqual(404, response.status_code)
         self.assertIn('error', response.data.decode('utf-8'))
 
         # Test error for bad datetime format
-        response = self.app.get(
+        response = self.client.get(
             url_template.format(dag_id, 'not_a_datetime')
         )
         self.assertEqual(400, response.status_code)
         self.assertIn('error', response.data.decode('utf-8'))
 
-class TestPoolApiExperimental(unittest.TestCase):
+
+class TestPoolApiExperimental(TestBase):
 
     @classmethod
     def setUpClass(cls):
@@ -259,10 +286,7 @@ def setUpClass(cls):
 
     def setUp(self):
         super(TestPoolApiExperimental, self).setUp()
-        configuration.load_test_config()
-        app, _ = application.create_app(testing=True)
-        self.app = app.test_client()
-        self.session = Session()
+
         self.pools = []
         for i in range(2):
             name = 'experimental_%s' % (i + 1)
@@ -283,12 +307,12 @@ def tearDown(self):
         super(TestPoolApiExperimental, self).tearDown()
 
     def _get_pool_count(self):
-        response = self.app.get('/api/experimental/pools')
+        response = self.client.get('/api/experimental/pools')
         self.assertEqual(response.status_code, 200)
         return len(json.loads(response.data.decode('utf-8')))
 
     def test_get_pool(self):
-        response = self.app.get(
+        response = self.client.get(
             '/api/experimental/pools/{}'.format(self.pool.pool),
         )
         self.assertEqual(response.status_code, 200)
@@ -296,13 +320,13 @@ def test_get_pool(self):
                          self.pool.to_json())
 
     def test_get_pool_non_existing(self):
-        response = self.app.get('/api/experimental/pools/foo')
+        response = self.client.get('/api/experimental/pools/foo')
         self.assertEqual(response.status_code, 404)
         self.assertEqual(json.loads(response.data.decode('utf-8'))['error'],
                          "Pool 'foo' doesn't exist")
 
     def test_get_pools(self):
-        response = self.app.get('/api/experimental/pools')
+        response = self.client.get('/api/experimental/pools')
         self.assertEqual(response.status_code, 200)
         pools = json.loads(response.data.decode('utf-8'))
         self.assertEqual(len(pools), 2)
@@ -310,7 +334,7 @@ def test_get_pools(self):
             self.assertDictEqual(pool, self.pools[i].to_json())
 
     def test_create_pool(self):
-        response = self.app.post(
+        response = self.client.post(
             '/api/experimental/pools',
             data=json.dumps({
                 'name': 'foo',
@@ -328,7 +352,7 @@ def test_create_pool(self):
 
     def test_create_pool_with_bad_name(self):
         for name in ('', '    '):
-            response = self.app.post(
+            response = self.client.post(
                 '/api/experimental/pools',
                 data=json.dumps({
                     'name': name,
@@ -345,7 +369,7 @@ def test_create_pool_with_bad_name(self):
         self.assertEqual(self._get_pool_count(), 2)
 
     def test_delete_pool(self):
-        response = self.app.delete(
+        response = self.client.delete(
             '/api/experimental/pools/{}'.format(self.pool.pool),
         )
         self.assertEqual(response.status_code, 200)
@@ -354,7 +378,7 @@ def test_delete_pool(self):
         self.assertEqual(self._get_pool_count(), 1)
 
     def test_delete_pool_non_existing(self):
-        response = self.app.delete(
+        response = self.client.delete(
             '/api/experimental/pools/foo',
         )
         self.assertEqual(response.status_code, 404)
diff --git a/tests/www_rbac/api/experimental/test_kerberos_endpoints.py b/tests/www_rbac/api/experimental/test_kerberos_endpoints.py
index 69a8117868f3e..788613b504ba4 100644
--- a/tests/www_rbac/api/experimental/test_kerberos_endpoints.py
+++ b/tests/www_rbac/api/experimental/test_kerberos_endpoints.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -64,7 +64,7 @@ def test_trigger_dag(self):
 
             response.url = 'http://{}'.format(socket.getfqdn())
 
-            class Request():
+            class Request(object):
                 headers = {}
 
             response.request = Request()
diff --git a/tests/www_rbac/test_security.py b/tests/www_rbac/test_security.py
index ae1a5c6fa3f6d..d140f5eb787dd 100644
--- a/tests/www_rbac/test_security.py
+++ b/tests/www_rbac/test_security.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,15 +21,18 @@
 
 import unittest
 import logging
+import mock
 
 from flask import Flask
 from flask_appbuilder import AppBuilder, SQLA, Model, has_access, expose
 from flask_appbuilder.models.sqla.interface import SQLAInterface
+from flask_appbuilder.security.sqla import models as sqla_models
 from flask_appbuilder.views import ModelView, BaseView
 
 from sqlalchemy import Column, Integer, String, Date, Float
 
-from airflow.www_rbac.security import init_role
+from airflow.www_rbac.security import AirflowSecurityManager, DAG_PERMS
+
 
 logging.basicConfig(format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')
 logging.getLogger().setLevel(logging.DEBUG)
@@ -70,11 +73,13 @@ def setUp(self):
         self.app.config['CSRF_ENABLED'] = False
         self.app.config['WTF_CSRF_ENABLED'] = False
         self.db = SQLA(self.app)
-        self.appbuilder = AppBuilder(self.app, self.db.session)
+        self.appbuilder = AppBuilder(self.app,
+                                     self.db.session,
+                                     security_manager_class=AirflowSecurityManager)
+        self.security_manager = self.appbuilder.sm
         self.appbuilder.add_view(SomeBaseView, "SomeBaseView", category="BaseViews")
         self.appbuilder.add_view(SomeModelView, "SomeModelView", category="ModelViews")
-
-        role_admin = self.appbuilder.sm.find_role('Admin')
+        role_admin = self.security_manager.find_role('Admin')
         self.user = self.appbuilder.sm.add_user('admin', 'admin', 'user', 'admin@fab.org',
                                                 role_admin, 'general')
         log.debug("Complete setup!")
@@ -89,7 +94,7 @@ def test_init_role_baseview(self):
         role_name = 'MyRole1'
         role_perms = ['can_some_action']
         role_vms = ['SomeBaseView']
-        init_role(self.appbuilder.sm, role_name, role_vms, role_perms)
+        self.security_manager.init_role(role_name, role_vms, role_perms)
         role = self.appbuilder.sm.find_role(role_name)
         self.assertIsNotNone(role)
         self.assertEqual(len(role_perms), len(role.permissions))
@@ -98,26 +103,102 @@ def test_init_role_modelview(self):
         role_name = 'MyRole2'
         role_perms = ['can_list', 'can_show', 'can_add', 'can_edit', 'can_delete']
         role_vms = ['SomeModelView']
-        init_role(self.appbuilder.sm, role_name, role_vms, role_perms)
+        self.security_manager.init_role(role_name, role_vms, role_perms)
         role = self.appbuilder.sm.find_role(role_name)
         self.assertIsNotNone(role)
         self.assertEqual(len(role_perms), len(role.permissions))
 
-    def test_invalid_perms(self):
-        role_name = 'MyRole3'
-        role_perms = ['can_foo']
-        role_vms = ['SomeBaseView']
-        with self.assertRaises(Exception) as context:
-            init_role(self.appbuilder.sm, role_name, role_vms, role_perms)
-        self.assertEqual("The following permissions are not valid: ['can_foo']",
-                         str(context.exception))
+    def test_update_and_verify_permission_role(self):
+        role_name = 'Test_Role'
+        self.security_manager.init_role(role_name, [], [])
+        role = self.security_manager.find_role(role_name)
+
+        perm = self.security_manager.\
+            find_permission_view_menu('can_edit', 'RoleModelView')
+        self.security_manager.add_permission_role(role, perm)
+        role_perms_len = len(role.permissions)
 
-    def test_invalid_vms(self):
-        role_name = 'MyRole4'
+        self.security_manager.init_role(role_name, [], [])
+        new_role_perms_len = len(role.permissions)
+
+        self.assertEqual(role_perms_len, new_role_perms_len)
+
+    def test_get_user_roles(self):
+        user = mock.MagicMock()
+        user.is_anonymous = False
+        roles = self.appbuilder.sm.find_role('Admin')
+        user.roles = roles
+        self.assertEqual(self.security_manager.get_user_roles(user), roles)
+
+    @mock.patch('airflow.www_rbac.security.AirflowSecurityManager.get_user_roles')
+    def test_get_all_permissions_views(self, mock_get_user_roles):
+        role_name = 'MyRole1'
         role_perms = ['can_some_action']
-        role_vms = ['NonExistentBaseView']
-        with self.assertRaises(Exception) as context:
-            init_role(self.appbuilder.sm, role_name, role_vms, role_perms)
-        self.assertEqual("The following view menus are not valid: "
-                         "['NonExistentBaseView']",
-                         str(context.exception))
+        role_vms = ['SomeBaseView']
+        self.security_manager.init_role(role_name, role_vms, role_perms)
+        role = self.security_manager.find_role(role_name)
+
+        mock_get_user_roles.return_value = [role]
+        self.assertEqual(self.security_manager
+                         .get_all_permissions_views(),
+                         {('can_some_action', 'SomeBaseView')})
+
+        mock_get_user_roles.return_value = []
+        self.assertEqual(len(self.security_manager
+                             .get_all_permissions_views()), 0)
+
+    @mock.patch('airflow.www_rbac.security.AirflowSecurityManager'
+                '.get_all_permissions_views')
+    @mock.patch('airflow.www_rbac.security.AirflowSecurityManager'
+                '.get_user_roles')
+    def test_get_accessible_dag_ids(self, mock_get_user_roles,
+                                    mock_get_all_permissions_views):
+        user = mock.MagicMock()
+        role_name = 'MyRole1'
+        role_perms = ['can_dag_read']
+        role_vms = ['dag_id']
+        self.security_manager.init_role(role_name, role_vms, role_perms)
+        role = self.security_manager.find_role(role_name)
+        user.roles = [role]
+        user.is_anonymous = False
+        mock_get_all_permissions_views.return_value = {('can_dag_read', 'dag_id')}
+
+        mock_get_user_roles.return_value = [role]
+        self.assertEqual(self.security_manager
+                         .get_accessible_dag_ids(user), set(['dag_id']))
+
+    @mock.patch('airflow.www_rbac.security.AirflowSecurityManager._has_view_access')
+    def test_has_access(self, mock_has_view_access):
+        user = mock.MagicMock()
+        user.is_anonymous = False
+        mock_has_view_access.return_value = True
+        self.assertTrue(self.security_manager.has_access('perm', 'view', user))
+
+    def test_sync_perm_for_dag(self):
+        test_dag_id = 'TEST_DAG'
+        self.security_manager.sync_perm_for_dag(test_dag_id)
+        for dag_perm in DAG_PERMS:
+            self.assertIsNotNone(self.security_manager.
+                                 find_permission_view_menu(dag_perm, test_dag_id))
+
+    @mock.patch('airflow.www_rbac.security.AirflowSecurityManager._has_perm')
+    @mock.patch('airflow.www_rbac.security.AirflowSecurityManager._has_role')
+    def test_has_all_dag_access(self, mock_has_role, mock_has_perm):
+        mock_has_role.return_value = True
+        self.assertTrue(self.security_manager.has_all_dags_access())
+
+        mock_has_role.return_value = False
+        mock_has_perm.return_value = False
+        self.assertFalse(self.security_manager.has_all_dags_access())
+
+        mock_has_perm.return_value = True
+        self.assertTrue(self.security_manager.has_all_dags_access())
+
+    def test_no_additional_dag_permission_views_created(self):
+        ab_perm_view_role = sqla_models.assoc_permissionview_role
+
+        self.security_manager.sync_roles()
+        num_pv_before = self.db.session().query(ab_perm_view_role).count()
+        self.security_manager.sync_roles()
+        num_pv_after = self.db.session().query(ab_perm_view_role).count()
+        self.assertEqual(num_pv_before, num_pv_after)
diff --git a/tests/www_rbac/test_utils.py b/tests/www_rbac/test_utils.py
index 05114881ddecf..9c75ad39e84fb 100644
--- a/tests/www_rbac/test_utils.py
+++ b/tests/www_rbac/test_utils.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,16 +17,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import unittest
-from xml.dom import minidom
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+import mock
+import six
+from six.moves.urllib.parse import parse_qs
 
 from airflow.www_rbac import utils
 
+if six.PY2:
+    # Need `assertRegex` back-ported from unittest2
+    import unittest2 as unittest
+else:
+    import unittest
 
-class UtilsTest(unittest.TestCase):
 
-    def setUp(self):
-        super(UtilsTest, self).setUp()
+class UtilsTest(unittest.TestCase):
+    def test_empty_variable_should_not_be_hidden(self):
+        self.assertFalse(utils.should_hide_value_for_key(""))
+        self.assertFalse(utils.should_hide_value_for_key(None))
 
     def test_normal_variable_should_not_be_hidden(self):
         self.assertFalse(utils.should_hide_value_for_key("key"))
@@ -40,39 +50,43 @@ def test_sensitive_variable_should_be_hidden_ic(self):
     def check_generate_pages_html(self, current_page, total_pages,
                                   window=7, check_middle=False):
         extra_links = 4  # first, prev, next, last
-        html_str = utils.generate_pages(current_page, total_pages)
+        search = "'>\"/><img src=x onerror=alert(1)>"
+        html_str = utils.generate_pages(current_page, total_pages,
+                                        search=search)
 
-        # dom parser has issues with special &laquo; and &raquo;
-        html_str = html_str.replace('&laquo;', '')
-        html_str = html_str.replace('&raquo;', '')
-        dom = minidom.parseString(html_str)
+        self.assertNotIn(search, html_str,
+                         "The raw search string shouldn't appear in the output")
+        self.assertIn('search=%27%3E%22%2F%3E%3Cimg+src%3Dx+onerror%3Dalert%281%29%3E',
+                      html_str)
+
+        self.assertTrue(
+            callable(html_str.__html__),
+            "Should return something that is HTML-escaping aware"
+        )
+
+        dom = BeautifulSoup(html_str, 'html.parser')
         self.assertIsNotNone(dom)
 
-        ulist = dom.getElementsByTagName('ul')[0]
-        ulist_items = ulist.getElementsByTagName('li')
+        ulist = dom.ul
+        ulist_items = ulist.find_all('li')
         self.assertEqual(min(window, total_pages) + extra_links, len(ulist_items))
 
-        def get_text(nodelist):
-            rc = []
-            for node in nodelist:
-                if node.nodeType == node.TEXT_NODE:
-                    rc.append(node.data)
-            return ''.join(rc)
-
         page_items = ulist_items[2:-2]
         mid = int(len(page_items) / 2)
         for i, item in enumerate(page_items):
-            a_node = item.getElementsByTagName('a')[0]
-            href_link = a_node.getAttribute('href')
-            node_text = get_text(a_node.childNodes)
+            a_node = item.a
+            href_link = a_node['href']
+            node_text = a_node.string
             if node_text == str(current_page + 1):
                 if check_middle:
                     self.assertEqual(mid, i)
-                self.assertEqual('javascript:void(0)', a_node.getAttribute('href'))
-                self.assertIn('active', item.getAttribute('class'))
+                self.assertEqual('javascript:void(0)', href_link)
+                self.assertIn('active', item['class'])
             else:
-                link_str = '?page=' + str(int(node_text) - 1)
-                self.assertEqual(link_str, href_link)
+                self.assertRegex(href_link, r'^\?', 'Link is page-relative')
+                query = parse_qs(href_link[1:])
+                self.assertListEqual(query['page'], [str(int(node_text) - 1)])
+                self.assertListEqual(query['search'], [search])
 
     def test_generate_pager_current_start(self):
         self.check_generate_pages_html(current_page=0,
@@ -89,7 +103,7 @@ def test_generate_pager_current_end(self):
 
     def test_params_no_values(self):
         """Should return an empty string if no params are passed"""
-        self.assertEquals('', utils.get_params())
+        self.assertEqual('', utils.get_params())
 
     def test_params_search(self):
         self.assertEqual('search=bash_',
@@ -104,10 +118,112 @@ def test_params_showPaused_false(self):
         self.assertEqual('showPaused=False',
                          utils.get_params(showPaused=False))
 
+    def test_params_none_and_zero(self):
+        qs = utils.get_params(a=0, b=None)
+        # The order won't be consistent, but that doesn't affect behaviour of a browser
+        pairs = list(sorted(qs.split('&')))
+        self.assertListEqual(['a=0', 'b='], pairs)
+
     def test_params_all(self):
-        """Should return params string ordered by param key"""
-        self.assertEqual('page=3&search=bash_&showPaused=False',
-                         utils.get_params(showPaused=False, page=3, search='bash_'))
+        query = utils.get_params(showPaused=False, page=3, search='bash_')
+        self.assertEqual(
+            {'page': ['3'],
+             'search': ['bash_'],
+             'showPaused': ['False']},
+            parse_qs(query)
+        )
+
+    def test_params_escape(self):
+        self.assertEqual('search=%27%3E%22%2F%3E%3Cimg+src%3Dx+onerror%3Dalert%281%29%3E',
+                         utils.get_params(search="'>\"/><img src=x onerror=alert(1)>"))
+
+    def test_open_maybe_zipped_normal_file(self):
+        with mock.patch(
+                'io.open', mock.mock_open(read_data="data")) as mock_file:
+            utils.open_maybe_zipped('/path/to/some/file.txt')
+            mock_file.assert_called_with('/path/to/some/file.txt', mode='r')
+
+    def test_open_maybe_zipped_normal_file_with_zip_in_name(self):
+        path = '/path/to/fakearchive.zip.other/file.txt'
+        with mock.patch(
+                'io.open', mock.mock_open(read_data="data")) as mock_file:
+            utils.open_maybe_zipped(path)
+            mock_file.assert_called_with(path, mode='r')
+
+    @mock.patch("zipfile.is_zipfile")
+    @mock.patch("zipfile.ZipFile")
+    def test_open_maybe_zipped_archive(self, mocked_ZipFile, mocked_is_zipfile):
+        mocked_is_zipfile.return_value = True
+        instance = mocked_ZipFile.return_value
+        instance.open.return_value = mock.mock_open(read_data="data")
+
+        utils.open_maybe_zipped('/path/to/archive.zip/deep/path/to/file.txt')
+
+        mocked_is_zipfile.assert_called_once()
+        (args, kwargs) = mocked_is_zipfile.call_args_list[0]
+        self.assertEqual('/path/to/archive.zip', args[0])
+
+        mocked_ZipFile.assert_called_once()
+        (args, kwargs) = mocked_ZipFile.call_args_list[0]
+        self.assertEqual('/path/to/archive.zip', args[0])
+
+        instance.open.assert_called_once()
+        (args, kwargs) = instance.open.call_args_list[0]
+        self.assertEqual('deep/path/to/file.txt', args[0])
+
+    def test_state_token(self):
+        # It's shouldn't possible to set these odd values anymore, but lets
+        # ensure they are escaped!
+        html = str(utils.state_token('<script>alert(1)</script>'))
+
+        self.assertIn(
+            '&lt;script&gt;alert(1)&lt;/script&gt;',
+            html,
+        )
+        self.assertNotIn(
+            '<script>alert(1)</script>',
+            html,
+        )
+
+    def test_task_instance_link(self):
+
+        from airflow.www_rbac.app import cached_appbuilder
+        with cached_appbuilder(testing=True).app.test_request_context():
+            html = str(utils.task_instance_link({
+                'dag_id': '<a&1>',
+                'task_id': '<b2>',
+                'execution_date': datetime.now()
+            }))
+
+        self.assertIn('%3Ca%261%3E', html)
+        self.assertIn('%3Cb2%3E', html)
+        self.assertNotIn('<a&1>', html)
+        self.assertNotIn('<b2>', html)
+
+    def test_dag_link(self):
+        from airflow.www_rbac.app import cached_appbuilder
+        with cached_appbuilder(testing=True).app.test_request_context():
+            html = str(utils.dag_link({
+                'dag_id': '<a&1>',
+                'execution_date': datetime.now()
+            }))
+
+        self.assertIn('%3Ca%261%3E', html)
+        self.assertNotIn('<a&1>', html)
+
+    def test_dag_run_link(self):
+        from airflow.www_rbac.app import cached_appbuilder
+        with cached_appbuilder(testing=True).app.test_request_context():
+            html = str(utils.dag_run_link({
+                'dag_id': '<a&1>',
+                'run_id': '<b2>',
+                'execution_date': datetime.now()
+            }))
+
+        self.assertIn('%3Ca%261%3E', html)
+        self.assertIn('%3Cb2%3E', html)
+        self.assertNotIn('<a&1>', html)
+        self.assertNotIn('<b2>', html)
 
 
 if __name__ == '__main__':
diff --git a/tests/www_rbac/test_validators.py b/tests/www_rbac/test_validators.py
index d7709c4ee76c2..b50e88de11c5a 100644
--- a/tests/www_rbac/test_validators.py
+++ b/tests/www_rbac/test_validators.py
@@ -7,9 +7,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/tests/www_rbac/test_views.py b/tests/www_rbac/test_views.py
index 71dcab0d073a6..f11a9df0dede1 100644
--- a/tests/www_rbac/test_views.py
+++ b/tests/www_rbac/test_views.py
@@ -27,38 +27,48 @@
 import tempfile
 import unittest
 import urllib
+from datetime import timedelta
+from urllib.parse import quote_plus
 
+import mock
+from flask import url_for
 from flask._compat import PY2
-from flask_appbuilder.security.sqla.models import User as ab_user
-from urllib.parse import quote_plus
+from parameterized import parameterized
 from werkzeug.test import Client
+from werkzeug.wrappers import BaseResponse
 
 from airflow import configuration as conf
-from airflow import models
+from airflow import models, settings
 from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+from airflow.jobs import BaseJob
 from airflow.models import DAG, DagRun, TaskInstance
+from airflow.models.connection import Connection
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.settings import Session
-from airflow.utils import timezone
+from airflow.utils import dates, timezone
+from airflow.utils.db import create_session
 from airflow.utils.state import State
 from airflow.utils.timezone import datetime
 from airflow.www_rbac import app as application
 
 
 class TestBase(unittest.TestCase):
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         conf.load_test_config()
-        self.app, self.appbuilder = application.create_app(testing=True)
-        self.app.config['WTF_CSRF_ENABLED'] = False
+        cls.app, cls.appbuilder = application.create_app(session=Session, testing=True)
+        cls.app.config['WTF_CSRF_ENABLED'] = False
+        settings.configure_orm()
+        cls.session = Session
+
+    def setUp(self):
         self.client = self.app.test_client()
-        self.session = Session()
         self.login()
 
     def login(self):
-        sm_session = self.appbuilder.sm.get_session()
-        self.user = sm_session.query(ab_user).first()
-        if not self.user:
-            role_admin = self.appbuilder.sm.find_role('Admin')
+        role_admin = self.appbuilder.sm.find_role('Admin')
+        tester = self.appbuilder.sm.find_user(username='test')
+        if not tester:
             self.appbuilder.sm.add_user(
                 username='test',
                 first_name='test',
@@ -88,6 +98,15 @@ def check_content_in_response(self, text, resp, resp_code=200):
         else:
             self.assertIn(text, resp_html)
 
+    def check_content_not_in_response(self, text, resp, resp_code=200):
+        resp_html = resp.data.decode('utf-8')
+        self.assertEqual(resp_code, resp.status_code)
+        if isinstance(text, list):
+            for kw in text:
+                self.assertNotIn(kw, resp_html)
+        else:
+            self.assertNotIn(text, resp_html)
+
     def percent_encode(self, obj):
         if PY2:
             return urllib.quote_plus(str(obj))
@@ -108,7 +127,7 @@ def setUp(self):
         }
 
     def tearDown(self):
-        self.clear_table(models.Connection)
+        self.clear_table(Connection)
         super(TestConnectionModelView, self).tearDown()
 
     def test_create_connection(self):
@@ -137,10 +156,6 @@ def test_can_handle_error_on_decrypt(self):
         resp = self.client.post('/variable/add',
                                 data=self.variable,
                                 follow_redirects=True)
-        self.assertEqual(resp.status_code, 200)
-        v = self.session.query(models.Variable).first()
-        self.assertEqual(v.key, 'test_key')
-        self.assertEqual(v.val, 'text_val')
 
         # update the variable with a wrong value, given that is encrypted
         Var = models.Variable
@@ -168,7 +183,31 @@ def test_xss_prevention(self):
         self.assertNotIn("<img src='' onerror='alert(1);'>",
                          resp.data.decode("utf-8"))
 
-    def test_import_variables(self):
+    def test_import_variables_no_file(self):
+        resp = self.client.post('/variable/varimport',
+                                follow_redirects=True)
+        self.check_content_in_response('Missing file or syntax error.', resp)
+
+    def test_import_variables_failed(self):
+        content = '{"str_key": "str_value"}'
+
+        with mock.patch('airflow.models.Variable.set') as set_mock:
+            set_mock.side_effect = UnicodeEncodeError
+            self.assertEqual(self.session.query(models.Variable).count(), 0)
+
+            try:
+                # python 3+
+                bytes_content = io.BytesIO(bytes(content, encoding='utf-8'))
+            except TypeError:
+                # python 2.7
+                bytes_content = io.BytesIO(bytes(content))
+
+            resp = self.client.post('/variable/varimport',
+                                    data={'file': (bytes_content, 'test.json')},
+                                    follow_redirects=True)
+            self.check_content_in_response('1 variable(s) failed to be updated.', resp)
+
+    def test_import_variables_success(self):
         self.assertEqual(self.session.query(models.Variable).count(), 0)
 
         content = ('{"str_key": "str_value", "int_key": 60,'
@@ -220,34 +259,54 @@ def test_create_pool_with_empty_name(self):
                                 follow_redirects=True)
         self.check_content_in_response('This field is required.', resp)
 
+    def test_odd_name(self):
+        self.pool['pool'] = 'test-pool<script></script>'
+        self.session.add(models.Pool(**self.pool))
+        self.session.commit()
+        resp = self.client.get('/pool/list/')
+        self.check_content_in_response('test-pool&lt;script&gt;', resp)
+        self.check_content_not_in_response('test-pool<script>', resp)
+
 
 class TestMountPoint(unittest.TestCase):
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         application.app = None
-        super(TestMountPoint, self).setUp()
+        application.appbuilder = None
         conf.load_test_config()
-        conf.set("webserver", "base_url", "http://localhost:8080/test")
-        config = dict()
-        config['WTF_CSRF_METHODS'] = []
-        app = application.cached_app(config=config, testing=True)
-        self.client = Client(app)
+        conf.set("webserver", "base_url", "http://localhost/test")
+        app = application.cached_app(config={'WTF_CSRF_ENABLED': False}, session=Session, testing=True)
+        cls.client = Client(app, BaseResponse)
+
+    @classmethod
+    def tearDownClass(cls):
+        application.app = None
+        application.appbuilder = None
 
     def test_mount(self):
-        resp, _, _ = self.client.get('/', follow_redirects=True)
-        txt = b''.join(resp)
-        self.assertEqual(b"Apache Airflow is not at this location", txt)
+        # Test an endpoint that doesn't need auth!
+        resp = self.client.get('/test/health')
+        self.assertEqual(resp.status_code, 200)
+        self.assertIn(b"healthy", resp.data)
+
+    def test_not_found(self):
+        resp = self.client.get('/', follow_redirects=True)
+        self.assertEqual(resp.status_code, 404)
 
-        resp, _, _ = self.client.get('/test/home', follow_redirects=True)
-        resp_html = b''.join(resp)
-        self.assertIn(b"DAGs", resp_html)
+    def test_index(self):
+        resp = self.client.get('/test/')
+        self.assertEqual(resp.status_code, 302)
+        self.assertEqual(resp.headers['Location'], 'http://localhost/test/home')
 
 
 class TestAirflowBaseViews(TestBase):
-    default_date = timezone.datetime(2018, 3, 1)
-    run_id = "test_{}".format(models.DagRun.id_for_date(default_date))
+    EXAMPLE_DAG_DEFAULT_DATE = dates.days_ago(2)
+    run_id = "test_{}".format(models.DagRun.id_for_date(EXAMPLE_DAG_DEFAULT_DATE))
 
     def setUp(self):
         super(TestAirflowBaseViews, self).setUp()
+        self.logout()
+        self.login()
         self.cleanup_dagruns()
         self.prepare_dagruns()
 
@@ -266,24 +325,27 @@ def cleanup_dagruns(self):
     def prepare_dagruns(self):
         dagbag = models.DagBag(include_examples=True)
         self.bash_dag = dagbag.dags['example_bash_operator']
+        self.bash_dag.sync_to_db()
         self.sub_dag = dagbag.dags['example_subdag_operator']
+        self.sub_dag.sync_to_db()
         self.xcom_dag = dagbag.dags['example_xcom']
+        self.xcom_dag.sync_to_db()
 
         self.bash_dagrun = self.bash_dag.create_dagrun(
             run_id=self.run_id,
-            execution_date=self.default_date,
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
             start_date=timezone.utcnow(),
             state=State.RUNNING)
 
         self.sub_dagrun = self.sub_dag.create_dagrun(
             run_id=self.run_id,
-            execution_date=self.default_date,
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
             start_date=timezone.utcnow(),
             state=State.RUNNING)
 
         self.xcom_dagrun = self.xcom_dag.create_dagrun(
             run_id=self.run_id,
-            execution_date=self.default_date,
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
             start_date=timezone.utcnow(),
             state=State.RUNNING)
 
@@ -292,8 +354,66 @@ def test_index(self):
         self.check_content_in_response('DAGs', resp)
 
     def test_health(self):
-        resp = self.client.get('health')
-        self.check_content_in_response('The server is healthy!', resp)
+
+        # case-1: healthy scheduler status
+        last_scheduler_heartbeat_for_testing_1 = timezone.utcnow()
+        self.session.add(BaseJob(job_type='SchedulerJob',
+                                 state='running',
+                                 latest_heartbeat=last_scheduler_heartbeat_for_testing_1))
+        self.session.commit()
+
+        resp_json = json.loads(self.client.get('health', follow_redirects=True).data.decode('utf-8'))
+
+        self.assertEqual('healthy', resp_json['metadatabase']['status'])
+        self.assertEqual('healthy', resp_json['scheduler']['status'])
+        self.assertEqual(str(last_scheduler_heartbeat_for_testing_1),
+                         resp_json['scheduler']['latest_scheduler_heartbeat'])
+
+        self.session.query(BaseJob).\
+            filter(BaseJob.job_type == 'SchedulerJob',
+                   BaseJob.state == 'running',
+                   BaseJob.latest_heartbeat == last_scheduler_heartbeat_for_testing_1).\
+            delete()
+        self.session.commit()
+
+        # case-2: unhealthy scheduler status - scenario 1 (SchedulerJob is running too slowly)
+        last_scheduler_heartbeat_for_testing_2 = timezone.utcnow() - timedelta(minutes=1)
+        (self.session
+             .query(BaseJob)
+             .filter(BaseJob.job_type == 'SchedulerJob')
+             .update({'latest_heartbeat': last_scheduler_heartbeat_for_testing_2 - timedelta(seconds=1)}))
+        self.session.add(BaseJob(job_type='SchedulerJob',
+                                 state='running',
+                                 latest_heartbeat=last_scheduler_heartbeat_for_testing_2))
+        self.session.commit()
+
+        resp_json = json.loads(self.client.get('health', follow_redirects=True).data.decode('utf-8'))
+
+        self.assertEqual('healthy', resp_json['metadatabase']['status'])
+        self.assertEqual('unhealthy', resp_json['scheduler']['status'])
+        self.assertEqual(str(last_scheduler_heartbeat_for_testing_2),
+                         resp_json['scheduler']['latest_scheduler_heartbeat'])
+
+        self.session.query(BaseJob).\
+            filter(BaseJob.job_type == 'SchedulerJob',
+                   BaseJob.state == 'running',
+                   BaseJob.latest_heartbeat == last_scheduler_heartbeat_for_testing_2).\
+            delete()
+        self.session.commit()
+
+        # case-3: unhealthy scheduler status - scenario 2 (no running SchedulerJob)
+        self.session.query(BaseJob).\
+            filter(BaseJob.job_type == 'SchedulerJob',
+                   BaseJob.state == 'running').\
+            delete()
+        self.session.commit()
+
+        resp_json = json.loads(self.client.get('health', follow_redirects=True).data.decode('utf-8'))
+
+        self.assertEqual('healthy', resp_json['metadatabase']['status'])
+        self.assertEqual('unhealthy', resp_json['scheduler']['status'])
+        self.assertEqual('None',
+                         resp_json['scheduler']['latest_scheduler_heartbeat'])
 
     def test_home(self):
         resp = self.client.get('home', follow_redirects=True)
@@ -301,19 +421,28 @@ def test_home(self):
 
     def test_task(self):
         url = ('task?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
-               .format(self.percent_encode(self.default_date)))
+               .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
         resp = self.client.get(url, follow_redirects=True)
         self.check_content_in_response('Task Instance Details', resp)
 
     def test_xcom(self):
         url = ('xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
-               .format(self.percent_encode(self.default_date)))
+               .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
         resp = self.client.get(url, follow_redirects=True)
         self.check_content_in_response('XCom', resp)
 
+    def test_edit_dagrun_page(self):
+        resp = self.client.get('dagmodel/edit/example_bash_operator', follow_redirects=False)
+        self.assertEqual(resp.status_code, 200)
+
+    def test_edit_dagrun_url(self):
+        with self.app.test_request_context():
+            url = url_for('DagModelView.edit', pk='example_bash_operator')
+            self.assertEqual(url, '/dagmodel/edit/example_bash_operator')
+
     def test_rendered(self):
         url = ('rendered?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
-               .format(self.percent_encode(self.default_date)))
+               .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
         resp = self.client.get(url, follow_redirects=True)
         self.check_content_in_response('Rendered Template', resp)
 
@@ -355,13 +484,18 @@ def test_duration(self):
         resp = self.client.get(url, follow_redirects=True)
         self.check_content_in_response('example_bash_operator', resp)
 
+    def test_duration_missing(self):
+        url = 'duration?days=30&dag_id=missing_dag'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('seems to be missing', resp)
+
     def test_tries(self):
         url = 'tries?days=30&dag_id=example_bash_operator'
         resp = self.client.get(url, follow_redirects=True)
         self.check_content_in_response('example_bash_operator', resp)
 
     def test_landing_times(self):
-        url = 'landing_times?days=30&dag_id=test_example_bash_operator'
+        url = 'landing_times?days=30&dag_id=example_bash_operator'
         resp = self.client.get(url, follow_redirects=True)
         self.check_content_in_response('example_bash_operator', resp)
 
@@ -380,35 +514,100 @@ def test_paused(self):
         resp = self.client.post(url, follow_redirects=True)
         self.check_content_in_response('OK', resp)
 
-    def test_success(self):
+    def test_failed(self):
+        form = dict(
+            task_id="run_this_last",
+            dag_id="example_bash_operator",
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
+            upstream="false",
+            downstream="false",
+            future="false",
+            past="false",
+        )
+        resp = self.client.post("failed", data=form)
+        self.check_content_in_response('Wait a minute', resp)
 
-        url = ('success?task_id=run_this_last&dag_id=example_bash_operator&'
-               'execution_date={}&upstream=false&downstream=false&future=false&past=false'
-               .format(self.percent_encode(self.default_date)))
-        resp = self.client.get(url)
+    def test_success(self):
+        form = dict(
+            task_id="run_this_last",
+            dag_id="example_bash_operator",
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
+            upstream="false",
+            downstream="false",
+            future="false",
+            past="false",
+        )
+        resp = self.client.post('success', data=form)
         self.check_content_in_response('Wait a minute', resp)
 
     def test_clear(self):
-        url = ('clear?task_id=runme_1&dag_id=example_bash_operator&'
-               'execution_date={}&upstream=false&downstream=false&future=false&past=false'
-               .format(self.percent_encode(self.default_date)))
-        resp = self.client.get(url)
+        form = dict(
+            task_id="runme_1",
+            dag_id="example_bash_operator",
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
+            upstream="false",
+            downstream="false",
+            future="false",
+            past="false",
+        )
+        resp = self.client.post("clear", data=form)
         self.check_content_in_response(['example_bash_operator', 'Wait a minute'], resp)
 
     def test_run(self):
-        url = ('run?task_id=runme_0&dag_id=example_bash_operator&ignore_all_deps=false&'
-               'ignore_ti_state=true&execution_date={}'
-               .format(self.percent_encode(self.default_date)))
-        resp = self.client.get(url)
+        form = dict(
+            task_id="runme_0",
+            dag_id="example_bash_operator",
+            ignore_all_deps="false",
+            ignore_ti_state="true",
+            execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
+        )
+        resp = self.client.post('run', data=form)
         self.check_content_in_response('', resp, resp_code=302)
 
     def test_refresh(self):
-        resp = self.client.get('refresh?dag_id=example_bash_operator')
+        resp = self.client.post('refresh?dag_id=example_bash_operator')
         self.check_content_in_response('', resp, resp_code=302)
 
+    def test_delete_dag_button_normal(self):
+        resp = self.client.get('/', follow_redirects=True)
+        self.check_content_in_response('/delete?dag_id=example_bash_operator', resp)
+        self.check_content_in_response("return confirmDeleteDag(this, 'example_bash_operator')", resp)
+
+    def test_delete_dag_button_for_dag_on_scheduler_only(self):
+        # Test for JIRA AIRFLOW-3233 (PR 4069):
+        # The delete-dag URL should be generated correctly for DAGs
+        # that exist on the scheduler (DB) but not the webserver DagBag
+
+        test_dag_id = "non_existent_dag"
+
+        DM = models.DagModel
+        self.session.query(DM).filter(DM.dag_id == 'example_bash_operator').update({'dag_id': test_dag_id})
+        self.session.commit()
+
+        resp = self.client.get('/', follow_redirects=True)
+        self.check_content_in_response('/delete?dag_id={}'.format(test_dag_id), resp)
+        self.check_content_in_response("return confirmDeleteDag(this, '{}')".format(test_dag_id), resp)
+
+        self.session.query(DM).filter(DM.dag_id == test_dag_id).update({'dag_id': 'example_bash_operator'})
+        self.session.commit()
+
 
 class TestConfigurationView(TestBase):
-    def test_configuration(self):
+
+    def test_configuration_do_not_expose_config(self):
+        self.logout()
+        self.login()
+        conf.set("webserver", "expose_config", "False")
+        resp = self.client.get('configuration', follow_redirects=True)
+        self.check_content_in_response(
+            ['Airflow Configuration', '# Your Airflow administrator chose not to expose the configuration, '
+                                      'most likely for security reasons.'], resp)
+
+    def test_configuration_expose_config(self):
+        self.logout()
+        self.login()
+        conf.set("webserver", "expose_config", "True")
+
         resp = self.client.get('configuration', follow_redirects=True)
         self.check_content_in_response(
             ['Airflow Configuration', 'Running Configuration'], resp)
@@ -431,6 +630,7 @@ def setUp(self):
         current_dir = os.path.dirname(os.path.abspath(__file__))
         logging_config['handlers']['task']['base_log_folder'] = os.path.normpath(
             os.path.join(current_dir, 'test_logs'))
+
         logging_config['handlers']['task']['filename_template'] = \
             '{{ ti.dag_id }}/{{ ti.task_id }}/' \
             '{{ ts | replace(":", ".") }}/{{ try_number }}.log'
@@ -444,20 +644,20 @@ def setUp(self):
         sys.path.append(self.settings_folder)
         conf.set('core', 'logging_config_class', 'airflow_local_settings.LOGGING_CONFIG')
 
-        self.app, self.appbuilder = application.create_app(testing=True)
+        self.app, self.appbuilder = application.create_app(session=Session, testing=True)
         self.app.config['WTF_CSRF_ENABLED'] = False
         self.client = self.app.test_client()
+        settings.configure_orm()
         self.login()
-        self.session = Session()
 
         from airflow.www_rbac.views import dagbag
         dag = DAG(self.DAG_ID, start_date=self.DEFAULT_DATE)
         task = DummyOperator(task_id=self.TASK_ID, dag=dag)
         dagbag.bag_dag(dag, parent_dag=dag, root_dag=dag)
-        ti = TaskInstance(task=task, execution_date=self.DEFAULT_DATE)
-        ti.try_number = 1
-        self.session.merge(ti)
-        self.session.commit()
+        with create_session() as session:
+            self.ti = TaskInstance(task=task, execution_date=self.DEFAULT_DATE)
+            self.ti.try_number = 1
+            session.merge(self.ti)
 
     def tearDown(self):
         logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
@@ -469,14 +669,53 @@ def tearDown(self):
         self.logout()
         super(TestLogView, self).tearDown()
 
-    def test_get_file_task_log(self):
+    @parameterized.expand([
+        [State.NONE, 0, 0],
+        [State.UP_FOR_RETRY, 2, 2],
+        [State.UP_FOR_RESCHEDULE, 0, 1],
+        [State.UP_FOR_RESCHEDULE, 1, 2],
+        [State.RUNNING, 1, 1],
+        [State.SUCCESS, 1, 1],
+        [State.FAILED, 3, 3],
+    ])
+    def test_get_file_task_log(self, state, try_number, expected_num_logs_visible):
+        with create_session() as session:
+            self.ti.state = state
+            self.ti.try_number = try_number
+            session.merge(self.ti)
+
         response = self.client.get(
-            TestLogView.ENDPOINT,
-            follow_redirects=True,
-        )
+            TestLogView.ENDPOINT, data=dict(
+                username='test',
+                password='test'), follow_redirects=True)
         self.assertEqual(response.status_code, 200)
-        self.assertIn('Log by attempts',
-                      response.data.decode('utf-8'))
+        self.assertIn('Log by attempts', response.data.decode('utf-8'))
+        for num in range(1, expected_num_logs_visible + 1):
+            self.assertIn('try-{}'.format(num), response.data.decode('utf-8'))
+        self.assertNotIn('try-0', response.data.decode('utf-8'))
+        self.assertNotIn('try-{}'.format(expected_num_logs_visible + 1), response.data.decode('utf-8'))
+
+    def test_get_logs_with_metadata_as_download_file(self):
+        url_template = "get_logs_with_metadata?dag_id={}&" \
+                       "task_id={}&execution_date={}&" \
+                       "try_number={}&metadata={}&format=file"
+        try_number = 1
+        url = url_template.format(self.DAG_ID,
+                                  self.TASK_ID,
+                                  quote_plus(self.DEFAULT_DATE.isoformat()),
+                                  try_number,
+                                  json.dumps({}))
+        response = self.client.get(url)
+        expected_filename = '{}/{}/{}/{}.log'.format(self.DAG_ID,
+                                                     self.TASK_ID,
+                                                     self.DEFAULT_DATE.isoformat(),
+                                                     try_number)
+
+        content_disposition = response.headers.get('Content-Disposition')
+        self.assertTrue(content_disposition.startswith('attachment'))
+        self.assertTrue(expected_filename in content_disposition)
+        self.assertEqual(200, response.status_code)
+        self.assertIn('Log for testing.', response.data.decode('utf-8'))
 
     def test_get_logs_with_metadata(self):
         url_template = "get_logs_with_metadata?dag_id={}&" \
@@ -487,7 +726,10 @@ def test_get_logs_with_metadata(self):
                                                 self.TASK_ID,
                                                 quote_plus(self.DEFAULT_DATE.isoformat()),
                                                 1,
-                                                json.dumps({})), follow_redirects=True)
+                                                json.dumps({})), data=dict(
+                                                    username='test',
+                                                    password='test'),
+                            follow_redirects=True)
 
         self.assertIn('"message":', response.data.decode('utf-8'))
         self.assertIn('"metadata":', response.data.decode('utf-8'))
@@ -502,7 +744,10 @@ def test_get_logs_with_null_metadata(self):
             self.client.get(url_template.format(self.DAG_ID,
                                                 self.TASK_ID,
                                                 quote_plus(self.DEFAULT_DATE.isoformat()),
-                                                1), follow_redirects=True)
+                                                1), data=dict(
+                                                    username='test',
+                                                    password='test'),
+                            follow_redirects=True)
 
         self.assertIn('"message":', response.data.decode('utf-8'))
         self.assertIn('"metadata":', response.data.decode('utf-8'))
@@ -512,7 +757,10 @@ def test_get_logs_with_null_metadata(self):
 
 class TestVersionView(TestBase):
     def test_version(self):
-        resp = self.client.get('version', follow_redirects=True)
+        resp = self.client.get('version', data=dict(
+            username='test',
+            password='test'
+        ), follow_redirects=True)
         self.check_content_in_response('Version Info', resp)
 
 
@@ -576,8 +824,9 @@ def test_with_default_parameters(self):
         Should set base date to current date (not asserted)
         """
         response = self.test.client.get(
-            self.endpoint
-        )
+            self.endpoint, data=dict(
+                username='test',
+                password='test'), follow_redirects=True)
         self.test.assertEqual(response.status_code, 200)
         data = response.data.decode('utf-8')
         self.test.assertIn('Base date:', data)
@@ -597,7 +846,11 @@ def test_with_execution_date_parameter_only(self):
         """
         response = self.test.client.get(
             self.endpoint + '&execution_date={}'.format(
-                self.runs[1].execution_date.isoformat())
+                self.runs[1].execution_date.isoformat()),
+            data=dict(
+                username='test',
+                password='test'
+            ), follow_redirects=True
         )
         self.test.assertEqual(response.status_code, 200)
         data = response.data.decode('utf-8')
@@ -620,7 +873,11 @@ def test_with_base_date_and_num_runs_parmeters_only(self):
         """
         response = self.test.client.get(
             self.endpoint + '&base_date={}&num_runs=2'.format(
-                self.runs[1].execution_date.isoformat())
+                self.runs[1].execution_date.isoformat()),
+            data=dict(
+                username='test',
+                password='test'
+            ), follow_redirects=True
         )
         self.test.assertEqual(response.status_code, 200)
         data = response.data.decode('utf-8')
@@ -642,7 +899,11 @@ def test_with_base_date_and_num_runs_and_execution_date_outside(self):
         response = self.test.client.get(
             self.endpoint + '&base_date={}&num_runs=42&execution_date={}'.format(
                 self.runs[1].execution_date.isoformat(),
-                self.runs[0].execution_date.isoformat())
+                self.runs[0].execution_date.isoformat()),
+            data=dict(
+                username='test',
+                password='test'
+            ), follow_redirects=True
         )
         self.test.assertEqual(response.status_code, 200)
         data = response.data.decode('utf-8')
@@ -664,7 +925,11 @@ def test_with_base_date_and_num_runs_and_execution_date_within(self):
         response = self.test.client.get(
             self.endpoint + '&base_date={}&num_runs=5&execution_date={}'.format(
                 self.runs[2].execution_date.isoformat(),
-                self.runs[3].execution_date.isoformat())
+                self.runs[3].execution_date.isoformat()),
+            data=dict(
+                username='test',
+                password='test'
+            ), follow_redirects=True
         )
         self.test.assertEqual(response.status_code, 200)
         data = response.data.decode('utf-8')
@@ -753,5 +1018,625 @@ def test_dt_nr_dr_form_with_base_date_and_num_runs_and_execution_date_within(sel
         self.tester.test_with_base_date_and_num_runs_and_execution_date_within()
 
 
+class TestDagACLView(TestBase):
+    """
+    Test Airflow DAG acl
+    """
+    default_date = timezone.datetime(2018, 6, 1)
+    run_id = "test_{}".format(models.DagRun.id_for_date(default_date))
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestDagACLView, cls).setUpClass()
+
+    def cleanup_dagruns(self):
+        DR = models.DagRun
+        dag_ids = ['example_bash_operator',
+                   'example_subdag_operator']
+        (self.session
+             .query(DR)
+             .filter(DR.dag_id.in_(dag_ids))
+             .filter(DR.run_id == self.run_id)
+             .delete(synchronize_session='fetch'))
+        self.session.commit()
+
+    def prepare_dagruns(self):
+        dagbag = models.DagBag(include_examples=True)
+        self.bash_dag = dagbag.dags['example_bash_operator']
+        self.sub_dag = dagbag.dags['example_subdag_operator']
+
+        self.bash_dagrun = self.bash_dag.create_dagrun(
+            run_id=self.run_id,
+            execution_date=self.default_date,
+            start_date=timezone.utcnow(),
+            state=State.RUNNING)
+
+        self.sub_dagrun = self.sub_dag.create_dagrun(
+            run_id=self.run_id,
+            execution_date=self.default_date,
+            start_date=timezone.utcnow(),
+            state=State.RUNNING)
+
+    def setUp(self):
+        super(TestDagACLView, self).setUp()
+        self.cleanup_dagruns()
+        self.prepare_dagruns()
+        self.logout()
+        self.appbuilder.sm.sync_roles()
+        self.add_permission_for_role()
+
+    def login(self, username=None, password=None):
+        role_admin = self.appbuilder.sm.find_role('Admin')
+        tester = self.appbuilder.sm.find_user(username='test')
+        if not tester:
+            self.appbuilder.sm.add_user(
+                username='test',
+                first_name='test',
+                last_name='test',
+                email='test@fab.org',
+                role=role_admin,
+                password='test')
+
+        role_user = self.appbuilder.sm.find_role('User')
+        test_user = self.appbuilder.sm.find_user(username='test_user')
+        if not test_user:
+            self.appbuilder.sm.add_user(
+                username='test_user',
+                first_name='test_user',
+                last_name='test_user',
+                email='test_user@fab.org',
+                role=role_user,
+                password='test_user')
+
+        dag_acl_role = self.appbuilder.sm.add_role('dag_acl_tester')
+        dag_tester = self.appbuilder.sm.find_user(username='dag_tester')
+        if not dag_tester:
+            self.appbuilder.sm.add_user(
+                username='dag_tester',
+                first_name='dag_test',
+                last_name='dag_test',
+                email='dag_test@fab.org',
+                role=dag_acl_role,
+                password='dag_test')
+
+        # create an user without permission
+        dag_no_role = self.appbuilder.sm.add_role('dag_acl_faker')
+        dag_faker = self.appbuilder.sm.find_user(username='dag_faker')
+        if not dag_faker:
+            self.appbuilder.sm.add_user(
+                username='dag_faker',
+                first_name='dag_faker',
+                last_name='dag_faker',
+                email='dag_fake@fab.org',
+                role=dag_no_role,
+                password='dag_faker')
+
+        # create an user with only read permission
+        dag_read_only_role = self.appbuilder.sm.add_role('dag_acl_read_only')
+        dag_read_only = self.appbuilder.sm.find_user(username='dag_read_only')
+        if not dag_read_only:
+            self.appbuilder.sm.add_user(
+                username='dag_read_only',
+                first_name='dag_read_only',
+                last_name='dag_read_only',
+                email='dag_read_only@fab.org',
+                role=dag_read_only_role,
+                password='dag_read_only')
+
+        # create an user that has all dag access
+        all_dag_role = self.appbuilder.sm.add_role('all_dag_role')
+        all_dag_tester = self.appbuilder.sm.find_user(username='all_dag_user')
+        if not all_dag_tester:
+            self.appbuilder.sm.add_user(
+                username='all_dag_user',
+                first_name='all_dag_user',
+                last_name='all_dag_user',
+                email='all_dag_user@fab.org',
+                role=all_dag_role,
+                password='all_dag_user')
+
+        user = username if username else 'dag_tester'
+        passwd = password if password else 'dag_test'
+
+        return self.client.post('/login/', data=dict(
+            username=user,
+            password=passwd
+        ))
+
+    def logout(self):
+        return self.client.get('/logout/')
+
+    def add_permission_for_role(self):
+        self.logout()
+        self.login(username='test',
+                   password='test')
+        perm_on_dag = self.appbuilder.sm.\
+            find_permission_view_menu('can_dag_edit', 'example_bash_operator')
+        dag_tester_role = self.appbuilder.sm.find_role('dag_acl_tester')
+        self.appbuilder.sm.add_permission_role(dag_tester_role, perm_on_dag)
+
+        perm_on_all_dag = self.appbuilder.sm.\
+            find_permission_view_menu('can_dag_edit', 'all_dags')
+        all_dag_role = self.appbuilder.sm.find_role('all_dag_role')
+        self.appbuilder.sm.add_permission_role(all_dag_role, perm_on_all_dag)
+
+        role_user = self.appbuilder.sm.find_role('User')
+        self.appbuilder.sm.add_permission_role(role_user, perm_on_all_dag)
+
+        read_only_perm_on_dag = self.appbuilder.sm.\
+            find_permission_view_menu('can_dag_read', 'example_bash_operator')
+        dag_read_only_role = self.appbuilder.sm.find_role('dag_acl_read_only')
+        self.appbuilder.sm.add_permission_role(dag_read_only_role, read_only_perm_on_dag)
+
+    def test_permission_exist(self):
+        self.logout()
+        self.login(username='test',
+                   password='test')
+        test_view_menu = self.appbuilder.sm.find_view_menu('example_bash_operator')
+        perms_views = self.appbuilder.sm.find_permissions_view_menu(test_view_menu)
+        self.assertEqual(len(perms_views), 2)
+        # each dag view will create one write, and one read permission
+        self.assertTrue(str(perms_views[0]).startswith('can dag'))
+        self.assertTrue(str(perms_views[1]).startswith('can dag'))
+
+    def test_role_permission_associate(self):
+        self.logout()
+        self.login(username='test',
+                   password='test')
+        test_role = self.appbuilder.sm.find_role('dag_acl_tester')
+        perms = set([str(perm) for perm in test_role.permissions])
+        self.assertIn('can dag edit on example_bash_operator', perms)
+        self.assertNotIn('can dag read on example_bash_operator', perms)
+
+    def test_index_success(self):
+        self.logout()
+        self.login()
+        resp = self.client.get('/', follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_index_failure(self):
+        self.logout()
+        self.login()
+        resp = self.client.get('/', follow_redirects=True)
+        # The user can only access/view example_bash_operator dag.
+        self.check_content_not_in_response('example_subdag_operator', resp)
+
+    def test_index_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        resp = self.client.get('/', follow_redirects=True)
+        # The all dag user can access/view all dags.
+        self.check_content_in_response('example_subdag_operator', resp)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_dag_stats_success(self):
+        self.logout()
+        self.login()
+        resp = self.client.get('dag_stats', follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_dag_stats_failure(self):
+        self.logout()
+        self.login()
+        resp = self.client.get('dag_stats', follow_redirects=True)
+        self.check_content_not_in_response('example_subdag_operator', resp)
+
+    def test_dag_stats_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        resp = self.client.get('dag_stats', follow_redirects=True)
+        self.check_content_in_response('example_subdag_operator', resp)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_task_stats_success(self):
+        self.logout()
+        self.login()
+        resp = self.client.get('task_stats', follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_task_stats_failure(self):
+        self.logout()
+        self.login()
+        resp = self.client.get('task_stats', follow_redirects=True)
+        self.check_content_not_in_response('example_subdag_operator', resp)
+
+    def test_task_stats_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        resp = self.client.get('task_stats', follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+        self.check_content_in_response('example_subdag_operator', resp)
+
+    def test_code_success(self):
+        self.logout()
+        self.login()
+        url = 'code?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_code_failure(self):
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        url = 'code?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('example_bash_operator', resp)
+
+    def test_code_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        url = 'code?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+        url = 'code?dag_id=example_subdag_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_subdag_operator', resp)
+
+    def test_dag_details_success(self):
+        self.logout()
+        self.login()
+        url = 'dag_details?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('DAG details', resp)
+
+    def test_dag_details_failure(self):
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        url = 'dag_details?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('DAG details', resp)
+
+    def test_dag_details_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        url = 'dag_details?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+        url = 'dag_details?dag_id=example_subdag_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_subdag_operator', resp)
+
+    def test_pickle_info_success(self):
+        self.logout()
+        self.login()
+        url = 'pickle_info?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.assertEqual(resp.status_code, 200)
+
+    def test_rendered_success(self):
+        self.logout()
+        self.login()
+        url = ('rendered?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('Rendered Template', resp)
+
+    def test_rendered_failure(self):
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        url = ('rendered?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('Rendered Template', resp)
+
+    def test_rendered_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        url = ('rendered?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('Rendered Template', resp)
+
+    def test_task_success(self):
+        self.logout()
+        self.login()
+        url = ('task?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('Task Instance Details', resp)
+
+    def test_task_failure(self):
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        url = ('task?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('Task Instance Details', resp)
+
+    def test_task_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        url = ('task?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('Task Instance Details', resp)
+
+    def test_xcom_success(self):
+        self.logout()
+        self.login()
+        url = ('xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('XCom', resp)
+
+    def test_xcom_failure(self):
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        url = ('xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('XCom', resp)
+
+    def test_xcom_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        url = ('xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('XCom', resp)
+
+    def test_run_success(self):
+        self.logout()
+        self.login()
+        form = dict(
+            task_id="runme_0",
+            dag_id="example_bash_operator",
+            ignore_all_deps="false",
+            ignore_ti_state="true",
+            execution_date=self.default_date,
+        )
+        resp = self.client.post('run', data=form)
+        self.check_content_in_response('', resp, resp_code=302)
+
+    def test_run_success_for_all_dag_user(self):
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        form = dict(
+            task_id="runme_0",
+            dag_id="example_bash_operator",
+            ignore_all_deps="false",
+            ignore_ti_state="true",
+            execution_date=self.default_date
+        )
+        resp = self.client.post('run', data=form)
+        self.check_content_in_response('', resp, resp_code=302)
+
+    def test_blocked_success(self):
+        url = 'blocked'
+        self.logout()
+        self.login()
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_blocked_success_for_all_dag_user(self):
+        url = 'blocked'
+        self.logout()
+        self.login(username='all_dag_user',
+                   password='all_dag_user')
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+        self.check_content_in_response('example_subdag_operator', resp)
+
+    def test_failed_success(self):
+        self.logout()
+        self.login()
+        form = dict(
+            task_id="run_this_last",
+            dag_id="example_bash_operator",
+            execution_date=self.default_date,
+            upstream="false",
+            downstream="false",
+            future="false",
+            past="false",
+        )
+        resp = self.client.post('failed', data=form)
+        self.check_content_in_response('Redirecting', resp, 302)
+
+    def test_duration_success(self):
+        url = 'duration?days=30&dag_id=example_bash_operator'
+        self.logout()
+        self.login()
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_duration_failure(self):
+        url = 'duration?days=30&dag_id=example_bash_operator'
+        self.logout()
+        # login as an user without permissions
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('example_bash_operator', resp)
+
+    def test_tries_success(self):
+        url = 'tries?days=30&dag_id=example_bash_operator'
+        self.logout()
+        self.login()
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_tries_failure(self):
+        url = 'tries?days=30&dag_id=example_bash_operator'
+        self.logout()
+        # login as an user without permissions
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('example_bash_operator', resp)
+
+    def test_landing_times_success(self):
+        url = 'landing_times?days=30&dag_id=example_bash_operator'
+        self.logout()
+        self.login()
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_landing_times_failure(self):
+        url = 'landing_times?days=30&dag_id=example_bash_operator'
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('example_bash_operator', resp)
+
+    def test_paused_success(self):
+        # post request failure won't test
+        url = 'paused?dag_id=example_bash_operator&is_paused=false'
+        self.logout()
+        self.login()
+        resp = self.client.post(url, follow_redirects=True)
+        self.check_content_in_response('OK', resp)
+
+    def test_refresh_success(self):
+        self.logout()
+        self.login()
+        resp = self.client.post('refresh?dag_id=example_bash_operator')
+        self.check_content_in_response('', resp, resp_code=302)
+
+    def test_gantt_success(self):
+        url = 'gantt?dag_id=example_bash_operator'
+        self.logout()
+        self.login()
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('example_bash_operator', resp)
+
+    def test_gantt_failure(self):
+        url = 'gantt?dag_id=example_bash_operator'
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('example_bash_operator', resp)
+
+    def test_success_fail_for_read_only_role(self):
+        # succcess endpoint need can_dag_edit, which read only role can not access
+        self.logout()
+        self.login(username='dag_read_only',
+                   password='dag_read_only')
+
+        form = dict(
+            task_id="run_this_last",
+            dag_id="example_bash_operator",
+            execution_date=self.default_date,
+            upstream="false",
+            downstream="false",
+            future="false",
+            past="false",
+        )
+        resp = self.client.post('success', data=form)
+        self.check_content_not_in_response('Wait a minute', resp, resp_code=302)
+
+    def test_tree_success_for_read_only_role(self):
+        # tree view only allows can_dag_read, which read only role could access
+        self.logout()
+        self.login(username='dag_read_only',
+                   password='dag_read_only')
+
+        url = 'tree?dag_id=example_bash_operator'
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('runme_1', resp)
+
+    def test_log_success(self):
+        self.logout()
+        self.login()
+        url = ('log?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('Log by attempts', resp)
+        url = ('get_logs_with_metadata?task_id=runme_0&dag_id=example_bash_operator&'
+               'execution_date={}&try_number=1&metadata=null'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('"message":', resp)
+        self.check_content_in_response('"metadata":', resp)
+
+    def test_log_failure(self):
+        self.logout()
+        self.login(username='dag_faker',
+                   password='dag_faker')
+        url = ('log?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('Log by attempts', resp)
+        url = ('get_logs_with_metadata?task_id=runme_0&dag_id=example_bash_operator&'
+               'execution_date={}&try_number=1&metadata=null'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_not_in_response('"message":', resp)
+        self.check_content_not_in_response('"metadata":', resp)
+
+    def test_log_success_for_user(self):
+        self.logout()
+        self.login(username='test_user',
+                   password='test_user')
+        url = ('log?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('Log by attempts', resp)
+        url = ('get_logs_with_metadata?task_id=runme_0&dag_id=example_bash_operator&'
+               'execution_date={}&try_number=1&metadata=null'
+               .format(self.percent_encode(self.default_date)))
+        resp = self.client.get(url, follow_redirects=True)
+        self.check_content_in_response('"message":', resp)
+        self.check_content_in_response('"metadata":', resp)
+
+
+class TestTaskInstanceView(TestBase):
+    TI_ENDPOINT = '/taskinstance/list/?_flt_0_execution_date={}'
+
+    def test_start_date_filter(self):
+        resp = self.client.get(self.TI_ENDPOINT.format(
+            self.percent_encode('2018-10-09 22:44:31')))
+        # We aren't checking the logic of the date filter itself (that is built
+        # in to FAB) but simply that our UTC conversion was run - i.e. it
+        # doesn't blow up!
+        self.check_content_in_response('List Task Instance', resp)
+        pass
+
+
+class TestTriggerDag(TestBase):
+
+    def setUp(self):
+        super(TestTriggerDag, self).setUp()
+        self.session = Session()
+        models.DagBag().get_dag("example_bash_operator").sync_to_db(session=self.session)
+
+    def test_trigger_dag_button_normal_exist(self):
+        resp = self.client.get('/', follow_redirects=True)
+        self.assertIn('/trigger?dag_id=example_bash_operator', resp.data.decode('utf-8'))
+        self.assertIn("return confirmDeleteDag(this, 'example_bash_operator')", resp.data.decode('utf-8'))
+
+    @unittest.skipIf('mysql' in conf.conf.get('core', 'sql_alchemy_conn'),
+                     "flaky when run on mysql")
+    def test_trigger_dag_button(self):
+
+        test_dag_id = "example_bash_operator"
+
+        DR = models.DagRun
+        self.session.query(DR).delete()
+        self.session.commit()
+
+        self.client.post('trigger?dag_id={}'.format(test_dag_id))
+
+        run = self.session.query(DR).filter(DR.dag_id == test_dag_id).first()
+        self.assertIsNotNone(run)
+        self.assertIn("manual__", run.run_id)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tox.ini b/tox.ini
index 88fcd426183bd..0480c233dae25 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,8 +17,8 @@
 # under the License.
 
 [tox]
-envlist = flake8,{py27,py35}-backend_{mysql,sqlite,postgres}
-skipsdist=True
+envlist = flake8,{py27,py35}-backend_{mysql,sqlite,postgres}-env_{docker,kubernetes}
+skipsdist = True
 
 [global]
 wheel_dir = {homedir}/.wheelhouse
@@ -26,14 +26,10 @@ find_links =
     {homedir}/.wheelhouse
     {homedir}/.pip-cache
 
-[flake8]
-max-line-length = 90
-ignore = E731
-
 [testenv]
 deps =
     wheel
-    coveralls
+    codecov
 
 basepython =
   py27: python2.7
@@ -43,45 +39,32 @@ setenv =
   HADOOP_DISTRO=cdh
   HADOOP_HOME=/tmp/hadoop-cdh
   HADOOP_OPTS=-D/tmp/krb5.conf
-  MINICLUSTER_HOME=/tmp/minicluster-1.1-SNAPSHOT
   HIVE_HOME=/tmp/hive
-  backend_mysql: AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://root@localhost/airflow
-  backend_postgres: AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres@localhost/airflow
-  backend_sqlite: AIRFLOW__CORE__SQL_ALCHEMY_CONN=sqlite:///{homedir}/airflow/airflow.db
+  MINICLUSTER_HOME=/tmp/minicluster-1.1-SNAPSHOT
+  KRB5_CONFIG=/etc/krb5.conf
+  KRB5_KTNAME=/etc/airflow.keytab
+  backend_mysql: AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://root@mysql/airflow
+  backend_mysql: AIRFLOW__CELERY__RESULT_BACKEND=db+mysql://root@mysql/airflow
+  backend_postgres: AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:airflow@postgres/airflow
+  backend_postgres: AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:airflow@postgres/airflow
+  backend_sqlite: AIRFLOW__CORE__SQL_ALCHEMY_CONN=sqlite:///{homedir}/airflow.db
   backend_sqlite: AIRFLOW__CORE__EXECUTOR=SequentialExecutor
 
-passenv =
-    HOME
-    JAVA_HOME
-    TRAVIS
-    TRAVIS_BRANCH
-    TRAVIS_BUILD_DIR
-    TRAVIS_JOB_ID
-    USER
-    TRAVIS_CACHE
-    TRAVIS_PULL_REQUEST
-    PATH
-    BOTO_CONFIG
-    KRB5_CONFIG
-    KRB5_KTNAME
+passenv = *
 
 commands =
-  pip wheel -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse -e .[devel_ci]
-  pip install --find-links={homedir}/.wheelhouse --no-index -e .[devel_ci]
-  sudo {toxinidir}/scripts/ci/setup_kdc.sh
-  {toxinidir}/scripts/ci/setup_env.sh
-  {toxinidir}/scripts/ci/ldap.sh
-  {toxinidir}/scripts/ci/load_fixtures.sh
-  {toxinidir}/scripts/ci/load_data.sh
-  {toxinidir}/scripts/ci/run_tests.sh []
-  {toxinidir}/scripts/ci/check-license.sh
+  pip wheel --no-use-pep517 --progress-bar off -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse -e .[devel_ci]
+  pip install --no-use-pep517 --progress-bar off --find-links={homedir}/.wheelhouse --no-index -e .[devel_ci]
+  env_docker: {toxinidir}/scripts/ci/1-setup-env.sh
+  env_docker: {toxinidir}/scripts/ci/2-setup-kdc.sh
+  backend_mysql: {toxinidir}/scripts/ci/3-setup-mysql.sh
+  {toxinidir}/scripts/ci/5-run-tests.sh []
+  codecov -e TOXENV
 
 [testenv:flake8]
 basepython = python3
 
 deps =
-    flake8
-
-commands =
-    {toxinidir}/scripts/ci/flake8_diff.sh
+    flake8>=3.6.0
 
+commands = flake8